1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
4	*
5	* Parts came from builtin-{top,stat,record}.c, see those files for further
6	* copyright notes.
7	*/
8	/*
9	* Powerpc needs __SANE_USERSPACE_TYPES__ before <linux/types.h> to select
10	* 'int-ll64.h' and avoid compile warnings when printing __u64 with %llu.
11	*/
12	#define __SANE_USERSPACE_TYPES__
13
14	#include <byteswap.h>
15	#include <errno.h>
16	#include <inttypes.h>
17	#include <linux/bitops.h>
18	#include <api/fs/fs.h>
19	#include <api/fs/tracing_path.h>
20	#include <linux/hw_breakpoint.h>
21	#include <linux/perf_event.h>
22	#include <linux/compiler.h>
23	#include <linux/err.h>
24	#include <linux/zalloc.h>
25	#include <sys/ioctl.h>
26	#include <sys/resource.h>
27	#include <sys/syscall.h>
28	#include <sys/types.h>
29	#include <dirent.h>
30	#include <stdlib.h>
31	#include <perf/evsel.h>
32	#include "asm/bug.h"
33	#include "bpf_counter.h"
34	#include "callchain.h"
35	#include "cgroup.h"
36	#include "counts.h"
37	#include "event.h"
38	#include "evsel.h"
39	#include "time-utils.h"
40	#include "util/env.h"
41	#include "util/evsel_config.h"
42	#include "util/evsel_fprintf.h"
43	#include "evlist.h"
44	#include <perf/cpumap.h>
45	#include "thread_map.h"
46	#include "target.h"
47	#include "perf_regs.h"
48	#include "record.h"
49	#include "debug.h"
50	#include "trace-event.h"
51	#include "session.h"
52	#include "stat.h"
53	#include "string2.h"
54	#include "memswap.h"
55	#include "util.h"
56	#include "util/hashmap.h"
57	#include "off_cpu.h"
58	#include "pmu.h"
59	#include "pmus.h"
60	#include "drm_pmu.h"
61	#include "hwmon_pmu.h"
62	#include "tool_pmu.h"
63	#include "tp_pmu.h"
64	#include "rlimit.h"
65	#include "../perf-sys.h"
66	#include "util/parse-branch-options.h"
67	#include "util/bpf-filter.h"
68	#include "util/hist.h"
69	#include <internal/xyarray.h>
70	#include <internal/lib.h>
71	#include <internal/threadmap.h>
72	#include "util/intel-tpebs.h"
73
74	#include <linux/ctype.h>
75
76	#ifdef HAVE_LIBTRACEEVENT
77	#include <event-parse.h>
78	#endif
79
80	struct perf_missing_features perf_missing_features;
81
82	static clockid_t clockid;
83
84	static int evsel__no_extra_init(struct evsel *evsel __maybe_unused)
85	{
86	return 0;
87	}
88
89	static bool test_attr__enabled(void)
90	{
91	static bool test_attr__enabled;
92	static bool test_attr__enabled_tested;
93
94	if (!test_attr__enabled_tested) {
95	char *dir = getenv("PERF_TEST_ATTR");
96
97	test_attr__enabled = (dir != NULL);
98	test_attr__enabled_tested = true;
99	}
100	return test_attr__enabled;
101	}
102
103	#define __WRITE_ASS(str, fmt, data) \
104	do { \
105	if (fprintf(file, #str "=%"fmt "\n", data) < 0) { \
106	perror("test attr - failed to write event file"); \
107	fclose(file); \
108	return -1; \
109	} \
110	} while (0)
111
112	#define WRITE_ASS(field, fmt) __WRITE_ASS(field, fmt, attr->field)
113
114	static int store_event(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu,
115	int fd, int group_fd, unsigned long flags)
116	{
117	FILE *file;
118	char path[PATH_MAX];
119	char *dir = getenv("PERF_TEST_ATTR");
120
121	snprintf(buf: path, PATH_MAX, fmt: "%s/event-%d-%llu-%d", dir,
122	attr->type, attr->config, fd);
123
124	file = fopen(path, "w+");
125	if (!file) {
126	perror("test attr - failed to open event file");
127	return -1;
128	}
129
130	if (fprintf(file, "[event-%d-%llu-%d]\n",
131	attr->type, attr->config, fd) < 0) {
132	perror("test attr - failed to write event file");
133	fclose(file);
134	return -1;
135	}
136
137	/* syscall arguments */
138	__WRITE_ASS(fd, "d", fd);
139	__WRITE_ASS(group_fd, "d", group_fd);
140	__WRITE_ASS(cpu, "d", cpu.cpu);
141	__WRITE_ASS(pid, "d", pid);
142	__WRITE_ASS(flags, "lu", flags);
143
144	/* struct perf_event_attr */
145	WRITE_ASS(type, PRIu32);
146	WRITE_ASS(size, PRIu32);
147	WRITE_ASS(config, "llu");
148	WRITE_ASS(sample_period, "llu");
149	WRITE_ASS(sample_type, "llu");
150	WRITE_ASS(read_format, "llu");
151	WRITE_ASS(disabled, "d");
152	WRITE_ASS(inherit, "d");
153	WRITE_ASS(pinned, "d");
154	WRITE_ASS(exclusive, "d");
155	WRITE_ASS(exclude_user, "d");
156	WRITE_ASS(exclude_kernel, "d");
157	WRITE_ASS(exclude_hv, "d");
158	WRITE_ASS(exclude_idle, "d");
159	WRITE_ASS(mmap, "d");
160	WRITE_ASS(comm, "d");
161	WRITE_ASS(freq, "d");
162	WRITE_ASS(inherit_stat, "d");
163	WRITE_ASS(enable_on_exec, "d");
164	WRITE_ASS(task, "d");
165	WRITE_ASS(watermark, "d");
166	WRITE_ASS(precise_ip, "d");
167	WRITE_ASS(mmap_data, "d");
168	WRITE_ASS(sample_id_all, "d");
169	WRITE_ASS(exclude_host, "d");
170	WRITE_ASS(exclude_guest, "d");
171	WRITE_ASS(exclude_callchain_kernel, "d");
172	WRITE_ASS(exclude_callchain_user, "d");
173	WRITE_ASS(mmap2, "d");
174	WRITE_ASS(comm_exec, "d");
175	WRITE_ASS(context_switch, "d");
176	WRITE_ASS(write_backward, "d");
177	WRITE_ASS(namespaces, "d");
178	WRITE_ASS(use_clockid, "d");
179	WRITE_ASS(wakeup_events, PRIu32);
180	WRITE_ASS(bp_type, PRIu32);
181	WRITE_ASS(config1, "llu");
182	WRITE_ASS(config2, "llu");
183	WRITE_ASS(branch_sample_type, "llu");
184	WRITE_ASS(sample_regs_user, "llu");
185	WRITE_ASS(sample_stack_user, PRIu32);
186
187	fclose(file);
188	return 0;
189	}
190
191	#undef __WRITE_ASS
192	#undef WRITE_ASS
193
194	static void test_attr__open(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu,
195	int fd, int group_fd, unsigned long flags)
196	{
197	int errno_saved = errno;
198
199	if ((fd != -1) && store_event(attr, pid, cpu: cpu, fd, group_fd, flags)) {
200	pr_err("test attr FAILED");
201	exit(128);
202	}
203
204	errno = errno_saved;
205	}
206
207	static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
208	{
209	}
210
211	static struct {
212	size_t size;
213	int (*init)(struct evsel *evsel);
214	void (*fini)(struct evsel *evsel);
215	} perf_evsel__object = {
216	.size = sizeof(struct evsel),
217	.init = evsel__no_extra_init,
218	.fini = evsel__no_extra_fini,
219	};
220
221	int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel),
222	void (*fini)(struct evsel *evsel))
223	{
224
225	if (object_size == 0)
226	goto set_methods;
227
228	if (perf_evsel__object.size > object_size)
229	return -EINVAL;
230
231	perf_evsel__object.size = object_size;
232
233	set_methods:
234	if (init != NULL)
235	perf_evsel__object.init = init;
236
237	if (fini != NULL)
238	perf_evsel__object.fini = fini;
239
240	return 0;
241	}
242
243	const char *evsel__pmu_name(const struct evsel *evsel)
244	{
245	struct perf_pmu *pmu = evsel__find_pmu(evsel);
246
247	if (pmu)
248	return pmu->name;
249
250	return event_type(type: evsel->core.attr.type);
251	}
252
253	#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
254
255	int __evsel__sample_size(u64 sample_type)
256	{
257	u64 mask = sample_type & PERF_SAMPLE_MASK;
258	int size = 0;
259	int i;
260
261	for (i = 0; i < 64; i++) {
262	if (mask & (1ULL << i))
263	size++;
264	}
265
266	size *= sizeof(u64);
267
268	return size;
269	}
270
271	/**
272	* __perf_evsel__calc_id_pos - calculate id_pos.
273	* @sample_type: sample type
274	*
275	* This function returns the position of the event id (PERF_SAMPLE_ID or
276	* PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct
277	* perf_record_sample.
278	*/
279	static int __perf_evsel__calc_id_pos(u64 sample_type)
280	{
281	int idx = 0;
282
283	if (sample_type & PERF_SAMPLE_IDENTIFIER)
284	return 0;
285
286	if (!(sample_type & PERF_SAMPLE_ID))
287	return -1;
288
289	if (sample_type & PERF_SAMPLE_IP)
290	idx += 1;
291
292	if (sample_type & PERF_SAMPLE_TID)
293	idx += 1;
294
295	if (sample_type & PERF_SAMPLE_TIME)
296	idx += 1;
297
298	if (sample_type & PERF_SAMPLE_ADDR)
299	idx += 1;
300
301	return idx;
302	}
303
304	/**
305	* __perf_evsel__calc_is_pos - calculate is_pos.
306	* @sample_type: sample type
307	*
308	* This function returns the position (counting backwards) of the event id
309	* (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if
310	* sample_id_all is used there is an id sample appended to non-sample events.
311	*/
312	static int __perf_evsel__calc_is_pos(u64 sample_type)
313	{
314	int idx = 1;
315
316	if (sample_type & PERF_SAMPLE_IDENTIFIER)
317	return 1;
318
319	if (!(sample_type & PERF_SAMPLE_ID))
320	return -1;
321
322	if (sample_type & PERF_SAMPLE_CPU)
323	idx += 1;
324
325	if (sample_type & PERF_SAMPLE_STREAM_ID)
326	idx += 1;
327
328	return idx;
329	}
330
331	void evsel__calc_id_pos(struct evsel *evsel)
332	{
333	evsel->id_pos = __perf_evsel__calc_id_pos(sample_type: evsel->core.attr.sample_type);
334	evsel->is_pos = __perf_evsel__calc_is_pos(sample_type: evsel->core.attr.sample_type);
335	}
336
337	void __evsel__set_sample_bit(struct evsel *evsel,
338	enum perf_event_sample_format bit)
339	{
340	if (!(evsel->core.attr.sample_type & bit)) {
341	evsel->core.attr.sample_type |= bit;
342	evsel->sample_size += sizeof(u64);
343	evsel__calc_id_pos(evsel);
344	}
345	}
346
347	void __evsel__reset_sample_bit(struct evsel *evsel,
348	enum perf_event_sample_format bit)
349	{
350	if (evsel->core.attr.sample_type & bit) {
351	evsel->core.attr.sample_type &= ~bit;
352	evsel->sample_size -= sizeof(u64);
353	evsel__calc_id_pos(evsel);
354	}
355	}
356
357	void evsel__set_sample_id(struct evsel *evsel,
358	bool can_sample_identifier)
359	{
360	if (can_sample_identifier) {
361	evsel__reset_sample_bit(evsel, ID);
362	evsel__set_sample_bit(evsel, IDENTIFIER);
363	} else {
364	evsel__set_sample_bit(evsel, ID);
365	}
366	evsel->core.attr.read_format |= PERF_FORMAT_ID;
367	}
368
369	/**
370	* evsel__is_function_event - Return whether given evsel is a function
371	* trace event
372	*
373	* @evsel - evsel selector to be tested
374	*
375	* Return %true if event is function trace event
376	*/
377	bool evsel__is_function_event(struct evsel *evsel)
378	{
379	#define FUNCTION_EVENT "ftrace:function"
380
381	return evsel->name &&
382	!strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
383
384	#undef FUNCTION_EVENT
385	}
386
387	void evsel__init(struct evsel *evsel,
388	struct perf_event_attr *attr, int idx)
389	{
390	perf_evsel__init(&evsel->core, attr, idx);
391	evsel->tracking = !idx;
392	evsel->unit = strdup("");
393	evsel->scale = 1.0;
394	evsel->max_events = ULONG_MAX;
395	evsel->evlist = NULL;
396	evsel->bpf_obj = NULL;
397	evsel->bpf_fd = -1;
398	INIT_LIST_HEAD(list: &evsel->config_terms);
399	INIT_LIST_HEAD(list: &evsel->bpf_counter_list);
400	INIT_LIST_HEAD(list: &evsel->bpf_filters);
401	perf_evsel__object.init(evsel);
402	evsel->sample_size = __evsel__sample_size(sample_type: attr->sample_type);
403	evsel__calc_id_pos(evsel);
404	evsel->cmdline_group_boundary = false;
405	evsel->per_pkg_mask = NULL;
406	evsel->collect_stat = false;
407	evsel->group_pmu_name = NULL;
408	evsel->skippable = false;
409	evsel->supported = true;
410	evsel->alternate_hw_config = PERF_COUNT_HW_MAX;
411	evsel->script_output_type = -1; // FIXME: OUTPUT_TYPE_UNSET, see builtin-script.c
412	}
413
414	struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx)
415	{
416	struct evsel *evsel = zalloc(perf_evsel__object.size);
417
418	if (!evsel)
419	return NULL;
420	evsel__init(evsel, attr, idx);
421
422	if (evsel__is_bpf_output(evsel) && !attr->sample_type) {
423	evsel->core.attr.sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
424	PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
425	evsel->core.attr.sample_period = 1;
426	}
427
428	if (evsel__is_clock(evsel)) {
429	free((char *)evsel->unit);
430	evsel->unit = strdup("msec");
431	evsel->scale = 1e-6;
432	}
433
434	return evsel;
435	}
436
437	int copy_config_terms(struct list_head *dst, struct list_head *src)
438	{
439	struct evsel_config_term *pos, *tmp;
440
441	list_for_each_entry(pos, src, list) {
442	tmp = malloc(sizeof(*tmp));
443	if (tmp == NULL)
444	return -ENOMEM;
445
446	*tmp = *pos;
447	if (tmp->free_str) {
448	tmp->val.str = strdup(pos->val.str);
449	if (tmp->val.str == NULL) {
450	free(tmp);
451	return -ENOMEM;
452	}
453	}
454	list_add_tail(new: &tmp->list, head: dst);
455	}
456	return 0;
457	}
458
459	static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
460	{
461	return copy_config_terms(dst: &dst->config_terms, src: &src->config_terms);
462	}
463
464	/**
465	* evsel__clone - create a new evsel copied from @orig
466	* @orig: original evsel
467	*
468	* The assumption is that @orig is not configured nor opened yet.
469	* So we only care about the attributes that can be set while it's parsed.
470	*/
471	struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig)
472	{
473	struct evsel *evsel;
474
475	BUG_ON(orig->core.fd);
476	BUG_ON(orig->counts);
477	BUG_ON(orig->priv);
478	BUG_ON(orig->per_pkg_mask);
479
480	/* cannot handle BPF objects for now */
481	if (orig->bpf_obj)
482	return NULL;
483
484	if (dest)
485	evsel = dest;
486	else
487	evsel = evsel__new(attr: &orig->core.attr);
488
489	if (evsel == NULL)
490	return NULL;
491
492	evsel->core.cpus = perf_cpu_map__get(orig->core.cpus);
493	evsel->core.pmu_cpus = perf_cpu_map__get(orig->core.pmu_cpus);
494	evsel->core.threads = perf_thread_map__get(orig->core.threads);
495	evsel->core.nr_members = orig->core.nr_members;
496	evsel->core.system_wide = orig->core.system_wide;
497	evsel->core.requires_cpu = orig->core.requires_cpu;
498	evsel->core.is_pmu_core = orig->core.is_pmu_core;
499
500	if (orig->name) {
501	evsel->name = strdup(orig->name);
502	if (evsel->name == NULL)
503	goto out_err;
504	}
505	if (orig->group_name) {
506	evsel->group_name = strdup(orig->group_name);
507	if (evsel->group_name == NULL)
508	goto out_err;
509	}
510	if (orig->group_pmu_name) {
511	evsel->group_pmu_name = strdup(orig->group_pmu_name);
512	if (evsel->group_pmu_name == NULL)
513	goto out_err;
514	}
515	if (orig->filter) {
516	evsel->filter = strdup(orig->filter);
517	if (evsel->filter == NULL)
518	goto out_err;
519	}
520	if (orig->metric_id) {
521	evsel->metric_id = strdup(orig->metric_id);
522	if (evsel->metric_id == NULL)
523	goto out_err;
524	}
525	evsel->cgrp = cgroup__get(cgroup: orig->cgrp);
526	#ifdef HAVE_LIBTRACEEVENT
527	if (orig->tp_sys) {
528	evsel->tp_sys = strdup(orig->tp_sys);
529	if (evsel->tp_sys == NULL)
530	goto out_err;
531	}
532	if (orig->tp_name) {
533	evsel->tp_name = strdup(orig->tp_name);
534	if (evsel->tp_name == NULL)
535	goto out_err;
536	}
537	evsel->tp_format = orig->tp_format;
538	#endif
539	evsel->handler = orig->handler;
540	evsel->core.leader = orig->core.leader;
541	evsel->metric_leader = orig->metric_leader;
542
543	evsel->max_events = orig->max_events;
544	zfree(&evsel->unit);
545	if (orig->unit) {
546	evsel->unit = strdup(orig->unit);
547	if (evsel->unit == NULL)
548	goto out_err;
549	}
550	evsel->scale = orig->scale;
551	evsel->snapshot = orig->snapshot;
552	evsel->per_pkg = orig->per_pkg;
553	evsel->percore = orig->percore;
554	evsel->precise_max = orig->precise_max;
555	evsel->is_libpfm_event = orig->is_libpfm_event;
556
557	evsel->exclude_GH = orig->exclude_GH;
558	evsel->sample_read = orig->sample_read;
559	evsel->collect_stat = orig->collect_stat;
560	evsel->weak_group = orig->weak_group;
561	evsel->use_config_name = orig->use_config_name;
562	evsel->pmu = orig->pmu;
563	evsel->first_wildcard_match = orig->first_wildcard_match;
564
565	if (evsel__copy_config_terms(dst: evsel, src: orig) < 0)
566	goto out_err;
567
568	evsel->alternate_hw_config = orig->alternate_hw_config;
569
570	return evsel;
571
572	out_err:
573	evsel__delete(evsel);
574	return NULL;
575	}
576
577	/*
578	* Returns pointer with encoded error via <linux/err.h> interface.
579	*/
580	struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool format)
581	{
582	struct perf_event_attr attr = {
583	.type = PERF_TYPE_TRACEPOINT,
584	.sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
585	PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
586	};
587	struct evsel *evsel = zalloc(perf_evsel__object.size);
588	int err = -ENOMEM, id = -1;
589
590	if (evsel == NULL)
591	goto out_err;
592
593
594	if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
595	goto out_free;
596
597	#ifdef HAVE_LIBTRACEEVENT
598	evsel->tp_sys = strdup(sys);
599	if (!evsel->tp_sys)
600	goto out_free;
601
602	evsel->tp_name = strdup(name);
603	if (!evsel->tp_name)
604	goto out_free;
605	#endif
606
607	event_attr_init(attr: &attr);
608
609	if (format) {
610	id = tp_pmu__id(sys, name);
611	if (id < 0) {
612	err = id;
613	goto out_free;
614	}
615	}
616	attr.config = (__u64)id;
617	attr.sample_period = 1;
618	evsel__init(evsel, attr: &attr, idx);
619	return evsel;
620
621	out_free:
622	zfree(&evsel->name);
623	#ifdef HAVE_LIBTRACEEVENT
624	zfree(&evsel->tp_sys);
625	zfree(&evsel->tp_name);
626	#endif
627	free(evsel);
628	out_err:
629	return ERR_PTR(error: err);
630	}
631
632	#ifdef HAVE_LIBTRACEEVENT
633	struct tep_event *evsel__tp_format(struct evsel *evsel)
634	{
635	struct tep_event *tp_format = evsel->tp_format;
636
637	if (tp_format)
638	return tp_format;
639
640	if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT)
641	return NULL;
642
643	if (!evsel->tp_sys)
644	tp_format = trace_event__tp_format_id(evsel->core.attr.config);
645	else
646	tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name);
647
648	if (IS_ERR(tp_format)) {
649	int err = -PTR_ERR(evsel->tp_format);
650
651	pr_err("Error getting tracepoint format '%s' '%s'(%d)\n",
652	evsel__name(evsel), strerror(err), err);
653	return NULL;
654	}
655	evsel->tp_format = tp_format;
656	return evsel->tp_format;
657	}
658	#endif
659
660	const char *const evsel__hw_names[PERF_COUNT_HW_MAX] = {
661	"cycles",
662	"instructions",
663	"cache-references",
664	"cache-misses",
665	"branches",
666	"branch-misses",
667	"bus-cycles",
668	"stalled-cycles-frontend",
669	"stalled-cycles-backend",
670	"ref-cycles",
671	};
672
673	char *evsel__bpf_counter_events;
674
675	bool evsel__match_bpf_counter_events(const char *name)
676	{
677	int name_len;
678	bool match;
679	char *ptr;
680
681	if (!evsel__bpf_counter_events)
682	return false;
683
684	ptr = strstr(evsel__bpf_counter_events, name);
685	name_len = strlen(name);
686
687	/* check name matches a full token in evsel__bpf_counter_events */
688	match = (ptr != NULL) &&
689	((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) &&
690	((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0'));
691
692	return match;
693	}
694
695	static const char *__evsel__hw_name(u64 config)
696	{
697	if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config])
698	return evsel__hw_names[config];
699
700	return "unknown-hardware";
701	}
702
703	static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
704	{
705	int colon = 0, r = 0;
706	struct perf_event_attr *attr = &evsel->core.attr;
707
708	#define MOD_PRINT(context, mod) do { \
709	if (!attr->exclude_##context) { \
710	if (!colon) colon = ++r; \
711	r += scnprintf(bf + r, size - r, "%c", mod); \
712	} } while(0)
713
714	if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
715	MOD_PRINT(kernel, 'k');
716	MOD_PRINT(user, 'u');
717	MOD_PRINT(hv, 'h');
718	}
719
720	if (attr->precise_ip) {
721	if (!colon)
722	colon = ++r;
723	r += scnprintf(buf: bf + r, size: size - r, fmt: "%.*s", attr->precise_ip, "ppp");
724	}
725
726	if (attr->exclude_host || attr->exclude_guest) {
727	MOD_PRINT(host, 'H');
728	MOD_PRINT(guest, 'G');
729	}
730	#undef MOD_PRINT
731	if (colon)
732	bf[colon - 1] = ':';
733	return r;
734	}
735
736	int __weak arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
737	{
738	return scnprintf(buf: bf, size, fmt: "%s", __evsel__hw_name(config: evsel->core.attr.config));
739	}
740
741	static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
742	{
743	int r = arch_evsel__hw_name(evsel, bf, size);
744	return r + evsel__add_modifiers(evsel, bf: bf + r, size: size - r);
745	}
746
747	const char *const evsel__sw_names[PERF_COUNT_SW_MAX] = {
748	"cpu-clock",
749	"task-clock",
750	"page-faults",
751	"context-switches",
752	"cpu-migrations",
753	"minor-faults",
754	"major-faults",
755	"alignment-faults",
756	"emulation-faults",
757	"dummy",
758	};
759
760	static const char *__evsel__sw_name(u64 config)
761	{
762	if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config])
763	return evsel__sw_names[config];
764	return "unknown-software";
765	}
766
767	static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
768	{
769	int r = scnprintf(buf: bf, size, fmt: "%s", __evsel__sw_name(config: evsel->core.attr.config));
770	return r + evsel__add_modifiers(evsel, bf: bf + r, size: size - r);
771	}
772
773	static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
774	{
775	int r;
776
777	r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
778
779	if (type & HW_BREAKPOINT_R)
780	r += scnprintf(buf: bf + r, size: size - r, fmt: "r");
781
782	if (type & HW_BREAKPOINT_W)
783	r += scnprintf(buf: bf + r, size: size - r, fmt: "w");
784
785	if (type & HW_BREAKPOINT_X)
786	r += scnprintf(buf: bf + r, size: size - r, fmt: "x");
787
788	return r;
789	}
790
791	static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
792	{
793	struct perf_event_attr *attr = &evsel->core.attr;
794	int r = __evsel__bp_name(bf, size, addr: attr->bp_addr, type: attr->bp_type);
795	return r + evsel__add_modifiers(evsel, bf: bf + r, size: size - r);
796	}
797
798	const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = {
799	{ "L1-dcache", "l1-d", "l1d", "L1-data", },
800	{ "L1-icache", "l1-i", "l1i", "L1-instruction", },
801	{ "LLC", "L2", },
802	{ "dTLB", "d-tlb", "Data-TLB", },
803	{ "iTLB", "i-tlb", "Instruction-TLB", },
804	{ "branch", "branches", "bpu", "btb", "bpc", },
805	{ "node", },
806	};
807
808	const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = {
809	{ "load", "loads", "read", },
810	{ "store", "stores", "write", },
811	{ "prefetch", "prefetches", "speculative-read", "speculative-load", },
812	};
813
814	const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = {
815	{ "refs", "Reference", "ops", "access", },
816	{ "misses", "miss", },
817	};
818
819	#define C(x) PERF_COUNT_HW_CACHE_##x
820	#define CACHE_READ (1 << C(OP_READ))
821	#define CACHE_WRITE (1 << C(OP_WRITE))
822	#define CACHE_PREFETCH (1 << C(OP_PREFETCH))
823	#define COP(x) (1 << x)
824
825	/*
826	* cache operation stat
827	* L1I : Read and prefetch only
828	* ITLB and BPU : Read-only
829	*/
830	static const unsigned long evsel__hw_cache_stat[C(MAX)] = {
831	[C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
832	[C(L1I)] = (CACHE_READ | CACHE_PREFETCH),
833	[C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
834	[C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
835	[C(ITLB)] = (CACHE_READ),
836	[C(BPU)] = (CACHE_READ),
837	[C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
838	};
839
840	bool evsel__is_cache_op_valid(u8 type, u8 op)
841	{
842	if (evsel__hw_cache_stat[type] & COP(op))
843	return true; /* valid */
844	else
845	return false; /* invalid */
846	}
847
848	int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size)
849	{
850	if (result) {
851	return scnprintf(buf: bf, size, fmt: "%s-%s-%s", evsel__hw_cache[type][0],
852	evsel__hw_cache_op[op][0],
853	evsel__hw_cache_result[result][0]);
854	}
855
856	return scnprintf(buf: bf, size, fmt: "%s-%s", evsel__hw_cache[type][0],
857	evsel__hw_cache_op[op][1]);
858	}
859
860	static int __evsel__hw_cache_name(u64 config, char *bf, size_t size)
861	{
862	u8 op, result, type = (config >> 0) & 0xff;
863	const char *err = "unknown-ext-hardware-cache-type";
864
865	if (type >= PERF_COUNT_HW_CACHE_MAX)
866	goto out_err;
867
868	op = (config >> 8) & 0xff;
869	err = "unknown-ext-hardware-cache-op";
870	if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
871	goto out_err;
872
873	result = (config >> 16) & 0xff;
874	err = "unknown-ext-hardware-cache-result";
875	if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
876	goto out_err;
877
878	err = "invalid-cache";
879	if (!evsel__is_cache_op_valid(type, op))
880	goto out_err;
881
882	return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
883	out_err:
884	return scnprintf(buf: bf, size, fmt: "%s", err);
885	}
886
887	static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
888	{
889	int ret = __evsel__hw_cache_name(config: evsel->core.attr.config, bf, size);
890	return ret + evsel__add_modifiers(evsel, bf: bf + ret, size: size - ret);
891	}
892
893	static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
894	{
895	int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
896	return ret + evsel__add_modifiers(evsel, bf: bf + ret, size: size - ret);
897	}
898
899	const char *evsel__name(struct evsel *evsel)
900	{
901	char bf[128];
902
903	if (!evsel)
904	goto out_unknown;
905
906	if (evsel->name)
907	return evsel->name;
908
909	switch (evsel->core.attr.type) {
910	case PERF_TYPE_RAW:
911	evsel__raw_name(evsel, bf, size: sizeof(bf));
912	break;
913
914	case PERF_TYPE_HARDWARE:
915	evsel__hw_name(evsel, bf, size: sizeof(bf));
916	break;
917
918	case PERF_TYPE_HW_CACHE:
919	evsel__hw_cache_name(evsel, bf, size: sizeof(bf));
920	break;
921
922	case PERF_TYPE_SOFTWARE:
923	evsel__sw_name(evsel, bf, size: sizeof(bf));
924	break;
925
926	case PERF_TYPE_TRACEPOINT:
927	scnprintf(buf: bf, size: sizeof(bf), fmt: "%s", "unknown tracepoint");
928	break;
929
930	case PERF_TYPE_BREAKPOINT:
931	evsel__bp_name(evsel, bf, size: sizeof(bf));
932	break;
933
934	case PERF_PMU_TYPE_TOOL:
935	scnprintf(buf: bf, size: sizeof(bf), fmt: "%s", evsel__tool_pmu_event_name(evsel));
936	break;
937
938	default:
939	scnprintf(buf: bf, size: sizeof(bf), fmt: "unknown attr type: %d",
940	evsel->core.attr.type);
941	break;
942	}
943
944	evsel->name = strdup(bf);
945
946	if (evsel->name)
947	return evsel->name;
948	out_unknown:
949	return "unknown";
950	}
951
952	bool evsel__name_is(struct evsel *evsel, const char *name)
953	{
954	return !strcmp(evsel__name(evsel), name);
955	}
956
957	const char *evsel__metric_id(const struct evsel *evsel)
958	{
959	if (evsel->metric_id)
960	return evsel->metric_id;
961
962	if (evsel__is_tool(evsel))
963	return evsel__tool_pmu_event_name(evsel);
964
965	return "unknown";
966	}
967
968	const char *evsel__group_name(struct evsel *evsel)
969	{
970	return evsel->group_name ?: "anon group";
971	}
972
973	/*
974	* Returns the group details for the specified leader,
975	* with following rules.
976	*
977	* For record -e '{cycles,instructions}'
978	* 'anon group { cycles:u, instructions:u }'
979	*
980	* For record -e 'cycles,instructions' and report --group
981	* 'cycles:u, instructions:u'
982	*/
983	int evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
984	{
985	int ret = 0;
986	bool first = true;
987	struct evsel *pos;
988	const char *group_name = evsel__group_name(evsel);
989
990	if (!evsel->forced_leader)
991	ret = scnprintf(buf, size, fmt: "%s { ", group_name);
992
993	for_each_group_evsel(pos, evsel) {
994	if (symbol_conf.skip_empty &&
995	evsel__hists(pos)->stats.nr_samples == 0)
996	continue;
997
998	ret += scnprintf(buf: buf + ret, size: size - ret, fmt: "%s%s",
999	first ? "" : ", ", evsel__name(evsel: pos));
1000	first = false;
1001	}
1002
1003	if (!evsel->forced_leader)
1004	ret += scnprintf(buf: buf + ret, size: size - ret, fmt: " }");
1005
1006	return ret;
1007	}
1008
1009	static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
1010	struct callchain_param *param)
1011	{
1012	bool function = evsel__is_function_event(evsel);
1013	struct perf_event_attr *attr = &evsel->core.attr;
1014
1015	evsel__set_sample_bit(evsel, CALLCHAIN);
1016
1017	attr->sample_max_stack = param->max_stack;
1018
1019	if (opts->kernel_callchains)
1020	attr->exclude_callchain_user = 1;
1021	if (opts->user_callchains)
1022	attr->exclude_callchain_kernel = 1;
1023	if (param->record_mode == CALLCHAIN_LBR) {
1024	if (!opts->branch_stack) {
1025	if (attr->exclude_user) {
1026	pr_warning("LBR callstack option is only available "
1027	"to get user callchain information. "
1028	"Falling back to framepointers.\n");
1029	} else {
1030	evsel__set_sample_bit(evsel, BRANCH_STACK);
1031	attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
1032	PERF_SAMPLE_BRANCH_CALL_STACK |
1033	PERF_SAMPLE_BRANCH_NO_CYCLES |
1034	PERF_SAMPLE_BRANCH_NO_FLAGS |
1035	PERF_SAMPLE_BRANCH_HW_INDEX;
1036	}
1037	} else
1038	pr_warning("Cannot use LBR callstack with branch stack. "
1039	"Falling back to framepointers.\n");
1040	}
1041
1042	if (param->record_mode == CALLCHAIN_DWARF) {
1043	if (!function) {
1044	const char *arch = perf_env__arch(env: evsel__env(evsel));
1045
1046	evsel__set_sample_bit(evsel, REGS_USER);
1047	evsel__set_sample_bit(evsel, STACK_USER);
1048	if (opts->sample_user_regs &&
1049	DWARF_MINIMAL_REGS(arch) != arch__user_reg_mask()) {
1050	attr->sample_regs_user |= DWARF_MINIMAL_REGS(arch);
1051	pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
1052	"specifying a subset with --user-regs may render DWARF unwinding unreliable, "
1053	"so the minimal registers set (IP, SP) is explicitly forced.\n");
1054	} else {
1055	attr->sample_regs_user |= arch__user_reg_mask();
1056	}
1057	attr->sample_stack_user = param->dump_size;
1058	attr->exclude_callchain_user = 1;
1059	} else {
1060	pr_info("Cannot use DWARF unwind for function trace event,"
1061	" falling back to framepointers.\n");
1062	}
1063	}
1064
1065	if (function) {
1066	pr_info("Disabling user space callchains for function trace event.\n");
1067	attr->exclude_callchain_user = 1;
1068	}
1069
1070	if (param->defer && !attr->exclude_callchain_user)
1071	attr->defer_callchain = 1;
1072	}
1073
1074	void evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
1075	struct callchain_param *param)
1076	{
1077	if (param->enabled)
1078	return __evsel__config_callchain(evsel, opts, param);
1079	}
1080
1081	static void evsel__reset_callgraph(struct evsel *evsel, struct callchain_param *param)
1082	{
1083	struct perf_event_attr *attr = &evsel->core.attr;
1084
1085	evsel__reset_sample_bit(evsel, CALLCHAIN);
1086	if (param->record_mode == CALLCHAIN_LBR) {
1087	evsel__reset_sample_bit(evsel, BRANCH_STACK);
1088	attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
1089	PERF_SAMPLE_BRANCH_CALL_STACK |
1090	PERF_SAMPLE_BRANCH_HW_INDEX);
1091	}
1092	if (param->record_mode == CALLCHAIN_DWARF) {
1093	evsel__reset_sample_bit(evsel, REGS_USER);
1094	evsel__reset_sample_bit(evsel, STACK_USER);
1095	}
1096	}
1097
1098	static void evsel__apply_ratio_to_prev(struct evsel *evsel,
1099	struct perf_event_attr *attr,
1100	struct record_opts *opts,
1101	const char *buf)
1102	{
1103	struct perf_event_attr *prev_attr = NULL;
1104	struct evsel *evsel_prev = NULL;
1105	u64 type = evsel->core.attr.sample_type;
1106	u64 prev_type = 0;
1107	double rtp;
1108
1109	rtp = strtod(buf, NULL);
1110	if (rtp <= 0) {
1111	pr_err("Invalid ratio-to-prev value %lf\n", rtp);
1112	return;
1113	}
1114	if (evsel == evsel__leader(evsel)) {
1115	pr_err("Invalid use of ratio-to-prev term without preceding element in group\n");
1116	return;
1117	}
1118	if (!evsel->pmu->is_core) {
1119	pr_err("Event using ratio-to-prev term must have a core PMU\n");
1120	return;
1121	}
1122
1123	evsel_prev = evsel__prev(evsel);
1124	if (!evsel_prev) {
1125	pr_err("Previous event does not exist.\n");
1126	return;
1127	}
1128
1129	if (evsel_prev->pmu->type != evsel->pmu->type) {
1130	pr_err("Compared events (\"%s\", \"%s\") must have same PMU\n",
1131	evsel->name, evsel_prev->name);
1132	return;
1133	}
1134
1135	prev_attr = &evsel_prev->core.attr;
1136	prev_type = evsel_prev->core.attr.sample_type;
1137
1138	if (!(prev_type & PERF_SAMPLE_PERIOD)) {
1139	attr->sample_period = prev_attr->sample_period * rtp;
1140	attr->freq = 0;
1141	evsel__reset_sample_bit(evsel, PERIOD);
1142	} else if (!(type & PERF_SAMPLE_PERIOD)) {
1143	prev_attr->sample_period = attr->sample_period / rtp;
1144	prev_attr->freq = 0;
1145	evsel__reset_sample_bit(evsel_prev, PERIOD);
1146	} else {
1147	if (opts->user_interval != ULLONG_MAX) {
1148	prev_attr->sample_period = opts->user_interval;
1149	attr->sample_period = prev_attr->sample_period * rtp;
1150	prev_attr->freq = 0;
1151	attr->freq = 0;
1152	evsel__reset_sample_bit(evsel_prev, PERIOD);
1153	evsel__reset_sample_bit(evsel, PERIOD);
1154	} else {
1155	pr_err("Event period term or count (-c) must be set when using ratio-to-prev term.\n");
1156	return;
1157	}
1158	}
1159
1160	arch_evsel__apply_ratio_to_prev(evsel, attr);
1161	}
1162
1163	static void evsel__apply_config_terms(struct evsel *evsel,
1164	struct record_opts *opts, bool track)
1165	{
1166	struct evsel_config_term *term;
1167	struct list_head *config_terms = &evsel->config_terms;
1168	struct perf_event_attr *attr = &evsel->core.attr;
1169	/* callgraph default */
1170	struct callchain_param param = {
1171	.record_mode = callchain_param.record_mode,
1172	};
1173	u32 dump_size = 0;
1174	int max_stack = 0;
1175	const char *callgraph_buf = NULL;
1176	const char *rtp_buf = NULL;
1177
1178	list_for_each_entry(term, config_terms, list) {
1179	switch (term->type) {
1180	case EVSEL__CONFIG_TERM_PERIOD:
1181	if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
1182	attr->sample_period = term->val.period;
1183	attr->freq = 0;
1184	evsel__reset_sample_bit(evsel, PERIOD);
1185	}
1186	break;
1187	case EVSEL__CONFIG_TERM_FREQ:
1188	if (!(term->weak && opts->user_freq != UINT_MAX)) {
1189	attr->sample_freq = term->val.freq;
1190	attr->freq = 1;
1191	evsel__set_sample_bit(evsel, PERIOD);
1192	}
1193	break;
1194	case EVSEL__CONFIG_TERM_TIME:
1195	if (term->val.time)
1196	evsel__set_sample_bit(evsel, TIME);
1197	else
1198	evsel__reset_sample_bit(evsel, TIME);
1199	break;
1200	case EVSEL__CONFIG_TERM_CALLGRAPH:
1201	callgraph_buf = term->val.str;
1202	break;
1203	case EVSEL__CONFIG_TERM_BRANCH:
1204	if (term->val.str && strcmp(term->val.str, "no")) {
1205	evsel__set_sample_bit(evsel, BRANCH_STACK);
1206	parse_branch_str(term->val.str,
1207	&attr->branch_sample_type);
1208	} else
1209	evsel__reset_sample_bit(evsel, BRANCH_STACK);
1210	break;
1211	case EVSEL__CONFIG_TERM_STACK_USER:
1212	dump_size = term->val.stack_user;
1213	break;
1214	case EVSEL__CONFIG_TERM_MAX_STACK:
1215	max_stack = term->val.max_stack;
1216	break;
1217	case EVSEL__CONFIG_TERM_MAX_EVENTS:
1218	evsel->max_events = term->val.max_events;
1219	break;
1220	case EVSEL__CONFIG_TERM_INHERIT:
1221	/*
1222	* attr->inherit should has already been set by
1223	* evsel__config. If user explicitly set
1224	* inherit using config terms, override global
1225	* opt->no_inherit setting.
1226	*/
1227	attr->inherit = term->val.inherit ? 1 : 0;
1228	break;
1229	case EVSEL__CONFIG_TERM_OVERWRITE:
1230	attr->write_backward = term->val.overwrite ? 1 : 0;
1231	break;
1232	case EVSEL__CONFIG_TERM_DRV_CFG:
1233	break;
1234	case EVSEL__CONFIG_TERM_PERCORE:
1235	break;
1236	case EVSEL__CONFIG_TERM_AUX_OUTPUT:
1237	attr->aux_output = term->val.aux_output ? 1 : 0;
1238	break;
1239	case EVSEL__CONFIG_TERM_AUX_ACTION:
1240	/* Already applied by auxtrace */
1241	break;
1242	case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE:
1243	/* Already applied by auxtrace */
1244	break;
1245	case EVSEL__CONFIG_TERM_CFG_CHG:
1246	break;
1247	case EVSEL__CONFIG_TERM_RATIO_TO_PREV:
1248	rtp_buf = term->val.str;
1249	break;
1250	default:
1251	break;
1252	}
1253	}
1254
1255	/* User explicitly set per-event callgraph, clear the old setting and reset. */
1256	if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
1257	bool sample_address = false;
1258
1259	if (max_stack) {
1260	param.max_stack = max_stack;
1261	if (callgraph_buf == NULL)
1262	callgraph_buf = "fp";
1263	}
1264
1265	/* parse callgraph parameters */
1266	if (callgraph_buf != NULL) {
1267	if (!strcmp(callgraph_buf, "no")) {
1268	param.enabled = false;
1269	param.record_mode = CALLCHAIN_NONE;
1270	} else {
1271	param.enabled = true;
1272	if (parse_callchain_record(arg: callgraph_buf, param: &param)) {
1273	pr_err("per-event callgraph setting for %s failed. "
1274	"Apply callgraph global setting for it\n",
1275	evsel->name);
1276	return;
1277	}
1278	if (param.record_mode == CALLCHAIN_DWARF)
1279	sample_address = true;
1280	}
1281	}
1282	if (dump_size > 0) {
1283	dump_size = round_up(dump_size, sizeof(u64));
1284	param.dump_size = dump_size;
1285	}
1286
1287	/* If global callgraph set, clear it */
1288	if (callchain_param.enabled)
1289	evsel__reset_callgraph(evsel, param: &callchain_param);
1290
1291	/* set perf-event callgraph */
1292	if (param.enabled) {
1293	if (sample_address) {
1294	evsel__set_sample_bit(evsel, ADDR);
1295	evsel__set_sample_bit(evsel, DATA_SRC);
1296	evsel->core.attr.mmap_data = track;
1297	}
1298	evsel__config_callchain(evsel, opts, param: &param);
1299	}
1300	}
1301	if (rtp_buf)
1302	evsel__apply_ratio_to_prev(evsel, attr, opts, buf: rtp_buf);
1303	}
1304
1305	struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type)
1306	{
1307	struct evsel_config_term *term, *found_term = NULL;
1308
1309	list_for_each_entry(term, &evsel->config_terms, list) {
1310	if (term->type == type)
1311	found_term = term;
1312	}
1313
1314	return found_term;
1315	}
1316
1317	void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
1318	{
1319	evsel__set_sample_bit(evsel, WEIGHT);
1320	}
1321
1322	void __weak arch__post_evsel_config(struct evsel *evsel __maybe_unused,
1323	struct perf_event_attr *attr __maybe_unused)
1324	{
1325	}
1326
1327	void __weak arch_evsel__apply_ratio_to_prev(struct evsel *evsel __maybe_unused,
1328	struct perf_event_attr *attr __maybe_unused)
1329	{
1330	}
1331
1332	static void evsel__set_default_freq_period(struct record_opts *opts,
1333	struct perf_event_attr *attr)
1334	{
1335	if (opts->freq) {
1336	attr->freq = 1;
1337	attr->sample_freq = opts->freq;
1338	} else {
1339	attr->sample_period = opts->default_interval;
1340	}
1341	}
1342
1343	bool evsel__is_offcpu_event(struct evsel *evsel)
1344	{
1345	return evsel__is_bpf_output(evsel) && evsel__name_is(evsel, OFFCPU_EVENT) &&
1346	evsel->core.attr.sample_type & PERF_SAMPLE_RAW;
1347	}
1348
1349	/*
1350	* The enable_on_exec/disabled value strategy:
1351	*
1352	* 1) For any type of traced program:
1353	* - all independent events and group leaders are disabled
1354	* - all group members are enabled
1355	*
1356	* Group members are ruled by group leaders. They need to
1357	* be enabled, because the group scheduling relies on that.
1358	*
1359	* 2) For traced programs executed by perf:
1360	* - all independent events and group leaders have
1361	* enable_on_exec set
1362	* - we don't specifically enable or disable any event during
1363	* the record command
1364	*
1365	* Independent events and group leaders are initially disabled
1366	* and get enabled by exec. Group members are ruled by group
1367	* leaders as stated in 1).
1368	*
1369	* 3) For traced programs attached by perf (pid/tid):
1370	* - we specifically enable or disable all events during
1371	* the record command
1372	*
1373	* When attaching events to already running traced we
1374	* enable/disable events specifically, as there's no
1375	* initial traced exec call.
1376	*/
1377	void evsel__config(struct evsel *evsel, struct record_opts *opts,
1378	struct callchain_param *callchain)
1379	{
1380	struct evsel *leader = evsel__leader(evsel);
1381	struct perf_event_attr *attr = &evsel->core.attr;
1382	int track = evsel->tracking;
1383	bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
1384
1385	attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
1386	attr->inherit = target__has_cpu(target: &opts->target) ? 0 : !opts->no_inherit;
1387	attr->write_backward = opts->overwrite ? 1 : 0;
1388	attr->read_format = PERF_FORMAT_LOST;
1389
1390	evsel__set_sample_bit(evsel, IP);
1391	evsel__set_sample_bit(evsel, TID);
1392
1393	if (evsel->sample_read) {
1394	evsel__set_sample_bit(evsel, READ);
1395
1396	/*
1397	* We need ID even in case of single event, because
1398	* PERF_SAMPLE_READ process ID specific data.
1399	*/
1400	evsel__set_sample_id(evsel, can_sample_identifier: false);
1401
1402	/*
1403	* Apply group format only if we belong to group
1404	* with more than one members.
1405	*/
1406	if (leader->core.nr_members > 1) {
1407	attr->read_format |= PERF_FORMAT_GROUP;
1408	}
1409
1410	/*
1411	* Inherit + SAMPLE_READ requires SAMPLE_TID in the read_format
1412	*/
1413	if (attr->inherit) {
1414	evsel__set_sample_bit(evsel, TID);
1415	evsel->core.attr.read_format |=
1416	PERF_FORMAT_ID;
1417	}
1418	}
1419
1420	/*
1421	* We default some events to have a default interval. But keep
1422	* it a weak assumption overridable by the user.
1423	*/
1424	if ((evsel->is_libpfm_event && !attr->sample_period) ||
1425	(!evsel->is_libpfm_event && (!attr->sample_period ||
1426	opts->user_freq != UINT_MAX ||
1427	opts->user_interval != ULLONG_MAX)))
1428	evsel__set_default_freq_period(opts, attr);
1429
1430	/*
1431	* If attr->freq was set (here or earlier), ask for period
1432	* to be sampled.
1433	*/
1434	if (attr->freq)
1435	evsel__set_sample_bit(evsel, PERIOD);
1436
1437	if (opts->no_samples)
1438	attr->sample_freq = 0;
1439
1440	if (opts->inherit_stat) {
1441	evsel->core.attr.read_format |=
1442	PERF_FORMAT_TOTAL_TIME_ENABLED |
1443	PERF_FORMAT_TOTAL_TIME_RUNNING |
1444	PERF_FORMAT_ID;
1445	attr->inherit_stat = 1;
1446	}
1447
1448	if (opts->sample_address) {
1449	evsel__set_sample_bit(evsel, ADDR);
1450	attr->mmap_data = track;
1451	}
1452
1453	/*
1454	* We don't allow user space callchains for function trace
1455	* event, due to issues with page faults while tracing page
1456	* fault handler and its overall trickiness nature.
1457	*/
1458	if (evsel__is_function_event(evsel))
1459	evsel->core.attr.exclude_callchain_user = 1;
1460
1461	if (callchain && callchain->enabled && !evsel->no_aux_samples)
1462	evsel__config_callchain(evsel, opts, param: callchain);
1463
1464	if (opts->sample_intr_regs && !evsel->no_aux_samples &&
1465	!evsel__is_dummy_event(evsel)) {
1466	attr->sample_regs_intr = opts->sample_intr_regs;
1467	evsel__set_sample_bit(evsel, REGS_INTR);
1468	}
1469
1470	if (opts->sample_user_regs && !evsel->no_aux_samples &&
1471	!evsel__is_dummy_event(evsel)) {
1472	attr->sample_regs_user |= opts->sample_user_regs;
1473	evsel__set_sample_bit(evsel, REGS_USER);
1474	}
1475
1476	if (target__has_cpu(target: &opts->target) || opts->sample_cpu)
1477	evsel__set_sample_bit(evsel, CPU);
1478
1479	/*
1480	* When the user explicitly disabled time don't force it here.
1481	*/
1482	if (opts->sample_time &&
1483	(!perf_missing_features.sample_id_all &&
1484	(!opts->no_inherit || target__has_cpu(target: &opts->target) || per_cpu ||
1485	opts->sample_time_set)))
1486	evsel__set_sample_bit(evsel, TIME);
1487
1488	if (opts->raw_samples && !evsel->no_aux_samples) {
1489	evsel__set_sample_bit(evsel, TIME);
1490	evsel__set_sample_bit(evsel, RAW);
1491	evsel__set_sample_bit(evsel, CPU);
1492	}
1493
1494	if (opts->sample_data_src)
1495	evsel__set_sample_bit(evsel, DATA_SRC);
1496
1497	if (opts->sample_phys_addr)
1498	evsel__set_sample_bit(evsel, PHYS_ADDR);
1499
1500	if (opts->no_buffering) {
1501	attr->watermark = 0;
1502	attr->wakeup_events = 1;
1503	}
1504	if (opts->branch_stack && !evsel->no_aux_samples) {
1505	evsel__set_sample_bit(evsel, BRANCH_STACK);
1506	attr->branch_sample_type = opts->branch_stack;
1507	}
1508
1509	if (opts->sample_weight || evsel->retire_lat) {
1510	arch_evsel__set_sample_weight(evsel);
1511	evsel->retire_lat = false;
1512	}
1513	attr->task = track;
1514	attr->mmap = track;
1515	attr->mmap2 = track && !perf_missing_features.mmap2;
1516	attr->comm = track;
1517	attr->build_id = track && opts->build_id;
1518	attr->defer_output = track && callchain && callchain->defer;
1519
1520	/*
1521	* ksymbol is tracked separately with text poke because it needs to be
1522	* system wide and enabled immediately.
1523	*/
1524	if (!opts->text_poke)
1525	attr->ksymbol = track && !perf_missing_features.ksymbol;
1526	attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1527
1528	if (opts->record_namespaces)
1529	attr->namespaces = track;
1530
1531	if (opts->record_cgroup) {
1532	attr->cgroup = track && !perf_missing_features.cgroup;
1533	evsel__set_sample_bit(evsel, CGROUP);
1534	}
1535
1536	if (opts->sample_data_page_size)
1537	evsel__set_sample_bit(evsel, DATA_PAGE_SIZE);
1538
1539	if (opts->sample_code_page_size)
1540	evsel__set_sample_bit(evsel, CODE_PAGE_SIZE);
1541
1542	if (opts->record_switch_events)
1543	attr->context_switch = track;
1544
1545	if (opts->sample_transaction)
1546	evsel__set_sample_bit(evsel, TRANSACTION);
1547
1548	if (opts->running_time) {
1549	evsel->core.attr.read_format |=
1550	PERF_FORMAT_TOTAL_TIME_ENABLED |
1551	PERF_FORMAT_TOTAL_TIME_RUNNING;
1552	}
1553
1554	/*
1555	* XXX see the function comment above
1556	*
1557	* Disabling only independent events or group leaders,
1558	* keeping group members enabled.
1559	*/
1560	if (evsel__is_group_leader(evsel))
1561	attr->disabled = 1;
1562
1563	/*
1564	* Setting enable_on_exec for independent events and
1565	* group leaders for traced executed by perf.
1566	*/
1567	if (target__none(target: &opts->target) && evsel__is_group_leader(evsel) &&
1568	!opts->target.initial_delay)
1569	attr->enable_on_exec = 1;
1570
1571	if (evsel->immediate) {
1572	attr->disabled = 0;
1573	attr->enable_on_exec = 0;
1574	}
1575
1576	clockid = opts->clockid;
1577	if (opts->use_clockid) {
1578	attr->use_clockid = 1;
1579	attr->clockid = opts->clockid;
1580	}
1581
1582	if (evsel->precise_max)
1583	attr->precise_ip = 3;
1584
1585	if (opts->all_user) {
1586	attr->exclude_kernel = 1;
1587	attr->exclude_user = 0;
1588	}
1589
1590	if (opts->all_kernel) {
1591	attr->exclude_kernel = 0;
1592	attr->exclude_user = 1;
1593	}
1594
1595	if (evsel->core.pmu_cpus || evsel->unit)
1596	evsel->core.attr.read_format |= PERF_FORMAT_ID;
1597
1598	/*
1599	* Apply event specific term settings,
1600	* it overloads any global configuration.
1601	*/
1602	evsel__apply_config_terms(evsel, opts, track);
1603
1604	evsel->ignore_missing_thread = opts->ignore_missing_thread;
1605
1606	/* The --period option takes the precedence. */
1607	if (opts->period_set) {
1608	if (opts->period)
1609	evsel__set_sample_bit(evsel, PERIOD);
1610	else
1611	evsel__reset_sample_bit(evsel, PERIOD);
1612	}
1613
1614	/*
1615	* A dummy event never triggers any actual counter and therefore
1616	* cannot be used with branch_stack.
1617	*
1618	* For initial_delay, a dummy event is added implicitly.
1619	* The software event will trigger -EOPNOTSUPP error out,
1620	* if BRANCH_STACK bit is set.
1621	*/
1622	if (evsel__is_dummy_event(evsel))
1623	evsel__reset_sample_bit(evsel, BRANCH_STACK);
1624
1625	if (evsel__is_offcpu_event(evsel)) {
1626	evsel->core.attr.sample_type &= OFFCPU_SAMPLE_TYPES;
1627	attr->inherit = 0;
1628	}
1629
1630	arch__post_evsel_config(evsel, attr);
1631	}
1632
1633	int evsel__set_filter(struct evsel *evsel, const char *filter)
1634	{
1635	char *new_filter = strdup(filter);
1636
1637	if (new_filter != NULL) {
1638	free(evsel->filter);
1639	evsel->filter = new_filter;
1640	return 0;
1641	}
1642
1643	return -1;
1644	}
1645
1646	static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter)
1647	{
1648	char *new_filter;
1649
1650	if (evsel->filter == NULL)
1651	return evsel__set_filter(evsel, filter);
1652
1653	if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1654	free(evsel->filter);
1655	evsel->filter = new_filter;
1656	return 0;
1657	}
1658
1659	return -1;
1660	}
1661
1662	int evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1663	{
1664	return evsel__append_filter(evsel, fmt: "(%s) && (%s)", filter);
1665	}
1666
1667	int evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1668	{
1669	return evsel__append_filter(evsel, fmt: "%s,%s", filter);
1670	}
1671
1672	/* Caller has to clear disabled after going through all CPUs. */
1673	int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx)
1674	{
1675	return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx);
1676	}
1677
1678	int evsel__enable(struct evsel *evsel)
1679	{
1680	int err = perf_evsel__enable(&evsel->core);
1681
1682	if (!err)
1683	evsel->disabled = false;
1684	return err;
1685	}
1686
1687	/* Caller has to set disabled after going through all CPUs. */
1688	int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx)
1689	{
1690	return perf_evsel__disable_cpu(&evsel->core, cpu_map_idx);
1691	}
1692
1693	int evsel__disable(struct evsel *evsel)
1694	{
1695	int err = perf_evsel__disable(&evsel->core);
1696	/*
1697	* We mark it disabled here so that tools that disable a event can
1698	* ignore events after they disable it. I.e. the ring buffer may have
1699	* already a few more events queued up before the kernel got the stop
1700	* request.
1701	*/
1702	if (!err)
1703	evsel->disabled = true;
1704
1705	return err;
1706	}
1707
1708	void free_config_terms(struct list_head *config_terms)
1709	{
1710	struct evsel_config_term *term, *h;
1711
1712	list_for_each_entry_safe(term, h, config_terms, list) {
1713	list_del_init(entry: &term->list);
1714	if (term->free_str)
1715	zfree(&term->val.str);
1716	free(term);
1717	}
1718	}
1719
1720	static void evsel__free_config_terms(struct evsel *evsel)
1721	{
1722	free_config_terms(config_terms: &evsel->config_terms);
1723	}
1724
1725	static void (*evsel__priv_destructor)(void *priv);
1726
1727	void evsel__set_priv_destructor(void (*destructor)(void *priv))
1728	{
1729	assert(evsel__priv_destructor == NULL);
1730
1731	evsel__priv_destructor = destructor;
1732	}
1733
1734	void evsel__exit(struct evsel *evsel)
1735	{
1736	assert(list_empty(head: &evsel->core.node));
1737	assert(evsel->evlist == NULL);
1738	if (evsel__is_retire_lat(evsel))
1739	evsel__tpebs_close(evsel);
1740	bpf_counter__destroy(evsel);
1741	perf_bpf_filter__destroy(evsel);
1742	evsel__free_counts(evsel);
1743	perf_evsel__free_fd(&evsel->core);
1744	perf_evsel__free_id(&evsel->core);
1745	evsel__free_config_terms(evsel);
1746	cgroup__put(cgroup: evsel->cgrp);
1747	perf_evsel__exit(&evsel->core);
1748	zfree(&evsel->group_name);
1749	zfree(&evsel->name);
1750	#ifdef HAVE_LIBTRACEEVENT
1751	zfree(&evsel->tp_sys);
1752	zfree(&evsel->tp_name);
1753	#endif
1754	zfree(&evsel->filter);
1755	zfree(&evsel->group_pmu_name);
1756	zfree(&evsel->unit);
1757	zfree(&evsel->metric_id);
1758	evsel__zero_per_pkg(evsel);
1759	hashmap__free(map: evsel->per_pkg_mask);
1760	evsel->per_pkg_mask = NULL;
1761	if (evsel__priv_destructor)
1762	evsel__priv_destructor(evsel->priv);
1763	perf_evsel__object.fini(evsel);
1764	if (evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME ||
1765	evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME)
1766	xyarray__delete(evsel->start_times);
1767	}
1768
1769	void evsel__delete(struct evsel *evsel)
1770	{
1771	if (!evsel)
1772	return;
1773
1774	evsel__exit(evsel);
1775	free(evsel);
1776	}
1777
1778	void evsel__compute_deltas(struct evsel *evsel, int cpu_map_idx, int thread,
1779	struct perf_counts_values *count)
1780	{
1781	struct perf_counts_values tmp;
1782
1783	if (!evsel->prev_raw_counts)
1784	return;
1785
1786	tmp = *perf_counts(counts: evsel->prev_raw_counts, cpu_map_idx, thread);
1787	*perf_counts(counts: evsel->prev_raw_counts, cpu_map_idx, thread) = *count;
1788
1789	count->val = count->val - tmp.val;
1790	count->ena = count->ena - tmp.ena;
1791	count->run = count->run - tmp.run;
1792	}
1793
1794	static int evsel__read_one(struct evsel *evsel, int cpu_map_idx, int thread)
1795	{
1796	struct perf_counts_values *count = perf_counts(counts: evsel->counts, cpu_map_idx, thread);
1797
1798	return perf_evsel__read(&evsel->core, cpu_map_idx, thread, count);
1799	}
1800
1801	static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread,
1802	u64 val, u64 ena, u64 run, u64 lost)
1803	{
1804	struct perf_counts_values *count;
1805
1806	count = perf_counts(counts: counter->counts, cpu_map_idx, thread);
1807
1808	if (evsel__is_retire_lat(evsel: counter)) {
1809	evsel__tpebs_read(counter, cpu_map_idx, thread);
1810	perf_counts__set_loaded(counts: counter->counts, cpu_map_idx, thread, loaded: true);
1811	return;
1812	}
1813
1814	count->val = val;
1815	count->ena = ena;
1816	count->run = run;
1817	count->lost = lost;
1818
1819	perf_counts__set_loaded(counts: counter->counts, cpu_map_idx, thread, loaded: true);
1820	}
1821
1822	static bool evsel__group_has_tpebs(struct evsel *leader)
1823	{
1824	struct evsel *evsel;
1825
1826	for_each_group_evsel(evsel, leader) {
1827	if (evsel__is_retire_lat(evsel))
1828	return true;
1829	}
1830	return false;
1831	}
1832
1833	static u64 evsel__group_read_nr_members(struct evsel *leader)
1834	{
1835	u64 nr = leader->core.nr_members;
1836	struct evsel *evsel;
1837
1838	for_each_group_evsel(evsel, leader) {
1839	if (evsel__is_retire_lat(evsel))
1840	nr--;
1841	}
1842	return nr;
1843	}
1844
1845	static u64 evsel__group_read_size(struct evsel *leader)
1846	{
1847	u64 read_format = leader->core.attr.read_format;
1848	int entry = sizeof(u64); /* value */
1849	int size = 0;
1850	int nr = 1;
1851
1852	if (!evsel__group_has_tpebs(leader))
1853	return perf_evsel__read_size(&leader->core);
1854
1855	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1856	size += sizeof(u64);
1857
1858	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1859	size += sizeof(u64);
1860
1861	if (read_format & PERF_FORMAT_ID)
1862	entry += sizeof(u64);
1863
1864	if (read_format & PERF_FORMAT_LOST)
1865	entry += sizeof(u64);
1866
1867	if (read_format & PERF_FORMAT_GROUP) {
1868	nr = evsel__group_read_nr_members(leader);
1869	size += sizeof(u64);
1870	}
1871
1872	size += entry * nr;
1873	return size;
1874	}
1875
1876	static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int thread, u64 *data)
1877	{
1878	u64 read_format = leader->core.attr.read_format;
1879	struct sample_read_value *v;
1880	u64 nr, ena = 0, run = 0, lost = 0;
1881
1882	nr = *data++;
1883
1884	if (nr != evsel__group_read_nr_members(leader))
1885	return -EINVAL;
1886
1887	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1888	ena = *data++;
1889
1890	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1891	run = *data++;
1892
1893	v = (void *)data;
1894	sample_read_group__for_each(v, nr, read_format) {
1895	struct evsel *counter;
1896
1897	counter = evlist__id2evsel(evlist: leader->evlist, id: v->id);
1898	if (!counter)
1899	return -EINVAL;
1900
1901	if (read_format & PERF_FORMAT_LOST)
1902	lost = v->lost;
1903
1904	evsel__set_count(counter, cpu_map_idx, thread, val: v->value, ena, run, lost);
1905	}
1906
1907	return 0;
1908	}
1909
1910	static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread)
1911	{
1912	struct perf_stat_evsel *ps = leader->stats;
1913	u64 read_format = leader->core.attr.read_format;
1914	int size = evsel__group_read_size(leader);
1915	u64 *data = ps->group_data;
1916
1917	if (!(read_format & PERF_FORMAT_ID))
1918	return -EINVAL;
1919
1920	if (!evsel__is_group_leader(evsel: leader))
1921	return -EINVAL;
1922
1923	if (!data) {
1924	data = zalloc(size);
1925	if (!data)
1926	return -ENOMEM;
1927
1928	ps->group_data = data;
1929	}
1930
1931	if (FD(leader, cpu_map_idx, thread) < 0)
1932	return -EINVAL;
1933
1934	if (readn(FD(leader, cpu_map_idx, thread), data, size) <= 0)
1935	return -errno;
1936
1937	return evsel__process_group_data(leader, cpu_map_idx, thread, data);
1938	}
1939
1940	bool __evsel__match(const struct evsel *evsel, u32 type, u64 config)
1941	{
1942
1943	u32 e_type = evsel->core.attr.type;
1944	u64 e_config = evsel->core.attr.config;
1945
1946	if (e_type == type && e_config == config)
1947	return true;
1948	if (type != PERF_TYPE_HARDWARE && type != PERF_TYPE_HW_CACHE)
1949	return false;
1950	if ((e_type == PERF_TYPE_HARDWARE || e_type == PERF_TYPE_HW_CACHE) &&
1951	perf_pmus__supports_extended_type())
1952	e_config &= PERF_HW_EVENT_MASK;
1953	if (e_type == type && e_config == config)
1954	return true;
1955	if (type == PERF_TYPE_HARDWARE && evsel->pmu && evsel->pmu->is_core &&
1956	evsel->alternate_hw_config == config)
1957	return true;
1958	return false;
1959	}
1960
1961	int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread)
1962	{
1963	if (evsel__is_tool(evsel))
1964	return evsel__tool_pmu_read(evsel, cpu_map_idx, thread);
1965
1966	if (evsel__is_hwmon(evsel))
1967	return evsel__hwmon_pmu_read(evsel, cpu_map_idx, thread);
1968
1969	if (evsel__is_drm(evsel))
1970	return evsel__drm_pmu_read(evsel, cpu_map_idx, thread);
1971
1972	if (evsel__is_retire_lat(evsel))
1973	return evsel__tpebs_read(evsel, cpu_map_idx, thread);
1974
1975	if (evsel->core.attr.read_format & PERF_FORMAT_GROUP)
1976	return evsel__read_group(leader: evsel, cpu_map_idx, thread);
1977
1978	return evsel__read_one(evsel, cpu_map_idx, thread);
1979	}
1980
1981	int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale)
1982	{
1983	struct perf_counts_values count;
1984	size_t nv = scale ? 3 : 1;
1985
1986	if (FD(evsel, cpu_map_idx, thread) < 0)
1987	return -EINVAL;
1988
1989	if (evsel->counts == NULL && evsel__alloc_counts(evsel) < 0)
1990	return -ENOMEM;
1991
1992	if (readn(FD(evsel, cpu_map_idx, thread), &count, nv * sizeof(u64)) <= 0)
1993	return -errno;
1994
1995	evsel__compute_deltas(evsel, cpu_map_idx, thread, count: &count);
1996	perf_counts_values__scale(&count, scale, NULL);
1997	*perf_counts(counts: evsel->counts, cpu_map_idx, thread) = count;
1998	return 0;
1999	}
2000
2001	static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other,
2002	int cpu_map_idx)
2003	{
2004	struct perf_cpu cpu;
2005
2006	cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx);
2007	return perf_cpu_map__idx(other->core.cpus, cpu);
2008	}
2009
2010	static int evsel__hybrid_group_cpu_map_idx(struct evsel *evsel, int cpu_map_idx)
2011	{
2012	struct evsel *leader = evsel__leader(evsel);
2013
2014	if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(evsel: leader)) ||
2015	(!evsel__is_hybrid(evsel) && evsel__is_hybrid(evsel: leader))) {
2016	return evsel__match_other_cpu(evsel, other: leader, cpu_map_idx);
2017	}
2018
2019	return cpu_map_idx;
2020	}
2021
2022	static int get_group_fd(struct evsel *evsel, int cpu_map_idx, int thread)
2023	{
2024	struct evsel *leader = evsel__leader(evsel);
2025	int fd;
2026
2027	if (!evsel->supported || evsel__is_group_leader(evsel))
2028	return -1;
2029
2030	/*
2031	* Leader must be already processed/open,
2032	* if not it's a bug.
2033	*/
2034	BUG_ON(!leader->core.fd);
2035
2036	cpu_map_idx = evsel__hybrid_group_cpu_map_idx(evsel, cpu_map_idx);
2037	if (cpu_map_idx == -1)
2038	return -1;
2039
2040	fd = FD(leader, cpu_map_idx, thread);
2041	BUG_ON(fd == -1 && leader->supported);
2042
2043	/*
2044	* When the leader has been skipped, return -2 to distinguish from no
2045	* group leader case.
2046	*/
2047	return fd == -1 ? -2 : fd;
2048	}
2049
2050	static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx)
2051	{
2052	for (int cpu = 0; cpu < nr_cpus; cpu++)
2053	for (int thread = thread_idx; thread < nr_threads - 1; thread++)
2054	FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
2055	}
2056
2057	static int update_fds(struct evsel *evsel,
2058	int nr_cpus, int cpu_map_idx,
2059	int nr_threads, int thread_idx)
2060	{
2061	struct evsel *pos;
2062
2063	if (cpu_map_idx >= nr_cpus || thread_idx >= nr_threads)
2064	return -EINVAL;
2065
2066	evlist__for_each_entry(evsel->evlist, pos) {
2067	nr_cpus = pos != evsel ? nr_cpus : cpu_map_idx;
2068
2069	evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
2070
2071	/*
2072	* Since fds for next evsel has not been created,
2073	* there is no need to iterate whole event list.
2074	*/
2075	if (pos == evsel)
2076	break;
2077	}
2078	return 0;
2079	}
2080
2081	static bool evsel__ignore_missing_thread(struct evsel *evsel,
2082	int nr_cpus, int cpu_map_idx,
2083	struct perf_thread_map *threads,
2084	int thread, int err)
2085	{
2086	pid_t ignore_pid = perf_thread_map__pid(threads, thread);
2087
2088	if (!evsel->ignore_missing_thread)
2089	return false;
2090
2091	/* The system wide setup does not work with threads. */
2092	if (evsel->core.system_wide)
2093	return false;
2094
2095	/* The -ESRCH is perf event syscall errno for pid's not found. */
2096	if (err != -ESRCH)
2097	return false;
2098
2099	/* If there's only one thread, let it fail. */
2100	if (threads->nr == 1)
2101	return false;
2102
2103	/*
2104	* We should remove fd for missing_thread first
2105	* because thread_map__remove() will decrease threads->nr.
2106	*/
2107	if (update_fds(evsel, nr_cpus, cpu_map_idx, nr_threads: threads->nr, thread_idx: thread))
2108	return false;
2109
2110	if (thread_map__remove(threads, idx: thread))
2111	return false;
2112
2113	pr_warning("WARNING: Ignored open failure for pid %d\n",
2114	ignore_pid);
2115	return true;
2116	}
2117
2118	static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
2119	void *priv __maybe_unused)
2120	{
2121	return fprintf(fp, " %-32s %s\n", name, val);
2122	}
2123
2124	static void display_attr(struct perf_event_attr *attr)
2125	{
2126	if (verbose >= 2 || debug_peo_args) {
2127	fprintf(stderr, "%.60s\n", graph_dotted_line);
2128	fprintf(stderr, "perf_event_attr:\n");
2129	perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
2130	fprintf(stderr, "%.60s\n", graph_dotted_line);
2131	}
2132	}
2133
2134	bool evsel__precise_ip_fallback(struct evsel *evsel)
2135	{
2136	/* Do not try less precise if not requested. */
2137	if (!evsel->precise_max)
2138	return false;
2139
2140	/*
2141	* We tried all the precise_ip values, and it's
2142	* still failing, so leave it to standard fallback.
2143	*/
2144	if (!evsel->core.attr.precise_ip) {
2145	evsel->core.attr.precise_ip = evsel->precise_ip_original;
2146	return false;
2147	}
2148
2149	if (!evsel->precise_ip_original)
2150	evsel->precise_ip_original = evsel->core.attr.precise_ip;
2151
2152	evsel->core.attr.precise_ip--;
2153	pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
2154	display_attr(attr: &evsel->core.attr);
2155	return true;
2156	}
2157
2158	static struct perf_cpu_map *empty_cpu_map;
2159	static struct perf_thread_map *empty_thread_map;
2160
2161	static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
2162	struct perf_thread_map *threads)
2163	{
2164	int ret = 0;
2165	int nthreads = perf_thread_map__nr(threads);
2166
2167	if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
2168	(perf_missing_features.aux_output && evsel->core.attr.aux_output))
2169	return -EINVAL;
2170
2171	if (cpus == NULL) {
2172	if (empty_cpu_map == NULL) {
2173	empty_cpu_map = perf_cpu_map__new_any_cpu();
2174	if (empty_cpu_map == NULL)
2175	return -ENOMEM;
2176	}
2177
2178	cpus = empty_cpu_map;
2179	}
2180
2181	if (threads == NULL) {
2182	if (empty_thread_map == NULL) {
2183	empty_thread_map = thread_map__new_by_tid(tid: -1);
2184	if (empty_thread_map == NULL)
2185	return -ENOMEM;
2186	}
2187
2188	threads = empty_thread_map;
2189	}
2190
2191	if (evsel->core.fd == NULL &&
2192	perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0)
2193	return -ENOMEM;
2194
2195	if (evsel__is_tool(evsel))
2196	ret = evsel__tool_pmu_prepare_open(evsel, cpus, nthreads);
2197
2198	evsel->open_flags = PERF_FLAG_FD_CLOEXEC;
2199	if (evsel->cgrp)
2200	evsel->open_flags |= PERF_FLAG_PID_CGROUP;
2201
2202	return ret;
2203	}
2204
2205	static void evsel__disable_missing_features(struct evsel *evsel)
2206	{
2207	if (perf_missing_features.defer_callchain && evsel->core.attr.defer_callchain)
2208	evsel->core.attr.defer_callchain = 0;
2209	if (perf_missing_features.defer_callchain && evsel->core.attr.defer_output)
2210	evsel->core.attr.defer_output = 0;
2211	if (perf_missing_features.inherit_sample_read && evsel->core.attr.inherit &&
2212	(evsel->core.attr.sample_type & PERF_SAMPLE_READ))
2213	evsel->core.attr.inherit = 0;
2214	if (perf_missing_features.branch_counters)
2215	evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_COUNTERS;
2216	if (perf_missing_features.read_lost)
2217	evsel->core.attr.read_format &= ~PERF_FORMAT_LOST;
2218	if (perf_missing_features.weight_struct) {
2219	evsel__set_sample_bit(evsel, WEIGHT);
2220	evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
2221	}
2222	if (perf_missing_features.clockid_wrong)
2223	evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */
2224	if (perf_missing_features.clockid) {
2225	evsel->core.attr.use_clockid = 0;
2226	evsel->core.attr.clockid = 0;
2227	}
2228	if (perf_missing_features.cloexec)
2229	evsel->open_flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
2230	if (perf_missing_features.mmap2)
2231	evsel->core.attr.mmap2 = 0;
2232	if (evsel->pmu && evsel->pmu->missing_features.exclude_guest)
2233	evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
2234	if (perf_missing_features.lbr_flags)
2235	evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
2236	PERF_SAMPLE_BRANCH_NO_CYCLES);
2237	if (perf_missing_features.group_read && evsel->core.attr.inherit)
2238	evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
2239	if (perf_missing_features.ksymbol)
2240	evsel->core.attr.ksymbol = 0;
2241	if (perf_missing_features.bpf)
2242	evsel->core.attr.bpf_event = 0;
2243	if (perf_missing_features.branch_hw_idx)
2244	evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX;
2245	if (perf_missing_features.sample_id_all)
2246	evsel->core.attr.sample_id_all = 0;
2247	}
2248
2249	int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
2250	struct perf_thread_map *threads)
2251	{
2252	int err;
2253
2254	err = __evsel__prepare_open(evsel, cpus, threads);
2255	if (err)
2256	return err;
2257
2258	evsel__disable_missing_features(evsel);
2259
2260	return err;
2261	}
2262
2263	static bool __has_attr_feature(struct perf_event_attr *attr,
2264	struct perf_cpu cpu, unsigned long flags)
2265	{
2266	int fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu,
2267	/*group_fd=*/-1, flags);
2268	close(fd);
2269
2270	if (fd < 0) {
2271	attr->exclude_kernel = 1;
2272
2273	fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu,
2274	/*group_fd=*/-1, flags);
2275	close(fd);
2276	}
2277
2278	if (fd < 0) {
2279	attr->exclude_hv = 1;
2280
2281	fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu,
2282	/*group_fd=*/-1, flags);
2283	close(fd);
2284	}
2285
2286	if (fd < 0) {
2287	attr->exclude_guest = 1;
2288
2289	fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu,
2290	/*group_fd=*/-1, flags);
2291	close(fd);
2292	}
2293
2294	attr->exclude_kernel = 0;
2295	attr->exclude_guest = 0;
2296	attr->exclude_hv = 0;
2297
2298	return fd >= 0;
2299	}
2300
2301	static bool has_attr_feature(struct perf_event_attr *attr, unsigned long flags)
2302	{
2303	struct perf_cpu cpu = {.cpu = -1};
2304
2305	return __has_attr_feature(attr, cpu: cpu, flags);
2306	}
2307
2308	static void evsel__detect_missing_pmu_features(struct evsel *evsel)
2309	{
2310	struct perf_event_attr attr = {
2311	.type = evsel->core.attr.type,
2312	.config = evsel->core.attr.config,
2313	.disabled = 1,
2314	};
2315	struct perf_pmu *pmu = evsel->pmu;
2316	int old_errno;
2317
2318	old_errno = errno;
2319
2320	if (pmu == NULL)
2321	pmu = evsel->pmu = evsel__find_pmu(evsel);
2322
2323	if (pmu == NULL || pmu->missing_features.checked)
2324	goto out;
2325
2326	/*
2327	* Must probe features in the order they were added to the
2328	* perf_event_attr interface. These are kernel core limitation but
2329	* specific to PMUs with branch stack. So we can detect with the given
2330	* hardware event and stop on the first one succeeded.
2331	*/
2332
2333	/* Please add new feature detection here. */
2334
2335	attr.exclude_guest = 1;
2336	if (has_attr_feature(attr: &attr, /*flags=*/0))
2337	goto found;
2338	pmu->missing_features.exclude_guest = true;
2339	pr_debug2("switching off exclude_guest for PMU %s\n", pmu->name);
2340
2341	found:
2342	pmu->missing_features.checked = true;
2343	out:
2344	errno = old_errno;
2345	}
2346
2347	static void evsel__detect_missing_brstack_features(struct evsel *evsel)
2348	{
2349	static bool detection_done = false;
2350	struct perf_event_attr attr = {
2351	.type = evsel->core.attr.type,
2352	.config = evsel->core.attr.config,
2353	.disabled = 1,
2354	.sample_type = PERF_SAMPLE_BRANCH_STACK,
2355	.sample_period = 1000,
2356	};
2357	int old_errno;
2358
2359	if (detection_done)
2360	return;
2361
2362	old_errno = errno;
2363
2364	/*
2365	* Must probe features in the order they were added to the
2366	* perf_event_attr interface. These are PMU specific limitation
2367	* so we can detect with the given hardware event and stop on the
2368	* first one succeeded.
2369	*/
2370
2371	/* Please add new feature detection here. */
2372
2373	attr.branch_sample_type = PERF_SAMPLE_BRANCH_COUNTERS;
2374	if (has_attr_feature(attr: &attr, /*flags=*/0))
2375	goto found;
2376	perf_missing_features.branch_counters = true;
2377	pr_debug2("switching off branch counters support\n");
2378
2379	attr.branch_sample_type = PERF_SAMPLE_BRANCH_HW_INDEX;
2380	if (has_attr_feature(attr: &attr, /*flags=*/0))
2381	goto found;
2382	perf_missing_features.branch_hw_idx = true;
2383	pr_debug2("switching off branch HW index support\n");
2384
2385	attr.branch_sample_type = PERF_SAMPLE_BRANCH_NO_CYCLES | PERF_SAMPLE_BRANCH_NO_FLAGS;
2386	if (has_attr_feature(attr: &attr, /*flags=*/0))
2387	goto found;
2388	perf_missing_features.lbr_flags = true;
2389	pr_debug2_peo("switching off branch sample type no (cycles/flags)\n");
2390
2391	found:
2392	detection_done = true;
2393	errno = old_errno;
2394	}
2395
2396	static bool evsel__probe_aux_action(struct evsel *evsel, struct perf_cpu cpu)
2397	{
2398	struct perf_event_attr attr = evsel->core.attr;
2399	int old_errno = errno;
2400
2401	attr.disabled = 1;
2402	attr.aux_start_paused = 1;
2403
2404	if (__has_attr_feature(attr: &attr, cpu: cpu, /*flags=*/0)) {
2405	errno = old_errno;
2406	return true;
2407	}
2408
2409	/*
2410	* EOPNOTSUPP means the kernel supports the feature but the PMU does
2411	* not, so keep that distinction if possible.
2412	*/
2413	if (errno != EOPNOTSUPP)
2414	errno = old_errno;
2415
2416	return false;
2417	}
2418
2419	static void evsel__detect_missing_aux_action_feature(struct evsel *evsel, struct perf_cpu cpu)
2420	{
2421	static bool detection_done;
2422	struct evsel *leader;
2423
2424	/*
2425	* Don't bother probing aux_action if it is not being used or has been
2426	* probed before.
2427	*/
2428	if (!evsel->core.attr.aux_action || detection_done)
2429	return;
2430
2431	detection_done = true;
2432
2433	/*
2434	* The leader is an AUX area event. If it has failed, assume the feature
2435	* is not supported.
2436	*/
2437	leader = evsel__leader(evsel);
2438	if (evsel == leader) {
2439	perf_missing_features.aux_action = true;
2440	return;
2441	}
2442
2443	/*
2444	* AUX area event with aux_action must have been opened successfully
2445	* already, so feature is supported.
2446	*/
2447	if (leader->core.attr.aux_action)
2448	return;
2449
2450	if (!evsel__probe_aux_action(evsel: leader, cpu: cpu))
2451	perf_missing_features.aux_action = true;
2452	}
2453
2454	static bool evsel__detect_missing_features(struct evsel *evsel, struct perf_cpu cpu)
2455	{
2456	static bool detection_done = false;
2457	struct perf_event_attr attr = {
2458	.type = PERF_TYPE_SOFTWARE,
2459	.config = PERF_COUNT_SW_TASK_CLOCK,
2460	.disabled = 1,
2461	};
2462	int old_errno;
2463
2464	evsel__detect_missing_aux_action_feature(evsel, cpu: cpu);
2465
2466	evsel__detect_missing_pmu_features(evsel);
2467
2468	if (evsel__has_br_stack(evsel))
2469	evsel__detect_missing_brstack_features(evsel);
2470
2471	if (detection_done)
2472	goto check;
2473
2474	old_errno = errno;
2475
2476	/*
2477	* Must probe features in the order they were added to the
2478	* perf_event_attr interface. These are kernel core limitation
2479	* not PMU-specific so we can detect with a software event and
2480	* stop on the first one succeeded.
2481	*/
2482
2483	/* Please add new feature detection here. */
2484
2485	attr.defer_callchain = true;
2486	if (has_attr_feature(attr: &attr, /*flags=*/0))
2487	goto found;
2488	perf_missing_features.defer_callchain = true;
2489	pr_debug2("switching off deferred callchain support\n");
2490	attr.defer_callchain = false;
2491
2492	attr.inherit = true;
2493	attr.sample_type = PERF_SAMPLE_READ | PERF_SAMPLE_TID;
2494	if (has_attr_feature(attr: &attr, /*flags=*/0))
2495	goto found;
2496	perf_missing_features.inherit_sample_read = true;
2497	pr_debug2("Using PERF_SAMPLE_READ / :S modifier is not compatible with inherit, falling back to no-inherit.\n");
2498	attr.inherit = false;
2499	attr.sample_type = 0;
2500
2501	attr.read_format = PERF_FORMAT_LOST;
2502	if (has_attr_feature(attr: &attr, /*flags=*/0))
2503	goto found;
2504	perf_missing_features.read_lost = true;
2505	pr_debug2("switching off PERF_FORMAT_LOST support\n");
2506	attr.read_format = 0;
2507
2508	attr.sample_type = PERF_SAMPLE_WEIGHT_STRUCT;
2509	if (has_attr_feature(attr: &attr, /*flags=*/0))
2510	goto found;
2511	perf_missing_features.weight_struct = true;
2512	pr_debug2("switching off weight struct support\n");
2513	attr.sample_type = 0;
2514
2515	attr.sample_type = PERF_SAMPLE_CODE_PAGE_SIZE;
2516	if (has_attr_feature(attr: &attr, /*flags=*/0))
2517	goto found;
2518	perf_missing_features.code_page_size = true;
2519	pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support\n");
2520	attr.sample_type = 0;
2521
2522	attr.sample_type = PERF_SAMPLE_DATA_PAGE_SIZE;
2523	if (has_attr_feature(attr: &attr, /*flags=*/0))
2524	goto found;
2525	perf_missing_features.data_page_size = true;
2526	pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support\n");
2527	attr.sample_type = 0;
2528
2529	attr.cgroup = 1;
2530	if (has_attr_feature(attr: &attr, /*flags=*/0))
2531	goto found;
2532	perf_missing_features.cgroup = true;
2533	pr_debug2_peo("Kernel has no cgroup sampling support\n");
2534	attr.cgroup = 0;
2535
2536	attr.aux_output = 1;
2537	if (has_attr_feature(attr: &attr, /*flags=*/0))
2538	goto found;
2539	perf_missing_features.aux_output = true;
2540	pr_debug2_peo("Kernel has no attr.aux_output support\n");
2541	attr.aux_output = 0;
2542
2543	attr.bpf_event = 1;
2544	if (has_attr_feature(attr: &attr, /*flags=*/0))
2545	goto found;
2546	perf_missing_features.bpf = true;
2547	pr_debug2_peo("switching off bpf_event\n");
2548	attr.bpf_event = 0;
2549
2550	attr.ksymbol = 1;
2551	if (has_attr_feature(attr: &attr, /*flags=*/0))
2552	goto found;
2553	perf_missing_features.ksymbol = true;
2554	pr_debug2_peo("switching off ksymbol\n");
2555	attr.ksymbol = 0;
2556
2557	attr.write_backward = 1;
2558	if (has_attr_feature(attr: &attr, /*flags=*/0))
2559	goto found;
2560	perf_missing_features.write_backward = true;
2561	pr_debug2_peo("switching off write_backward\n");
2562	attr.write_backward = 0;
2563
2564	attr.use_clockid = 1;
2565	attr.clockid = CLOCK_MONOTONIC;
2566	if (has_attr_feature(attr: &attr, /*flags=*/0))
2567	goto found;
2568	perf_missing_features.clockid = true;
2569	pr_debug2_peo("switching off clockid\n");
2570	attr.use_clockid = 0;
2571	attr.clockid = 0;
2572
2573	if (has_attr_feature(attr: &attr, /*flags=*/PERF_FLAG_FD_CLOEXEC))
2574	goto found;
2575	perf_missing_features.cloexec = true;
2576	pr_debug2_peo("switching off cloexec flag\n");
2577
2578	attr.mmap2 = 1;
2579	if (has_attr_feature(attr: &attr, /*flags=*/0))
2580	goto found;
2581	perf_missing_features.mmap2 = true;
2582	pr_debug2_peo("switching off mmap2\n");
2583	attr.mmap2 = 0;
2584
2585	/* set this unconditionally? */
2586	perf_missing_features.sample_id_all = true;
2587	pr_debug2_peo("switching off sample_id_all\n");
2588
2589	attr.inherit = 1;
2590	attr.read_format = PERF_FORMAT_GROUP;
2591	if (has_attr_feature(attr: &attr, /*flags=*/0))
2592	goto found;
2593	perf_missing_features.group_read = true;
2594	pr_debug2_peo("switching off group read\n");
2595	attr.inherit = 0;
2596	attr.read_format = 0;
2597
2598	found:
2599	detection_done = true;
2600	errno = old_errno;
2601
2602	check:
2603	if ((evsel->core.attr.defer_callchain || evsel->core.attr.defer_output) &&
2604	perf_missing_features.defer_callchain)
2605	return true;
2606
2607	if (evsel->core.attr.inherit &&
2608	(evsel->core.attr.sample_type & PERF_SAMPLE_READ) &&
2609	perf_missing_features.inherit_sample_read)
2610	return true;
2611
2612	if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) &&
2613	perf_missing_features.branch_counters)
2614	return true;
2615
2616	if ((evsel->core.attr.read_format & PERF_FORMAT_LOST) &&
2617	perf_missing_features.read_lost)
2618	return true;
2619
2620	if ((evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT) &&
2621	perf_missing_features.weight_struct)
2622	return true;
2623
2624	if (evsel->core.attr.use_clockid && evsel->core.attr.clockid != CLOCK_MONOTONIC &&
2625	!perf_missing_features.clockid) {
2626	perf_missing_features.clockid_wrong = true;
2627	return true;
2628	}
2629
2630	if (evsel->core.attr.use_clockid && perf_missing_features.clockid)
2631	return true;
2632
2633	if ((evsel->open_flags & PERF_FLAG_FD_CLOEXEC) &&
2634	perf_missing_features.cloexec)
2635	return true;
2636
2637	if (evsel->core.attr.mmap2 && perf_missing_features.mmap2)
2638	return true;
2639
2640	if ((evsel->core.attr.branch_sample_type & (PERF_SAMPLE_BRANCH_NO_FLAGS |
2641	PERF_SAMPLE_BRANCH_NO_CYCLES)) &&
2642	perf_missing_features.lbr_flags)
2643	return true;
2644
2645	if (evsel->core.attr.inherit && (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
2646	perf_missing_features.group_read)
2647	return true;
2648
2649	if (evsel->core.attr.ksymbol && perf_missing_features.ksymbol)
2650	return true;
2651
2652	if (evsel->core.attr.bpf_event && perf_missing_features.bpf)
2653	return true;
2654
2655	if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) &&
2656	perf_missing_features.branch_hw_idx)
2657	return true;
2658
2659	if (evsel->core.attr.sample_id_all && perf_missing_features.sample_id_all)
2660	return true;
2661
2662	return false;
2663	}
2664
2665	static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus,
2666	struct perf_thread_map *threads,
2667	int start_cpu_map_idx, int end_cpu_map_idx)
2668	{
2669	int idx, thread, nthreads;
2670	int pid = -1, err, old_errno;
2671	enum rlimit_action set_rlimit = NO_CHANGE;
2672	struct perf_cpu cpu;
2673
2674	if (evsel__is_retire_lat(evsel)) {
2675	err = evsel__tpebs_open(evsel);
2676	goto out;
2677	}
2678
2679	err = __evsel__prepare_open(evsel, cpus, threads);
2680	if (err)
2681	goto out;
2682
2683	if (cpus == NULL)
2684	cpus = empty_cpu_map;
2685
2686	if (threads == NULL)
2687	threads = empty_thread_map;
2688
2689	nthreads = perf_thread_map__nr(threads);
2690
2691	if (evsel->cgrp)
2692	pid = evsel->cgrp->fd;
2693
2694	fallback_missing_features:
2695	evsel__disable_missing_features(evsel);
2696
2697	pr_debug3("Opening: %s\n", evsel__name(evsel));
2698	display_attr(attr: &evsel->core.attr);
2699
2700	if (evsel__is_tool(evsel)) {
2701	err = evsel__tool_pmu_open(evsel, threads,
2702	start_cpu_map_idx,
2703	end_cpu_map_idx);
2704	goto out;
2705	}
2706	if (evsel__is_hwmon(evsel)) {
2707	err = evsel__hwmon_pmu_open(evsel, threads,
2708	start_cpu_map_idx,
2709	end_cpu_map_idx);
2710	goto out;
2711	}
2712	if (evsel__is_drm(evsel)) {
2713	err = evsel__drm_pmu_open(evsel, threads,
2714	start_cpu_map_idx,
2715	end_cpu_map_idx);
2716	goto out;
2717	}
2718
2719	for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) {
2720	cpu = perf_cpu_map__cpu(cpus, idx);
2721
2722	for (thread = 0; thread < nthreads; thread++) {
2723	int fd, group_fd;
2724	retry_open:
2725	if (thread >= nthreads)
2726	break;
2727
2728	if (!evsel->cgrp && !evsel->core.system_wide)
2729	pid = perf_thread_map__pid(threads, thread);
2730
2731	group_fd = get_group_fd(evsel, cpu_map_idx: idx, thread);
2732
2733	if (group_fd == -2) {
2734	pr_debug("broken group leader for %s\n", evsel->name);
2735	err = -EINVAL;
2736	goto out_close;
2737	}
2738
2739	/* Debug message used by test scripts */
2740	pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx",
2741	pid, cpu.cpu, group_fd, evsel->open_flags);
2742
2743	fd = sys_perf_event_open(attr: &evsel->core.attr, pid, cpu: cpu.cpu,
2744	group_fd, flags: evsel->open_flags);
2745
2746	FD(evsel, idx, thread) = fd;
2747
2748	if (fd < 0) {
2749	err = -errno;
2750
2751	pr_debug2_peo("\nsys_perf_event_open failed, error %d\n",
2752	err);
2753	goto try_fallback;
2754	}
2755
2756	bpf_counter__install_pe(evsel, cpu: idx, fd);
2757
2758	if (unlikely(test_attr__enabled())) {
2759	test_attr__open(attr: &evsel->core.attr, pid, cpu: cpu,
2760	fd, group_fd, flags: evsel->open_flags);
2761	}
2762
2763	/* Debug message used by test scripts */
2764	pr_debug2_peo(" = %d\n", fd);
2765
2766	if (evsel->bpf_fd >= 0) {
2767	int evt_fd = fd;
2768	int bpf_fd = evsel->bpf_fd;
2769
2770	err = ioctl(evt_fd,
2771	PERF_EVENT_IOC_SET_BPF,
2772	bpf_fd);
2773	if (err && errno != EEXIST) {
2774	pr_err("failed to attach bpf fd %d: %s\n",
2775	bpf_fd, strerror(errno));
2776	err = -EINVAL;
2777	goto out_close;
2778	}
2779	}
2780
2781	set_rlimit = NO_CHANGE;
2782
2783	/*
2784	* If we succeeded but had to kill clockid, fail and
2785	* have evsel__open_strerror() print us a nice error.
2786	*/
2787	if (perf_missing_features.clockid ||
2788	perf_missing_features.clockid_wrong) {
2789	err = -EINVAL;
2790	goto out_close;
2791	}
2792	}
2793	}
2794
2795	err = 0;
2796	goto out;
2797
2798	try_fallback:
2799	if (evsel__ignore_missing_thread(evsel, nr_cpus: perf_cpu_map__nr(cpus),
2800	cpu_map_idx: idx, threads, thread, err)) {
2801	/* We just removed 1 thread, so lower the upper nthreads limit. */
2802	nthreads--;
2803
2804	/* ... and pretend like nothing have happened. */
2805	err = 0;
2806	goto retry_open;
2807	}
2808	/*
2809	* perf stat needs between 5 and 22 fds per CPU. When we run out
2810	* of them try to increase the limits.
2811	*/
2812	if (err == -EMFILE && rlimit__increase_nofile(set_rlimit: &set_rlimit))
2813	goto retry_open;
2814
2815	if (err == -EINVAL && evsel__detect_missing_features(evsel, cpu: cpu))
2816	goto fallback_missing_features;
2817
2818	if (evsel__precise_ip_fallback(evsel))
2819	goto retry_open;
2820
2821	out_close:
2822	if (err)
2823	threads->err_thread = thread;
2824
2825	old_errno = errno;
2826	do {
2827	while (--thread >= 0) {
2828	if (FD(evsel, idx, thread) >= 0)
2829	close(FD(evsel, idx, thread));
2830	FD(evsel, idx, thread) = -1;
2831	}
2832	thread = nthreads;
2833	} while (--idx >= 0);
2834	errno = old_errno;
2835	out:
2836	if (err)
2837	evsel->supported = false;
2838	return err;
2839	}
2840
2841	int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
2842	struct perf_thread_map *threads)
2843	{
2844	return evsel__open_cpu(evsel, cpus, threads, start_cpu_map_idx: 0, end_cpu_map_idx: perf_cpu_map__nr(cpus));
2845	}
2846
2847	void evsel__close(struct evsel *evsel)
2848	{
2849	if (evsel__is_retire_lat(evsel))
2850	evsel__tpebs_close(evsel);
2851	perf_evsel__close(&evsel->core);
2852	perf_evsel__free_id(&evsel->core);
2853	}
2854
2855	int evsel__open_per_cpu_and_thread(struct evsel *evsel,
2856	struct perf_cpu_map *cpus, int cpu_map_idx,
2857	struct perf_thread_map *threads)
2858	{
2859	if (cpu_map_idx == -1)
2860	return evsel__open_cpu(evsel, cpus, threads, start_cpu_map_idx: 0, end_cpu_map_idx: perf_cpu_map__nr(cpus));
2861
2862	return evsel__open_cpu(evsel, cpus, threads, start_cpu_map_idx: cpu_map_idx, end_cpu_map_idx: cpu_map_idx + 1);
2863	}
2864
2865	int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx)
2866	{
2867	struct perf_thread_map *threads = thread_map__new_by_tid(tid: -1);
2868	int ret = evsel__open_per_cpu_and_thread(evsel, cpus, cpu_map_idx, threads);
2869
2870	perf_thread_map__put(threads);
2871	return ret;
2872	}
2873
2874	int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads)
2875	{
2876	struct perf_cpu_map *cpus = perf_cpu_map__new_any_cpu();
2877	int ret = evsel__open_per_cpu_and_thread(evsel, cpus, cpu_map_idx: -1, threads);
2878
2879	perf_cpu_map__put(cpus);
2880	return ret;
2881	}
2882
2883	static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2884	const union perf_event *event,
2885	struct perf_sample *sample)
2886	{
2887	u64 type = evsel->core.attr.sample_type;
2888	const __u64 *array = event->sample.array;
2889	bool swapped = evsel->needs_swap;
2890	union u64_swap u;
2891
2892	array += ((event->header.size -
2893	sizeof(event->header)) / sizeof(u64)) - 1;
2894
2895	if (type & PERF_SAMPLE_IDENTIFIER) {
2896	sample->id = *array;
2897	array--;
2898	}
2899
2900	if (type & PERF_SAMPLE_CPU) {
2901	u.val64 = *array;
2902	if (swapped) {
2903	/* undo swap of u64, then swap on individual u32s */
2904	u.val64 = bswap_64(u.val64);
2905	u.val32[0] = bswap_32(u.val32[0]);
2906	}
2907
2908	sample->cpu = u.val32[0];
2909	array--;
2910	}
2911
2912	if (type & PERF_SAMPLE_STREAM_ID) {
2913	sample->stream_id = *array;
2914	array--;
2915	}
2916
2917	if (type & PERF_SAMPLE_ID) {
2918	sample->id = *array;
2919	array--;
2920	}
2921
2922	if (type & PERF_SAMPLE_TIME) {
2923	sample->time = *array;
2924	array--;
2925	}
2926
2927	if (type & PERF_SAMPLE_TID) {
2928	u.val64 = *array;
2929	if (swapped) {
2930	/* undo swap of u64, then swap on individual u32s */
2931	u.val64 = bswap_64(u.val64);
2932	u.val32[0] = bswap_32(u.val32[0]);
2933	u.val32[1] = bswap_32(u.val32[1]);
2934	}
2935
2936	sample->pid = u.val32[0];
2937	sample->tid = u.val32[1];
2938	array--;
2939	}
2940
2941	return 0;
2942	}
2943
2944	static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2945	u64 size)
2946	{
2947	return size > max_size || offset + size > endp;
2948	}
2949
2950	#define OVERFLOW_CHECK(offset, size, max_size) \
2951	do { \
2952	if (overflow(endp, (max_size), (offset), (size))) \
2953	return -EFAULT; \
2954	} while (0)
2955
2956	#define OVERFLOW_CHECK_u64(offset) \
2957	OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2958
2959	static int
2960	perf_event__check_size(union perf_event *event, unsigned int sample_size)
2961	{
2962	/*
2963	* The evsel's sample_size is based on PERF_SAMPLE_MASK which includes
2964	* up to PERF_SAMPLE_PERIOD. After that overflow() must be used to
2965	* check the format does not go past the end of the event.
2966	*/
2967	if (sample_size + sizeof(event->header) > event->header.size)
2968	return -EFAULT;
2969
2970	return 0;
2971	}
2972
2973	static void perf_parse_sample_weight(struct perf_sample *data, const __u64 *array, u64 type)
2974	{
2975	union perf_sample_weight weight;
2976
2977	weight.full = *array;
2978	if (type & PERF_SAMPLE_WEIGHT_STRUCT) {
2979	data->weight = weight.var1_dw;
2980	data->ins_lat = weight.var2_w;
2981	data->weight3 = weight.var3_w;
2982	} else {
2983	data->weight = weight.full;
2984	}
2985	}
2986
2987	u64 evsel__bitfield_swap_branch_flags(u64 value)
2988	{
2989	u64 new_val = 0;
2990
2991	/*
2992	* branch_flags
2993	* union {
2994	* u64 values;
2995	* struct {
2996	* mispred:1 //target mispredicted
2997	* predicted:1 //target predicted
2998	* in_tx:1 //in transaction
2999	* abort:1 //transaction abort
3000	* cycles:16 //cycle count to last branch
3001	* type:4 //branch type
3002	* spec:2 //branch speculation info
3003	* new_type:4 //additional branch type
3004	* priv:3 //privilege level
3005	* reserved:31
3006	* }
3007	* }
3008	*
3009	* Avoid bswap64() the entire branch_flag.value,
3010	* as it has variable bit-field sizes. Instead the
3011	* macro takes the bit-field position/size,
3012	* swaps it based on the host endianness.
3013	*/
3014	if (host_is_bigendian()) {
3015	new_val = bitfield_swap(value, 0, 1);
3016	new_val |= bitfield_swap(value, 1, 1);
3017	new_val |= bitfield_swap(value, 2, 1);
3018	new_val |= bitfield_swap(value, 3, 1);
3019	new_val |= bitfield_swap(value, 4, 16);
3020	new_val |= bitfield_swap(value, 20, 4);
3021	new_val |= bitfield_swap(value, 24, 2);
3022	new_val |= bitfield_swap(value, 26, 4);
3023	new_val |= bitfield_swap(value, 30, 3);
3024	new_val |= bitfield_swap(value, 33, 31);
3025	} else {
3026	new_val = bitfield_swap(value, 63, 1);
3027	new_val |= bitfield_swap(value, 62, 1);
3028	new_val |= bitfield_swap(value, 61, 1);
3029	new_val |= bitfield_swap(value, 60, 1);
3030	new_val |= bitfield_swap(value, 44, 16);
3031	new_val |= bitfield_swap(value, 40, 4);
3032	new_val |= bitfield_swap(value, 38, 2);
3033	new_val |= bitfield_swap(value, 34, 4);
3034	new_val |= bitfield_swap(value, 31, 3);
3035	new_val |= bitfield_swap(value, 0, 31);
3036	}
3037
3038	return new_val;
3039	}
3040
3041	static inline bool evsel__has_branch_counters(const struct evsel *evsel)
3042	{
3043	struct evsel *leader = evsel__leader(evsel);
3044
3045	/* The branch counters feature only supports group */
3046	if (!leader || !evsel->evlist)
3047	return false;
3048
3049	if (evsel->evlist->nr_br_cntr < 0)
3050	evlist__update_br_cntr(evlist: evsel->evlist);
3051
3052	if (leader->br_cntr_nr > 0)
3053	return true;
3054
3055	return false;
3056	}
3057
3058	static int __set_offcpu_sample(struct perf_sample *data)
3059	{
3060	u64 *array = data->raw_data;
3061	u32 max_size = data->raw_size, *p32;
3062	const void *endp = (void *)array + max_size;
3063
3064	if (array == NULL)
3065	return -EFAULT;
3066
3067	OVERFLOW_CHECK_u64(array);
3068	p32 = (void *)array++;
3069	data->pid = p32[0];
3070	data->tid = p32[1];
3071
3072	OVERFLOW_CHECK_u64(array);
3073	data->period = *array++;
3074
3075	OVERFLOW_CHECK_u64(array);
3076	data->callchain = (struct ip_callchain *)array++;
3077	OVERFLOW_CHECK(array, data->callchain->nr * sizeof(u64), max_size);
3078	data->ip = data->callchain->ips[1];
3079	array += data->callchain->nr;
3080
3081	OVERFLOW_CHECK_u64(array);
3082	data->cgroup = *array;
3083
3084	return 0;
3085	}
3086
3087	int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
3088	struct perf_sample *data)
3089	{
3090	u64 type = evsel->core.attr.sample_type;
3091	bool swapped = evsel->needs_swap;
3092	const __u64 *array;
3093	u16 max_size = event->header.size;
3094	const void *endp = (void *)event + max_size;
3095	u64 sz;
3096
3097	/*
3098	* used for cross-endian analysis. See git commit 65014ab3
3099	* for why this goofiness is needed.
3100	*/
3101	union u64_swap u;
3102
3103	memset(data, 0, sizeof(*data));
3104	data->cpu = data->pid = data->tid = -1;
3105	data->stream_id = data->id = data->time = -1ULL;
3106	data->period = evsel->core.attr.sample_period;
3107	data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
3108	data->misc = event->header.misc;
3109	data->data_src = PERF_MEM_DATA_SRC_NONE;
3110	data->vcpu = -1;
3111
3112	if (event->header.type == PERF_RECORD_CALLCHAIN_DEFERRED) {
3113	const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
3114
3115	data->callchain = (struct ip_callchain *)&event->callchain_deferred.nr;
3116	if (data->callchain->nr > max_callchain_nr)
3117	return -EFAULT;
3118
3119	data->deferred_cookie = event->callchain_deferred.cookie;
3120
3121	if (evsel->core.attr.sample_id_all)
3122	perf_evsel__parse_id_sample(evsel, event, sample: data);
3123	return 0;
3124	}
3125
3126	if (event->header.type != PERF_RECORD_SAMPLE) {
3127	if (!evsel->core.attr.sample_id_all)
3128	return 0;
3129	return perf_evsel__parse_id_sample(evsel, event, sample: data);
3130	}
3131
3132	array = event->sample.array;
3133
3134	if (perf_event__check_size(event, sample_size: evsel->sample_size))
3135	return -EFAULT;
3136
3137	if (type & PERF_SAMPLE_IDENTIFIER) {
3138	data->id = *array;
3139	array++;
3140	}
3141
3142	if (type & PERF_SAMPLE_IP) {
3143	data->ip = *array;
3144	array++;
3145	}
3146
3147	if (type & PERF_SAMPLE_TID) {
3148	u.val64 = *array;
3149	if (swapped) {
3150	/* undo swap of u64, then swap on individual u32s */
3151	u.val64 = bswap_64(u.val64);
3152	u.val32[0] = bswap_32(u.val32[0]);
3153	u.val32[1] = bswap_32(u.val32[1]);
3154	}
3155
3156	data->pid = u.val32[0];
3157	data->tid = u.val32[1];
3158	array++;
3159	}
3160
3161	if (type & PERF_SAMPLE_TIME) {
3162	data->time = *array;
3163	array++;
3164	}
3165
3166	if (type & PERF_SAMPLE_ADDR) {
3167	data->addr = *array;
3168	array++;
3169	}
3170
3171	if (type & PERF_SAMPLE_ID) {
3172	data->id = *array;
3173	array++;
3174	}
3175
3176	if (type & PERF_SAMPLE_STREAM_ID) {
3177	data->stream_id = *array;
3178	array++;
3179	}
3180
3181	if (type & PERF_SAMPLE_CPU) {
3182
3183	u.val64 = *array;
3184	if (swapped) {
3185	/* undo swap of u64, then swap on individual u32s */
3186	u.val64 = bswap_64(u.val64);
3187	u.val32[0] = bswap_32(u.val32[0]);
3188	}
3189
3190	data->cpu = u.val32[0];
3191	array++;
3192	}
3193
3194	if (type & PERF_SAMPLE_PERIOD) {
3195	data->period = *array;
3196	array++;
3197	}
3198
3199	if (type & PERF_SAMPLE_READ) {
3200	u64 read_format = evsel->core.attr.read_format;
3201
3202	OVERFLOW_CHECK_u64(array);
3203	if (read_format & PERF_FORMAT_GROUP)
3204	data->read.group.nr = *array;
3205	else
3206	data->read.one.value = *array;
3207
3208	array++;
3209
3210	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3211	OVERFLOW_CHECK_u64(array);
3212	data->read.time_enabled = *array;
3213	array++;
3214	}
3215
3216	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3217	OVERFLOW_CHECK_u64(array);
3218	data->read.time_running = *array;
3219	array++;
3220	}
3221
3222	/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
3223	if (read_format & PERF_FORMAT_GROUP) {
3224	const u64 max_group_nr = UINT64_MAX /
3225	sizeof(struct sample_read_value);
3226
3227	if (data->read.group.nr > max_group_nr)
3228	return -EFAULT;
3229
3230	sz = data->read.group.nr * sample_read_value_size(read_format);
3231	OVERFLOW_CHECK(array, sz, max_size);
3232	data->read.group.values =
3233	(struct sample_read_value *)array;
3234	array = (void *)array + sz;
3235	} else {
3236	OVERFLOW_CHECK_u64(array);
3237	data->read.one.id = *array;
3238	array++;
3239
3240	if (read_format & PERF_FORMAT_LOST) {
3241	OVERFLOW_CHECK_u64(array);
3242	data->read.one.lost = *array;
3243	array++;
3244	}
3245	}
3246	}
3247
3248	if (type & PERF_SAMPLE_CALLCHAIN) {
3249	const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
3250	u64 callchain_nr;
3251
3252	OVERFLOW_CHECK_u64(array);
3253	data->callchain = (struct ip_callchain *)array++;
3254	callchain_nr = data->callchain->nr;
3255	if (callchain_nr > max_callchain_nr)
3256	return -EFAULT;
3257	sz = callchain_nr * sizeof(u64);
3258	/*
3259	* Save the cookie for the deferred user callchain. The last 2
3260	* entries in the callchain should be the context marker and the
3261	* cookie. The cookie will be used to match PERF_RECORD_
3262	* CALLCHAIN_DEFERRED later.
3263	*/
3264	if (evsel->core.attr.defer_callchain && callchain_nr >= 2 &&
3265	data->callchain->ips[callchain_nr - 2] == PERF_CONTEXT_USER_DEFERRED) {
3266	data->deferred_cookie = data->callchain->ips[callchain_nr - 1];
3267	data->deferred_callchain = true;
3268	}
3269	OVERFLOW_CHECK(array, sz, max_size);
3270	array = (void *)array + sz;
3271	}
3272
3273	if (type & PERF_SAMPLE_RAW) {
3274	OVERFLOW_CHECK_u64(array);
3275	u.val64 = *array;
3276
3277	/*
3278	* Undo swap of u64, then swap on individual u32s,
3279	* get the size of the raw area and undo all of the
3280	* swap. The pevent interface handles endianness by
3281	* itself.
3282	*/
3283	if (swapped) {
3284	u.val64 = bswap_64(u.val64);
3285	u.val32[0] = bswap_32(u.val32[0]);
3286	u.val32[1] = bswap_32(u.val32[1]);
3287	}
3288	data->raw_size = u.val32[0];
3289
3290	/*
3291	* The raw data is aligned on 64bits including the
3292	* u32 size, so it's safe to use mem_bswap_64.
3293	*/
3294	if (swapped)
3295	mem_bswap_64(src: (void *) array, byte_size: data->raw_size);
3296
3297	array = (void *)array + sizeof(u32);
3298
3299	OVERFLOW_CHECK(array, data->raw_size, max_size);
3300	data->raw_data = (void *)array;
3301	array = (void *)array + data->raw_size;
3302	}
3303
3304	if (type & PERF_SAMPLE_BRANCH_STACK) {
3305	const u64 max_branch_nr = UINT64_MAX /
3306	sizeof(struct branch_entry);
3307	struct branch_entry *e;
3308	unsigned int i;
3309
3310	OVERFLOW_CHECK_u64(array);
3311	data->branch_stack = (struct branch_stack *)array++;
3312
3313	if (data->branch_stack->nr > max_branch_nr)
3314	return -EFAULT;
3315
3316	sz = data->branch_stack->nr * sizeof(struct branch_entry);
3317	if (evsel__has_branch_hw_idx(evsel)) {
3318	sz += sizeof(u64);
3319	e = &data->branch_stack->entries[0];
3320	} else {
3321	data->no_hw_idx = true;
3322	/*
3323	* if the PERF_SAMPLE_BRANCH_HW_INDEX is not applied,
3324	* only nr and entries[] will be output by kernel.
3325	*/
3326	e = (struct branch_entry *)&data->branch_stack->hw_idx;
3327	}
3328
3329	if (swapped) {
3330	/*
3331	* struct branch_flag does not have endian
3332	* specific bit field definition. And bswap
3333	* will not resolve the issue, since these
3334	* are bit fields.
3335	*
3336	* evsel__bitfield_swap_branch_flags() uses a
3337	* bitfield_swap macro to swap the bit position
3338	* based on the host endians.
3339	*/
3340	for (i = 0; i < data->branch_stack->nr; i++, e++)
3341	e->flags.value = evsel__bitfield_swap_branch_flags(value: e->flags.value);
3342	}
3343
3344	OVERFLOW_CHECK(array, sz, max_size);
3345	array = (void *)array + sz;
3346
3347	if (evsel__has_branch_counters(evsel)) {
3348	data->branch_stack_cntr = (u64 *)array;
3349	sz = data->branch_stack->nr * sizeof(u64);
3350
3351	OVERFLOW_CHECK(array, sz, max_size);
3352	array = (void *)array + sz;
3353	}
3354	}
3355
3356	if (type & PERF_SAMPLE_REGS_USER) {
3357	struct regs_dump *regs = perf_sample__user_regs(data);
3358
3359	OVERFLOW_CHECK_u64(array);
3360	regs->abi = *array;
3361	array++;
3362
3363	if (regs->abi) {
3364	u64 mask = evsel->core.attr.sample_regs_user;
3365
3366	sz = hweight64(mask) * sizeof(u64);
3367	OVERFLOW_CHECK(array, sz, max_size);
3368	regs->mask = mask;
3369	regs->regs = (u64 *)array;
3370	array = (void *)array + sz;
3371	}
3372	}
3373
3374	if (type & PERF_SAMPLE_STACK_USER) {
3375	OVERFLOW_CHECK_u64(array);
3376	sz = *array++;
3377
3378	data->user_stack.offset = ((char *)(array - 1)
3379	- (char *) event);
3380
3381	if (!sz) {
3382	data->user_stack.size = 0;
3383	} else {
3384	OVERFLOW_CHECK(array, sz, max_size);
3385	data->user_stack.data = (char *)array;
3386	array = (void *)array + sz;
3387	OVERFLOW_CHECK_u64(array);
3388	data->user_stack.size = *array++;
3389	if (WARN_ONCE(data->user_stack.size > sz,
3390	"user stack dump failure\n"))
3391	return -EFAULT;
3392	}
3393	}
3394
3395	if (type & PERF_SAMPLE_WEIGHT_TYPE) {
3396	OVERFLOW_CHECK_u64(array);
3397	perf_parse_sample_weight(data, array, type);
3398	array++;
3399	}
3400
3401	if (type & PERF_SAMPLE_DATA_SRC) {
3402	OVERFLOW_CHECK_u64(array);
3403	data->data_src = *array;
3404	array++;
3405	}
3406
3407	if (type & PERF_SAMPLE_TRANSACTION) {
3408	OVERFLOW_CHECK_u64(array);
3409	data->transaction = *array;
3410	array++;
3411	}
3412
3413	if (type & PERF_SAMPLE_REGS_INTR) {
3414	struct regs_dump *regs = perf_sample__intr_regs(data);
3415
3416	OVERFLOW_CHECK_u64(array);
3417	regs->abi = *array;
3418	array++;
3419
3420	if (regs->abi != PERF_SAMPLE_REGS_ABI_NONE) {
3421	u64 mask = evsel->core.attr.sample_regs_intr;
3422
3423	sz = hweight64(mask) * sizeof(u64);
3424	OVERFLOW_CHECK(array, sz, max_size);
3425	regs->mask = mask;
3426	regs->regs = (u64 *)array;
3427	array = (void *)array + sz;
3428	}
3429	}
3430
3431	data->phys_addr = 0;
3432	if (type & PERF_SAMPLE_PHYS_ADDR) {
3433	data->phys_addr = *array;
3434	array++;
3435	}
3436
3437	data->cgroup = 0;
3438	if (type & PERF_SAMPLE_CGROUP) {
3439	data->cgroup = *array;
3440	array++;
3441	}
3442
3443	data->data_page_size = 0;
3444	if (type & PERF_SAMPLE_DATA_PAGE_SIZE) {
3445	data->data_page_size = *array;
3446	array++;
3447	}
3448
3449	data->code_page_size = 0;
3450	if (type & PERF_SAMPLE_CODE_PAGE_SIZE) {
3451	data->code_page_size = *array;
3452	array++;
3453	}
3454
3455	if (type & PERF_SAMPLE_AUX) {
3456	OVERFLOW_CHECK_u64(array);
3457	sz = *array++;
3458
3459	OVERFLOW_CHECK(array, sz, max_size);
3460	/* Undo swap of data */
3461	if (swapped)
3462	mem_bswap_64(src: (char *)array, byte_size: sz);
3463	data->aux_sample.size = sz;
3464	data->aux_sample.data = (char *)array;
3465	array = (void *)array + sz;
3466	}
3467
3468	if (evsel__is_offcpu_event(evsel))
3469	return __set_offcpu_sample(data);
3470
3471	return 0;
3472	}
3473
3474	int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event,
3475	u64 *timestamp)
3476	{
3477	u64 type = evsel->core.attr.sample_type;
3478	const __u64 *array;
3479
3480	if (!(type & PERF_SAMPLE_TIME))
3481	return -1;
3482
3483	if (event->header.type != PERF_RECORD_SAMPLE) {
3484	struct perf_sample data = {
3485	.time = -1ULL,
3486	};
3487
3488	if (!evsel->core.attr.sample_id_all)
3489	return -1;
3490	if (perf_evsel__parse_id_sample(evsel, event, sample: &data))
3491	return -1;
3492
3493	*timestamp = data.time;
3494	return 0;
3495	}
3496
3497	array = event->sample.array;
3498
3499	if (perf_event__check_size(event, sample_size: evsel->sample_size))
3500	return -EFAULT;
3501
3502	if (type & PERF_SAMPLE_IDENTIFIER)
3503	array++;
3504
3505	if (type & PERF_SAMPLE_IP)
3506	array++;
3507
3508	if (type & PERF_SAMPLE_TID)
3509	array++;
3510
3511	if (type & PERF_SAMPLE_TIME)
3512	*timestamp = *array;
3513
3514	return 0;
3515	}
3516
3517	u16 evsel__id_hdr_size(const struct evsel *evsel)
3518	{
3519	u64 sample_type = evsel->core.attr.sample_type;
3520	u16 size = 0;
3521
3522	if (sample_type & PERF_SAMPLE_TID)
3523	size += sizeof(u64);
3524
3525	if (sample_type & PERF_SAMPLE_TIME)
3526	size += sizeof(u64);
3527
3528	if (sample_type & PERF_SAMPLE_ID)
3529	size += sizeof(u64);
3530
3531	if (sample_type & PERF_SAMPLE_STREAM_ID)
3532	size += sizeof(u64);
3533
3534	if (sample_type & PERF_SAMPLE_CPU)
3535	size += sizeof(u64);
3536
3537	if (sample_type & PERF_SAMPLE_IDENTIFIER)
3538	size += sizeof(u64);
3539
3540	return size;
3541	}
3542
3543	#ifdef HAVE_LIBTRACEEVENT
3544	struct tep_format_field *evsel__field(struct evsel *evsel, const char *name)
3545	{
3546	struct tep_event *tp_format = evsel__tp_format(evsel);
3547
3548	return tp_format ? tep_find_field(tp_format, name) : NULL;
3549	}
3550
3551	struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name)
3552	{
3553	struct tep_event *tp_format = evsel__tp_format(evsel);
3554
3555	return tp_format ? tep_find_common_field(tp_format, name) : NULL;
3556	}
3557
3558	void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name)
3559	{
3560	struct tep_format_field *field = evsel__field(evsel, name);
3561	int offset;
3562
3563	if (!field)
3564	return NULL;
3565
3566	offset = field->offset;
3567
3568	if (field->flags & TEP_FIELD_IS_DYNAMIC) {
3569	offset = *(int *)(sample->raw_data + field->offset);
3570	offset &= 0xffff;
3571	if (tep_field_is_relative(field->flags))
3572	offset += field->offset + field->size;
3573	}
3574
3575	return sample->raw_data + offset;
3576	}
3577
3578	u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
3579	bool needs_swap)
3580	{
3581	u64 value;
3582	void *ptr = sample->raw_data + field->offset;
3583
3584	switch (field->size) {
3585	case 1:
3586	return *(u8 *)ptr;
3587	case 2:
3588	value = *(u16 *)ptr;
3589	break;
3590	case 4:
3591	value = *(u32 *)ptr;
3592	break;
3593	case 8:
3594	memcpy(&value, ptr, sizeof(u64));
3595	break;
3596	default:
3597	return 0;
3598	}
3599
3600	if (!needs_swap)
3601	return value;
3602
3603	switch (field->size) {
3604	case 2:
3605	return bswap_16(value);
3606	case 4:
3607	return bswap_32(value);
3608	case 8:
3609	return bswap_64(value);
3610	default:
3611	return 0;
3612	}
3613
3614	return 0;
3615	}
3616
3617	u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name)
3618	{
3619	struct tep_format_field *field = evsel__field(evsel, name);
3620
3621	return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
3622	}
3623
3624	u64 evsel__intval_common(struct evsel *evsel, struct perf_sample *sample, const char *name)
3625	{
3626	struct tep_format_field *field = evsel__common_field(evsel, name);
3627
3628	return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
3629	}
3630
3631	char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const char *name)
3632	{
3633	static struct tep_format_field *prev_state_field;
3634	static const char *states;
3635	struct tep_format_field *field;
3636	unsigned long long val;
3637	unsigned int bit;
3638	char state = '?'; /* '?' denotes unknown task state */
3639
3640	field = evsel__field(evsel, name);
3641
3642	if (!field)
3643	return state;
3644
3645	if (!states || field != prev_state_field) {
3646	states = parse_task_states(field);
3647	if (!states)
3648	return state;
3649	prev_state_field = field;
3650	}
3651
3652	/*
3653	* Note since the kernel exposes TASK_REPORT_MAX to userspace
3654	* to denote the 'preempted' state, we might as welll report
3655	* 'R' for this case, which make senses to users as well.
3656	*
3657	* We can change this if we have a good reason in the future.
3658	*/
3659	val = evsel__intval(evsel, sample, name);
3660	bit = val ? ffs(val) : 0;
3661	state = (!bit || bit > strlen(states)) ? 'R' : states[bit-1];
3662	return state;
3663	}
3664	#endif
3665
3666	bool evsel__fallback(struct evsel *evsel, struct target *target, int err,
3667	char *msg, size_t msgsize)
3668	{
3669	int paranoid;
3670
3671	if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
3672	evsel->core.attr.type == PERF_TYPE_HARDWARE &&
3673	evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
3674	/*
3675	* If it's cycles then fall back to hrtimer based cpu-clock sw
3676	* counter, which is always available even if no PMU support.
3677	*
3678	* PPC returns ENXIO until 2.6.37 (behavior changed with commit
3679	* b0a873e).
3680	*/
3681	evsel->core.attr.type = PERF_TYPE_SOFTWARE;
3682	evsel->core.attr.config = target__has_cpu(target)
3683	? PERF_COUNT_SW_CPU_CLOCK
3684	: PERF_COUNT_SW_TASK_CLOCK;
3685	scnprintf(buf: msg, size: msgsize,
3686	fmt: "The cycles event is not supported, trying to fall back to %s",
3687	target__has_cpu(target) ? "cpu-clock" : "task-clock");
3688
3689	zfree(&evsel->name);
3690	return true;
3691	} else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
3692	(paranoid = perf_event_paranoid()) > 1) {
3693	const char *name = evsel__name(evsel);
3694	char *new_name;
3695	const char *sep = ":";
3696
3697	/* If event has exclude user then don't exclude kernel. */
3698	if (evsel->core.attr.exclude_user)
3699	goto no_fallback;
3700
3701	/* Is there already the separator in the name. */
3702	if (strchr(name, '/') ||
3703	(strchr(name, ':') && !evsel->is_libpfm_event))
3704	sep = "";
3705
3706	if (asprintf(&new_name, "%s%su", name, sep) < 0)
3707	goto no_fallback;
3708
3709	free(evsel->name);
3710	evsel->name = new_name;
3711	scnprintf(buf: msg, size: msgsize, fmt: "kernel.perf_event_paranoid=%d, trying "
3712	"to fall back to excluding kernel and hypervisor "
3713	" samples", paranoid);
3714	evsel->core.attr.exclude_kernel = 1;
3715	evsel->core.attr.exclude_hv = 1;
3716
3717	return true;
3718	} else if (err == EOPNOTSUPP && !evsel->core.attr.exclude_guest &&
3719	!evsel->exclude_GH) {
3720	const char *name = evsel__name(evsel);
3721	char *new_name;
3722	const char *sep = ":";
3723
3724	/* Is there already the separator in the name. */
3725	if (strchr(name, '/') ||
3726	(strchr(name, ':') && !evsel->is_libpfm_event))
3727	sep = "";
3728
3729	if (asprintf(&new_name, "%s%sH", name, sep) < 0)
3730	goto no_fallback;
3731
3732	free(evsel->name);
3733	evsel->name = new_name;
3734	/* Apple M1 requires exclude_guest */
3735	scnprintf(buf: msg, size: msgsize, fmt: "Trying to fall back to excluding guest samples");
3736	evsel->core.attr.exclude_guest = 1;
3737
3738	return true;
3739	}
3740	no_fallback:
3741	scnprintf(buf: msg, size: msgsize, fmt: "No fallback found for '%s' for error %d",
3742	evsel__name(evsel), err);
3743	return false;
3744	}
3745
3746	static bool find_process(const char *name)
3747	{
3748	size_t len = strlen(name);
3749	DIR *dir;
3750	struct dirent *d;
3751	int ret = -1;
3752
3753	dir = opendir(procfs__mountpoint());
3754	if (!dir)
3755	return false;
3756
3757	/* Walk through the directory. */
3758	while (ret && (d = readdir(dir)) != NULL) {
3759	char path[PATH_MAX];
3760	char *data;
3761	size_t size;
3762
3763	if ((d->d_type != DT_DIR) ||
3764	!strcmp(".", d->d_name) ||
3765	!strcmp("..", d->d_name))
3766	continue;
3767
3768	scnprintf(buf: path, size: sizeof(path), fmt: "%s/%s/comm",
3769	procfs__mountpoint(), d->d_name);
3770
3771	if (filename__read_str(path, &data, &size))
3772	continue;
3773
3774	ret = strncmp(name, data, len);
3775	free(data);
3776	}
3777
3778	closedir(dir);
3779	return ret ? false : true;
3780	}
3781
3782	static int dump_perf_event_processes(char *msg, size_t size)
3783	{
3784	DIR *proc_dir;
3785	struct dirent *proc_entry;
3786	int printed = 0;
3787
3788	proc_dir = opendir(procfs__mountpoint());
3789	if (!proc_dir)
3790	return 0;
3791
3792	/* Walk through the /proc directory. */
3793	while ((proc_entry = readdir(proc_dir)) != NULL) {
3794	char buf[256];
3795	DIR *fd_dir;
3796	struct dirent *fd_entry;
3797	int fd_dir_fd;
3798
3799	if (proc_entry->d_type != DT_DIR ||
3800	!isdigit(c: proc_entry->d_name[0]) ||
3801	strlen(proc_entry->d_name) > sizeof(buf) - 4)
3802	continue;
3803
3804	scnprintf(buf, size: sizeof(buf), fmt: "%s/fd", proc_entry->d_name);
3805	fd_dir_fd = openat(dirfd(proc_dir), buf, O_DIRECTORY);
3806	if (fd_dir_fd == -1)
3807	continue;
3808	fd_dir = fdopendir(fd_dir_fd);
3809	if (!fd_dir) {
3810	close(fd_dir_fd);
3811	continue;
3812	}
3813	while ((fd_entry = readdir(fd_dir)) != NULL) {
3814	ssize_t link_size;
3815
3816	if (fd_entry->d_type != DT_LNK)
3817	continue;
3818	link_size = readlinkat(fd_dir_fd, fd_entry->d_name, buf, sizeof(buf));
3819	if (link_size < 0)
3820	continue;
3821	/* Take care as readlink doesn't null terminate the string. */
3822	if (!strncmp(buf, "anon_inode:[perf_event]", link_size)) {
3823	int cmdline_fd;
3824	ssize_t cmdline_size;
3825
3826	scnprintf(buf, size: sizeof(buf), fmt: "%s/cmdline", proc_entry->d_name);
3827	cmdline_fd = openat(dirfd(proc_dir), buf, O_RDONLY);
3828	if (cmdline_fd == -1)
3829	continue;
3830	cmdline_size = read(cmdline_fd, buf, sizeof(buf) - 1);
3831	close(cmdline_fd);
3832	if (cmdline_size < 0)
3833	continue;
3834	buf[cmdline_size] = '\0';
3835	for (ssize_t i = 0; i < cmdline_size; i++) {
3836	if (buf[i] == '\0')
3837	buf[i] = ' ';
3838	}
3839
3840	if (printed == 0)
3841	printed += scnprintf(buf: msg, size, fmt: "Possible processes:\n");
3842
3843	printed += scnprintf(buf: msg + printed, size: size - printed,
3844	fmt: "%s %s\n", proc_entry->d_name, buf);
3845	break;
3846	}
3847	}
3848	closedir(fd_dir);
3849	}
3850	closedir(proc_dir);
3851	return printed;
3852	}
3853
3854	int __weak arch_evsel__open_strerror(struct evsel *evsel __maybe_unused,
3855	int err __maybe_unused,
3856	char *msg __maybe_unused,
3857	size_t size __maybe_unused)
3858	{
3859	return 0;
3860	}
3861
3862	int evsel__open_strerror(struct evsel *evsel, struct target *target,
3863	int err, char *msg, size_t size)
3864	{
3865	struct perf_pmu *pmu;
3866	char sbuf[STRERR_BUFSIZE];
3867	int printed = 0, enforced = 0;
3868	int ret;
3869
3870	switch (err) {
3871	case EPERM:
3872	case EACCES:
3873	printed += scnprintf(buf: msg + printed, size: size - printed,
3874	fmt: "Access to performance monitoring and observability operations is limited.\n");
3875
3876	if (!sysfs__read_int("fs/selinux/enforce", &enforced)) {
3877	if (enforced) {
3878	printed += scnprintf(buf: msg + printed, size: size - printed,
3879	fmt: "Enforced MAC policy settings (SELinux) can limit access to performance\n"
3880	"monitoring and observability operations. Inspect system audit records for\n"
3881	"more perf_event access control information and adjusting the policy.\n");
3882	}
3883	}
3884
3885	if (err == EPERM)
3886	printed += scnprintf(buf: msg, size,
3887	fmt: "No permission to enable %s event.\n\n", evsel__name(evsel));
3888
3889	return printed + scnprintf(buf: msg + printed, size: size - printed,
3890	fmt: "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n"
3891	"access to performance monitoring and observability operations for processes\n"
3892	"without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n"
3893	"More information can be found at 'Perf events and tool security' document:\n"
3894	"https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n"
3895	"perf_event_paranoid setting is %d:\n"
3896	" -1: Allow use of (almost) all events by all users\n"
3897	" Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
3898	">= 0: Disallow raw and ftrace function tracepoint access\n"
3899	">= 1: Disallow CPU event access\n"
3900	">= 2: Disallow kernel profiling\n"
3901	"To make the adjusted perf_event_paranoid setting permanent preserve it\n"
3902	"in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)",
3903	perf_event_paranoid());
3904	case ENOENT:
3905	return scnprintf(buf: msg, size, fmt: "The %s event is not supported.", evsel__name(evsel));
3906	case EMFILE:
3907	return scnprintf(buf: msg, size, fmt: "%s",
3908	"Too many events are opened.\n"
3909	"Probably the maximum number of open file descriptors has been reached.\n"
3910	"Hint: Try again after reducing the number of events.\n"
3911	"Hint: Try increasing the limit with 'ulimit -n <limit>'");
3912	case ENOMEM:
3913	if (evsel__has_callchain(evsel) &&
3914	access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
3915	return scnprintf(buf: msg, size,
3916	fmt: "Not enough memory to setup event with callchain.\n"
3917	"Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
3918	"Hint: Current value: %d", sysctl__max_stack());
3919	break;
3920	case ENODEV:
3921	if (target->cpu_list)
3922	return scnprintf(buf: msg, size, fmt: "%s",
3923	"No such device - did you specify an out-of-range profile CPU?");
3924	break;
3925	case EOPNOTSUPP:
3926	if (evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK)
3927	return scnprintf(buf: msg, size,
3928	fmt: "%s: PMU Hardware or event type doesn't support branch stack sampling.",
3929	evsel__name(evsel));
3930	if (evsel->core.attr.aux_output)
3931	return scnprintf(buf: msg, size,
3932	fmt: "%s: PMU Hardware doesn't support 'aux_output' feature",
3933	evsel__name(evsel));
3934	if (evsel->core.attr.aux_action)
3935	return scnprintf(buf: msg, size,
3936	fmt: "%s: PMU Hardware doesn't support 'aux_action' feature",
3937	evsel__name(evsel));
3938	if (evsel->core.attr.sample_period != 0)
3939	return scnprintf(buf: msg, size,
3940	fmt: "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
3941	evsel__name(evsel));
3942	if (evsel->core.attr.precise_ip)
3943	return scnprintf(buf: msg, size, fmt: "%s",
3944	"\'precise\' request may not be supported. Try removing 'p' modifier.");
3945	#if defined(__i386__) || defined(__x86_64__)
3946	if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
3947	return scnprintf(buf: msg, size, fmt: "%s",
3948	"No hardware sampling interrupt available.\n");
3949	#endif
3950	if (!target__has_cpu(target))
3951	return scnprintf(buf: msg, size,
3952	fmt: "Unsupported event (%s) in per-thread mode, enable system wide with '-a'.",
3953	evsel__name(evsel));
3954	break;
3955	case EBUSY:
3956	if (find_process(name: "oprofiled"))
3957	return scnprintf(buf: msg, size,
3958	fmt: "The PMU counters are busy/taken by another profiler.\n"
3959	"We found oprofile daemon running, please stop it and try again.");
3960	printed += scnprintf(
3961	buf: msg, size,
3962	fmt: "The PMU %s counters are busy and in use by another process.\n",
3963	evsel->pmu ? evsel->pmu->name : "");
3964	return printed + dump_perf_event_processes(msg: msg + printed, size: size - printed);
3965	break;
3966	case EINVAL:
3967	if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size)
3968	return scnprintf(buf: msg, size, fmt: "Asking for the code page size isn't supported by this kernel.");
3969	if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size)
3970	return scnprintf(buf: msg, size, fmt: "Asking for the data page size isn't supported by this kernel.");
3971	if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
3972	return scnprintf(buf: msg, size, fmt: "Reading from overwrite event is not supported by this kernel.");
3973	if (perf_missing_features.clockid)
3974	return scnprintf(buf: msg, size, fmt: "clockid feature not supported.");
3975	if (perf_missing_features.clockid_wrong)
3976	return scnprintf(buf: msg, size, fmt: "wrong clockid (%d).", clockid);
3977	if (perf_missing_features.aux_action)
3978	return scnprintf(buf: msg, size, fmt: "The 'aux_action' feature is not supported, update the kernel.");
3979	if (perf_missing_features.aux_output)
3980	return scnprintf(buf: msg, size, fmt: "The 'aux_output' feature is not supported, update the kernel.");
3981	pmu = evsel__find_pmu(evsel);
3982	if (!pmu->is_core && !target__has_cpu(target))
3983	return scnprintf(buf: msg, size,
3984	fmt: "Invalid event (%s) in per-thread mode, enable system wide with '-a'.",
3985	evsel__name(evsel));
3986
3987	break;
3988	case ENODATA:
3989	return scnprintf(buf: msg, size, fmt: "Cannot collect data source with the load latency event alone. "
3990	"Please add an auxiliary event in front of the load latency event.");
3991	default:
3992	break;
3993	}
3994
3995	ret = arch_evsel__open_strerror(evsel, err, msg, size);
3996	if (ret)
3997	return ret;
3998
3999	return scnprintf(buf: msg, size,
4000	fmt: "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
4001	"\"dmesg | grep -i perf\" may provide additional information.\n",
4002	err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel));
4003	}
4004
4005	struct perf_session *evsel__session(struct evsel *evsel)
4006	{
4007	return evsel && evsel->evlist ? evsel->evlist->session : NULL;
4008	}
4009
4010	struct perf_env *evsel__env(struct evsel *evsel)
4011	{
4012	struct perf_session *session = evsel__session(evsel);
4013
4014	return session ? perf_session__env(session) : NULL;
4015	}
4016
4017	static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
4018	{
4019	int cpu_map_idx, thread;
4020
4021	if (evsel__is_retire_lat(evsel))
4022	return 0;
4023
4024	if (perf_pmu__kind(pmu: evsel->pmu) != PERF_PMU_KIND_PE)
4025	return 0;
4026
4027	for (cpu_map_idx = 0; cpu_map_idx < xyarray__max_x(evsel->core.fd); cpu_map_idx++) {
4028	for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
4029	thread++) {
4030	int fd = FD(evsel, cpu_map_idx, thread);
4031
4032	if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
4033	cpu_map_idx, thread, fd) < 0)
4034	return -1;
4035	}
4036	}
4037
4038	return 0;
4039	}
4040
4041	int evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
4042	{
4043	struct perf_cpu_map *cpus = evsel->core.cpus;
4044	struct perf_thread_map *threads = evsel->core.threads;
4045
4046	if (perf_evsel__alloc_id(&evsel->core, perf_cpu_map__nr(cpus), threads->nr))
4047	return -ENOMEM;
4048
4049	return store_evsel_ids(evsel, evlist);
4050	}
4051
4052	void evsel__zero_per_pkg(struct evsel *evsel)
4053	{
4054	struct hashmap_entry *cur;
4055	size_t bkt;
4056
4057	if (evsel->per_pkg_mask) {
4058	hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt)
4059	zfree(&cur->pkey);
4060
4061	hashmap__clear(map: evsel->per_pkg_mask);
4062	}
4063	}
4064
4065	/**
4066	* evsel__is_hybrid - does the evsel have a known PMU that is hybrid. Note, this
4067	* will be false on hybrid systems for hardware and legacy
4068	* cache events.
4069	*/
4070	bool evsel__is_hybrid(const struct evsel *evsel)
4071	{
4072	if (!evsel->core.is_pmu_core)
4073	return false;
4074
4075	return perf_pmus__num_core_pmus() > 1;
4076	}
4077
4078	struct evsel *evsel__leader(const struct evsel *evsel)
4079	{
4080	if (evsel->core.leader == NULL)
4081	return NULL;
4082	return container_of(evsel->core.leader, struct evsel, core);
4083	}
4084
4085	bool evsel__has_leader(struct evsel *evsel, struct evsel *leader)
4086	{
4087	return evsel->core.leader == &leader->core;
4088	}
4089
4090	bool evsel__is_leader(struct evsel *evsel)
4091	{
4092	return evsel__has_leader(evsel, leader: evsel);
4093	}
4094
4095	void evsel__set_leader(struct evsel *evsel, struct evsel *leader)
4096	{
4097	evsel->core.leader = &leader->core;
4098	}
4099
4100	int evsel__source_count(const struct evsel *evsel)
4101	{
4102	struct evsel *pos;
4103	int count = 0;
4104
4105	evlist__for_each_entry(evsel->evlist, pos) {
4106	if (pos->metric_leader == evsel)
4107	count++;
4108	}
4109	return count;
4110	}
4111
4112	bool __weak arch_evsel__must_be_in_group(const struct evsel *evsel __maybe_unused)
4113	{
4114	return false;
4115	}
4116
4117	/*
4118	* Remove an event from a given group (leader).
4119	* Some events, e.g., perf metrics Topdown events,
4120	* must always be grouped. Ignore the events.
4121	*/
4122	void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader)
4123	{
4124	if (!arch_evsel__must_be_in_group(evsel) && evsel != leader) {
4125	evsel__set_leader(evsel, leader: evsel);
4126	evsel->core.nr_members = 0;
4127	leader->core.nr_members--;
4128	}
4129	}
4130
4131	bool evsel__set_needs_uniquify(struct evsel *counter, const struct perf_stat_config *config)
4132	{
4133	struct evsel *evsel;
4134
4135	if (counter->needs_uniquify) {
4136	/* Already set. */
4137	return true;
4138	}
4139
4140	if (counter->use_config_name || counter->is_libpfm_event) {
4141	/* Original name will be used. */
4142	return false;
4143	}
4144
4145	if (!config->hybrid_merge && evsel__is_hybrid(evsel: counter)) {
4146	/* Unique hybrid counters necessary. */
4147	counter->needs_uniquify = true;
4148	return true;
4149	}
4150
4151	if (counter->core.attr.type < PERF_TYPE_MAX && counter->core.attr.type != PERF_TYPE_RAW) {
4152	/* Legacy event, don't uniquify. */
4153	return false;
4154	}
4155
4156	if (counter->pmu && counter->pmu->is_core &&
4157	counter->alternate_hw_config != PERF_COUNT_HW_MAX) {
4158	/* A sysfs or json event replacing a legacy event, don't uniquify. */
4159	return false;
4160	}
4161
4162	if (config->aggr_mode == AGGR_NONE) {
4163	/* Always unique with no aggregation. */
4164	counter->needs_uniquify = true;
4165	return true;
4166	}
4167
4168	if (counter->first_wildcard_match != NULL) {
4169	/*
4170	* If stats are merged then only the first_wildcard_match is
4171	* displayed, there is no need to uniquify this evsel as the
4172	* name won't be shown.
4173	*/
4174	return false;
4175	}
4176
4177	/*
4178	* Do other non-merged events in the evlist have the same name? If so
4179	* uniquify is necessary.
4180	*/
4181	evlist__for_each_entry(counter->evlist, evsel) {
4182	if (evsel == counter || evsel->first_wildcard_match || evsel->pmu == counter->pmu)
4183	continue;
4184
4185	if (evsel__name_is(evsel: counter, name: evsel__name(evsel))) {
4186	counter->needs_uniquify = true;
4187	return true;
4188	}
4189	}
4190	return false;
4191	}
4192
4193	void evsel__uniquify_counter(struct evsel *counter)
4194	{
4195	const char *name, *pmu_name, *config;
4196	char *new_name;
4197	int len, ret;
4198
4199	/* No uniquification necessary. */
4200	if (!counter->needs_uniquify)
4201	return;
4202
4203	/* The evsel was already uniquified. */
4204	if (counter->uniquified_name)
4205	return;
4206
4207	/* Avoid checking to uniquify twice. */
4208	counter->uniquified_name = true;
4209
4210	name = evsel__name(evsel: counter);
4211	config = strchr(name, '/');
4212	pmu_name = counter->pmu->name;
4213
4214	/* Already prefixed by the PMU name? */
4215	len = pmu_name_len_no_suffix(str: pmu_name);
4216
4217	if (!strncmp(name, pmu_name, len)) {
4218	/*
4219	* If the PMU name is there, then there is no sense in not
4220	* having a slash. Do this for robustness.
4221	*/
4222	if (config == NULL)
4223	config = name - 1;
4224
4225	ret = asprintf(&new_name, "%s/%s", pmu_name, config + 1);
4226	} else if (config) {
4227	len = config - name;
4228	if (config[1] == '/') {
4229	/* case: event// */
4230	ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 2);
4231	} else {
4232	/* case: event/.../ */
4233	ret = asprintf(&new_name, "%s/%.*s,%s", pmu_name, len, name, config + 1);
4234	}
4235	} else {
4236	config = strchr(name, ':');
4237	if (config) {
4238	/* case: event:.. */
4239	len = config - name;
4240
4241	ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 1);
4242	} else {
4243	/* case: event */
4244	ret = asprintf(&new_name, "%s/%s/", pmu_name, name);
4245	}
4246	}
4247	if (ret > 0) {
4248	free(counter->name);
4249	counter->name = new_name;
4250	} else {
4251	/* ENOMEM from asprintf. */
4252	counter->uniquified_name = false;
4253	}
4254	}
4255
4256	void evsel__warn_user_requested_cpus(struct evsel *evsel, struct perf_cpu_map *user_requested_cpus)
4257	{
4258	struct perf_cpu_map *intersect, *online = NULL;
4259	const struct perf_pmu *pmu = evsel__find_pmu(evsel);
4260
4261	if (pmu && pmu->is_core) {
4262	intersect = perf_cpu_map__intersect(pmu->cpus, user_requested_cpus);
4263	} else {
4264	online = cpu_map__online();
4265	intersect = perf_cpu_map__intersect(online, user_requested_cpus);
4266	}
4267	if (!perf_cpu_map__equal(intersect, user_requested_cpus)) {
4268	char buf1[128];
4269	char buf2[128];
4270
4271	cpu_map__snprint(map: user_requested_cpus, buf: buf1, size: sizeof(buf1));
4272	cpu_map__snprint(map: online ?: pmu->cpus, buf: buf2, size: sizeof(buf2));
4273	pr_warning("WARNING: A requested CPU in '%s' is not supported by PMU '%s' (CPUs %s) for event '%s'\n",
4274	buf1, pmu ? pmu->name : "cpu", buf2, evsel__name(evsel));
4275	}
4276	perf_cpu_map__put(intersect);
4277	perf_cpu_map__put(online);
4278	}
4279