1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* builtin-record.c
4	*
5	* Builtin record command: Record the profile of a workload
6	* (or a CPU, or a PID) into the perf.data output file - for
7	* later analysis via perf report.
8	*/
9	#include "builtin.h"
10
11	#include "util/build-id.h"
12	#include <subcmd/parse-options.h>
13	#include <internal/xyarray.h>
14	#include "util/parse-events.h"
15	#include "util/config.h"
16
17	#include "util/callchain.h"
18	#include "util/cgroup.h"
19	#include "util/header.h"
20	#include "util/event.h"
21	#include "util/evlist.h"
22	#include "util/evsel.h"
23	#include "util/debug.h"
24	#include "util/mmap.h"
25	#include "util/mutex.h"
26	#include "util/target.h"
27	#include "util/session.h"
28	#include "util/tool.h"
29	#include "util/symbol.h"
30	#include "util/record.h"
31	#include "util/cpumap.h"
32	#include "util/thread_map.h"
33	#include "util/data.h"
34	#include "util/perf_regs.h"
35	#include "util/auxtrace.h"
36	#include "util/tsc.h"
37	#include "util/parse-branch-options.h"
38	#include "util/parse-regs-options.h"
39	#include "util/perf_api_probe.h"
40	#include "util/trigger.h"
41	#include "util/perf-hooks.h"
42	#include "util/cpu-set-sched.h"
43	#include "util/synthetic-events.h"
44	#include "util/time-utils.h"
45	#include "util/units.h"
46	#include "util/bpf-event.h"
47	#include "util/util.h"
48	#include "util/pfm.h"
49	#include "util/pmu.h"
50	#include "util/pmus.h"
51	#include "util/clockid.h"
52	#include "util/off_cpu.h"
53	#include "util/bpf-filter.h"
54	#include "asm/bug.h"
55	#include "perf.h"
56	#include "cputopo.h"
57
58	#include <errno.h>
59	#include <inttypes.h>
60	#include <locale.h>
61	#include <poll.h>
62	#include <pthread.h>
63	#include <unistd.h>
64	#ifndef HAVE_GETTID
65	#include <syscall.h>
66	#endif
67	#include <sched.h>
68	#include <signal.h>
69	#ifdef HAVE_EVENTFD_SUPPORT
70	#include <sys/eventfd.h>
71	#endif
72	#include <sys/mman.h>
73	#include <sys/wait.h>
74	#include <sys/types.h>
75	#include <sys/stat.h>
76	#include <fcntl.h>
77	#include <linux/err.h>
78	#include <linux/string.h>
79	#include <linux/time64.h>
80	#include <linux/zalloc.h>
81	#include <linux/bitmap.h>
82	#include <sys/time.h>
83
84	struct switch_output {
85	bool enabled;
86	bool signal;
87	unsigned long size;
88	unsigned long time;
89	const char *str;
90	bool set;
91	char **filenames;
92	int num_files;
93	int cur_file;
94	};
95
96	struct thread_mask {
97	struct mmap_cpu_mask maps;
98	struct mmap_cpu_mask affinity;
99	};
100
101	struct record_thread {
102	pid_t tid;
103	struct thread_mask *mask;
104	struct {
105	int msg[2];
106	int ack[2];
107	} pipes;
108	struct fdarray pollfd;
109	int ctlfd_pos;
110	int nr_mmaps;
111	struct mmap **maps;
112	struct mmap **overwrite_maps;
113	struct record *rec;
114	unsigned long long samples;
115	unsigned long waking;
116	u64 bytes_written;
117	u64 bytes_transferred;
118	u64 bytes_compressed;
119	};
120
121	static __thread struct record_thread *thread;
122
123	enum thread_msg {
124	THREAD_MSG__UNDEFINED = 0,
125	THREAD_MSG__READY,
126	THREAD_MSG__MAX,
127	};
128
129	static const char *thread_msg_tags[THREAD_MSG__MAX] = {
130	"UNDEFINED", "READY"
131	};
132
133	enum thread_spec {
134	THREAD_SPEC__UNDEFINED = 0,
135	THREAD_SPEC__CPU,
136	THREAD_SPEC__CORE,
137	THREAD_SPEC__PACKAGE,
138	THREAD_SPEC__NUMA,
139	THREAD_SPEC__USER,
140	THREAD_SPEC__MAX,
141	};
142
143	static const char *thread_spec_tags[THREAD_SPEC__MAX] = {
144	"undefined", "cpu", "core", "package", "numa", "user"
145	};
146
147	struct pollfd_index_map {
148	int evlist_pollfd_index;
149	int thread_pollfd_index;
150	};
151
152	struct record {
153	struct perf_tool tool;
154	struct record_opts opts;
155	u64 bytes_written;
156	u64 thread_bytes_written;
157	struct perf_data data;
158	struct auxtrace_record *itr;
159	struct evlist *evlist;
160	struct perf_session *session;
161	struct evlist *sb_evlist;
162	pthread_t thread_id;
163	int realtime_prio;
164	bool switch_output_event_set;
165	bool no_buildid;
166	bool no_buildid_set;
167	bool no_buildid_cache;
168	bool no_buildid_cache_set;
169	bool buildid_all;
170	bool buildid_mmap;
171	bool timestamp_filename;
172	bool timestamp_boundary;
173	bool off_cpu;
174	struct switch_output switch_output;
175	unsigned long long samples;
176	unsigned long output_max_size; /* = 0: unlimited */
177	struct perf_debuginfod debuginfod;
178	int nr_threads;
179	struct thread_mask *thread_masks;
180	struct record_thread *thread_data;
181	struct pollfd_index_map *index_map;
182	size_t index_map_sz;
183	size_t index_map_cnt;
184	};
185
186	static volatile int done;
187
188	static volatile int auxtrace_record__snapshot_started;
189	static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
190	static DEFINE_TRIGGER(switch_output_trigger);
191
192	static const char *affinity_tags[PERF_AFFINITY_MAX] = {
193	"SYS", "NODE", "CPU"
194	};
195
196	#ifndef HAVE_GETTID
197	static inline pid_t gettid(void)
198	{
199	return (pid_t)syscall(__NR_gettid);
200	}
201	#endif
202
203	static int record__threads_enabled(struct record *rec)
204	{
205	return rec->opts.threads_spec;
206	}
207
208	static bool switch_output_signal(struct record *rec)
209	{
210	return rec->switch_output.signal &&
211	trigger_is_ready(t: &switch_output_trigger);
212	}
213
214	static bool switch_output_size(struct record *rec)
215	{
216	return rec->switch_output.size &&
217	trigger_is_ready(t: &switch_output_trigger) &&
218	(rec->bytes_written >= rec->switch_output.size);
219	}
220
221	static bool switch_output_time(struct record *rec)
222	{
223	return rec->switch_output.time &&
224	trigger_is_ready(t: &switch_output_trigger);
225	}
226
227	static u64 record__bytes_written(struct record *rec)
228	{
229	return rec->bytes_written + rec->thread_bytes_written;
230	}
231
232	static bool record__output_max_size_exceeded(struct record *rec)
233	{
234	return rec->output_max_size &&
235	(record__bytes_written(rec) >= rec->output_max_size);
236	}
237
238	static int record__write(struct record *rec, struct mmap *map __maybe_unused,
239	void *bf, size_t size)
240	{
241	struct perf_data_file *file = &rec->session->data->file;
242
243	if (map && map->file)
244	file = map->file;
245
246	if (perf_data_file__write(file, buf: bf, size) < 0) {
247	pr_err("failed to write perf data, error: %m\n");
248	return -1;
249	}
250
251	if (map && map->file) {
252	thread->bytes_written += size;
253	rec->thread_bytes_written += size;
254	} else {
255	rec->bytes_written += size;
256	}
257
258	if (record__output_max_size_exceeded(rec) && !done) {
259	fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
260	" stopping session ]\n",
261	record__bytes_written(rec) >> 10);
262	done = 1;
263	}
264
265	if (switch_output_size(rec))
266	trigger_hit(t: &switch_output_trigger);
267
268	return 0;
269	}
270
271	static int record__aio_enabled(struct record *rec);
272	static int record__comp_enabled(struct record *rec);
273	static ssize_t zstd_compress(struct perf_session *session, struct mmap *map,
274	void *dst, size_t dst_size, void *src, size_t src_size);
275
276	#ifdef HAVE_AIO_SUPPORT
277	static int record__aio_write(struct aiocb *cblock, int trace_fd,
278	void *buf, size_t size, off_t off)
279	{
280	int rc;
281
282	cblock->aio_fildes = trace_fd;
283	cblock->aio_buf = buf;
284	cblock->aio_nbytes = size;
285	cblock->aio_offset = off;
286	cblock->aio_sigevent.sigev_notify = SIGEV_NONE;
287
288	do {
289	rc = aio_write(cblock);
290	if (rc == 0) {
291	break;
292	} else if (errno != EAGAIN) {
293	cblock->aio_fildes = -1;
294	pr_err("failed to queue perf data, error: %m\n");
295	break;
296	}
297	} while (1);
298
299	return rc;
300	}
301
302	static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
303	{
304	void *rem_buf;
305	off_t rem_off;
306	size_t rem_size;
307	int rc, aio_errno;
308	ssize_t aio_ret, written;
309
310	aio_errno = aio_error(cblock);
311	if (aio_errno == EINPROGRESS)
312	return 0;
313
314	written = aio_ret = aio_return(cblock);
315	if (aio_ret < 0) {
316	if (aio_errno != EINTR)
317	pr_err("failed to write perf data, error: %m\n");
318	written = 0;
319	}
320
321	rem_size = cblock->aio_nbytes - written;
322
323	if (rem_size == 0) {
324	cblock->aio_fildes = -1;
325	/*
326	* md->refcount is incremented in record__aio_pushfn() for
327	* every aio write request started in record__aio_push() so
328	* decrement it because the request is now complete.
329	*/
330	perf_mmap__put(&md->core);
331	rc = 1;
332	} else {
333	/*
334	* aio write request may require restart with the
335	* reminder if the kernel didn't write whole
336	* chunk at once.
337	*/
338	rem_off = cblock->aio_offset + written;
339	rem_buf = (void *)(cblock->aio_buf + written);
340	record__aio_write(cblock, cblock->aio_fildes,
341	rem_buf, rem_size, rem_off);
342	rc = 0;
343	}
344
345	return rc;
346	}
347
348	static int record__aio_sync(struct mmap *md, bool sync_all)
349	{
350	struct aiocb **aiocb = md->aio.aiocb;
351	struct aiocb *cblocks = md->aio.cblocks;
352	struct timespec timeout = { 0, 1000 * 1000 * 1 }; /* 1ms */
353	int i, do_suspend;
354
355	do {
356	do_suspend = 0;
357	for (i = 0; i < md->aio.nr_cblocks; ++i) {
358	if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
359	if (sync_all)
360	aiocb[i] = NULL;
361	else
362	return i;
363	} else {
364	/*
365	* Started aio write is not complete yet
366	* so it has to be waited before the
367	* next allocation.
368	*/
369	aiocb[i] = &cblocks[i];
370	do_suspend = 1;
371	}
372	}
373	if (!do_suspend)
374	return -1;
375
376	while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
377	if (!(errno == EAGAIN || errno == EINTR))
378	pr_err("failed to sync perf data, error: %m\n");
379	}
380	} while (1);
381	}
382
383	struct record_aio {
384	struct record *rec;
385	void *data;
386	size_t size;
387	};
388
389	static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
390	{
391	struct record_aio *aio = to;
392
393	/*
394	* map->core.base data pointed by buf is copied into free map->aio.data[] buffer
395	* to release space in the kernel buffer as fast as possible, calling
396	* perf_mmap__consume() from perf_mmap__push() function.
397	*
398	* That lets the kernel to proceed with storing more profiling data into
399	* the kernel buffer earlier than other per-cpu kernel buffers are handled.
400	*
401	* Coping can be done in two steps in case the chunk of profiling data
402	* crosses the upper bound of the kernel buffer. In this case we first move
403	* part of data from map->start till the upper bound and then the reminder
404	* from the beginning of the kernel buffer till the end of the data chunk.
405	*/
406
407	if (record__comp_enabled(aio->rec)) {
408	ssize_t compressed = zstd_compress(aio->rec->session, NULL, aio->data + aio->size,
409	mmap__mmap_len(map) - aio->size,
410	buf, size);
411	if (compressed < 0)
412	return (int)compressed;
413
414	size = compressed;
415	} else {
416	memcpy(aio->data + aio->size, buf, size);
417	}
418
419	if (!aio->size) {
420	/*
421	* Increment map->refcount to guard map->aio.data[] buffer
422	* from premature deallocation because map object can be
423	* released earlier than aio write request started on
424	* map->aio.data[] buffer is complete.
425	*
426	* perf_mmap__put() is done at record__aio_complete()
427	* after started aio request completion or at record__aio_push()
428	* if the request failed to start.
429	*/
430	perf_mmap__get(&map->core);
431	}
432
433	aio->size += size;
434
435	return size;
436	}
437
438	static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
439	{
440	int ret, idx;
441	int trace_fd = rec->session->data->file.fd;
442	struct record_aio aio = { .rec = rec, .size = 0 };
443
444	/*
445	* Call record__aio_sync() to wait till map->aio.data[] buffer
446	* becomes available after previous aio write operation.
447	*/
448
449	idx = record__aio_sync(map, false);
450	aio.data = map->aio.data[idx];
451	ret = perf_mmap__push(map, &aio, record__aio_pushfn);
452	if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
453	return ret;
454
455	rec->samples++;
456	ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
457	if (!ret) {
458	*off += aio.size;
459	rec->bytes_written += aio.size;
460	if (switch_output_size(rec))
461	trigger_hit(&switch_output_trigger);
462	} else {
463	/*
464	* Decrement map->refcount incremented in record__aio_pushfn()
465	* back if record__aio_write() operation failed to start, otherwise
466	* map->refcount is decremented in record__aio_complete() after
467	* aio write operation finishes successfully.
468	*/
469	perf_mmap__put(&map->core);
470	}
471
472	return ret;
473	}
474
475	static off_t record__aio_get_pos(int trace_fd)
476	{
477	return lseek(trace_fd, 0, SEEK_CUR);
478	}
479
480	static void record__aio_set_pos(int trace_fd, off_t pos)
481	{
482	lseek(trace_fd, pos, SEEK_SET);
483	}
484
485	static void record__aio_mmap_read_sync(struct record *rec)
486	{
487	int i;
488	struct evlist *evlist = rec->evlist;
489	struct mmap *maps = evlist->mmap;
490
491	if (!record__aio_enabled(rec))
492	return;
493
494	for (i = 0; i < evlist->core.nr_mmaps; i++) {
495	struct mmap *map = &maps[i];
496
497	if (map->core.base)
498	record__aio_sync(map, true);
499	}
500	}
501
502	static int nr_cblocks_default = 1;
503	static int nr_cblocks_max = 4;
504
505	static int record__aio_parse(const struct option *opt,
506	const char *str,
507	int unset)
508	{
509	struct record_opts *opts = (struct record_opts *)opt->value;
510
511	if (unset) {
512	opts->nr_cblocks = 0;
513	} else {
514	if (str)
515	opts->nr_cblocks = strtol(str, NULL, 0);
516	if (!opts->nr_cblocks)
517	opts->nr_cblocks = nr_cblocks_default;
518	}
519
520	return 0;
521	}
522	#else /* HAVE_AIO_SUPPORT */
523	static int nr_cblocks_max = 0;
524
525	static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
526	off_t *off __maybe_unused)
527	{
528	return -1;
529	}
530
531	static off_t record__aio_get_pos(int trace_fd __maybe_unused)
532	{
533	return -1;
534	}
535
536	static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
537	{
538	}
539
540	static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
541	{
542	}
543	#endif
544
545	static int record__aio_enabled(struct record *rec)
546	{
547	return rec->opts.nr_cblocks > 0;
548	}
549
550	#define MMAP_FLUSH_DEFAULT 1
551	static int record__mmap_flush_parse(const struct option *opt,
552	const char *str,
553	int unset)
554	{
555	int flush_max;
556	struct record_opts *opts = (struct record_opts *)opt->value;
557	static struct parse_tag tags[] = {
558	{ .tag = 'B', .mult = 1 },
559	{ .tag = 'K', .mult = 1 << 10 },
560	{ .tag = 'M', .mult = 1 << 20 },
561	{ .tag = 'G', .mult = 1 << 30 },
562	{ .tag = 0 },
563	};
564
565	if (unset)
566	return 0;
567
568	if (str) {
569	opts->mmap_flush = parse_tag_value(str, tags);
570	if (opts->mmap_flush == (int)-1)
571	opts->mmap_flush = strtol(str, NULL, 0);
572	}
573
574	if (!opts->mmap_flush)
575	opts->mmap_flush = MMAP_FLUSH_DEFAULT;
576
577	flush_max = evlist__mmap_size(pages: opts->mmap_pages);
578	flush_max /= 4;
579	if (opts->mmap_flush > flush_max)
580	opts->mmap_flush = flush_max;
581
582	return 0;
583	}
584
585	#ifdef HAVE_ZSTD_SUPPORT
586	static unsigned int comp_level_default = 1;
587
588	static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
589	{
590	struct record_opts *opts = opt->value;
591
592	if (unset) {
593	opts->comp_level = 0;
594	} else {
595	if (str)
596	opts->comp_level = strtol(str, NULL, 0);
597	if (!opts->comp_level)
598	opts->comp_level = comp_level_default;
599	}
600
601	return 0;
602	}
603	#endif
604	static unsigned int comp_level_max = 22;
605
606	static int record__comp_enabled(struct record *rec)
607	{
608	return rec->opts.comp_level > 0;
609	}
610
611	static int process_synthesized_event(struct perf_tool *tool,
612	union perf_event *event,
613	struct perf_sample *sample __maybe_unused,
614	struct machine *machine __maybe_unused)
615	{
616	struct record *rec = container_of(tool, struct record, tool);
617	return record__write(rec, NULL, bf: event, size: event->header.size);
618	}
619
620	static struct mutex synth_lock;
621
622	static int process_locked_synthesized_event(struct perf_tool *tool,
623	union perf_event *event,
624	struct perf_sample *sample __maybe_unused,
625	struct machine *machine __maybe_unused)
626	{
627	int ret;
628
629	mutex_lock(&synth_lock);
630	ret = process_synthesized_event(tool, event, sample, machine);
631	mutex_unlock(lock: &synth_lock);
632	return ret;
633	}
634
635	static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
636	{
637	struct record *rec = to;
638
639	if (record__comp_enabled(rec)) {
640	ssize_t compressed = zstd_compress(session: rec->session, map, dst: map->data,
641	dst_size: mmap__mmap_len(map), src: bf, src_size: size);
642
643	if (compressed < 0)
644	return (int)compressed;
645
646	size = compressed;
647	bf = map->data;
648	}
649
650	thread->samples++;
651	return record__write(rec, map, bf, size);
652	}
653
654	static volatile sig_atomic_t signr = -1;
655	static volatile sig_atomic_t child_finished;
656	#ifdef HAVE_EVENTFD_SUPPORT
657	static volatile sig_atomic_t done_fd = -1;
658	#endif
659
660	static void sig_handler(int sig)
661	{
662	if (sig == SIGCHLD)
663	child_finished = 1;
664	else
665	signr = sig;
666
667	done = 1;
668	#ifdef HAVE_EVENTFD_SUPPORT
669	if (done_fd >= 0) {
670	u64 tmp = 1;
671	int orig_errno = errno;
672
673	/*
674	* It is possible for this signal handler to run after done is
675	* checked in the main loop, but before the perf counter fds are
676	* polled. If this happens, the poll() will continue to wait
677	* even though done is set, and will only break out if either
678	* another signal is received, or the counters are ready for
679	* read. To ensure the poll() doesn't sleep when done is set,
680	* use an eventfd (done_fd) to wake up the poll().
681	*/
682	if (write(done_fd, &tmp, sizeof(tmp)) < 0)
683	pr_err("failed to signal wakeup fd, error: %m\n");
684
685	errno = orig_errno;
686	}
687	#endif // HAVE_EVENTFD_SUPPORT
688	}
689
690	static void sigsegv_handler(int sig)
691	{
692	perf_hooks__recover();
693	sighandler_dump_stack(sig);
694	}
695
696	static void record__sig_exit(void)
697	{
698	if (signr == -1)
699	return;
700
701	signal(signr, SIG_DFL);
702	raise(signr);
703	}
704
705	#ifdef HAVE_AUXTRACE_SUPPORT
706
707	static int record__process_auxtrace(struct perf_tool *tool,
708	struct mmap *map,
709	union perf_event *event, void *data1,
710	size_t len1, void *data2, size_t len2)
711	{
712	struct record *rec = container_of(tool, struct record, tool);
713	struct perf_data *data = &rec->data;
714	size_t padding;
715	u8 pad[8] = {0};
716
717	if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
718	off_t file_offset;
719	int fd = perf_data__fd(data);
720	int err;
721
722	file_offset = lseek(fd, 0, SEEK_CUR);
723	if (file_offset == -1)
724	return -1;
725	err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
726	event, file_offset);
727	if (err)
728	return err;
729	}
730
731	/* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
732	padding = (len1 + len2) & 7;
733	if (padding)
734	padding = 8 - padding;
735
736	record__write(rec, map, event, event->header.size);
737	record__write(rec, map, data1, len1);
738	if (len2)
739	record__write(rec, map, data2, len2);
740	record__write(rec, map, &pad, padding);
741
742	return 0;
743	}
744
745	static int record__auxtrace_mmap_read(struct record *rec,
746	struct mmap *map)
747	{
748	int ret;
749
750	ret = auxtrace_mmap__read(map, rec->itr, &rec->tool,
751	record__process_auxtrace);
752	if (ret < 0)
753	return ret;
754
755	if (ret)
756	rec->samples++;
757
758	return 0;
759	}
760
761	static int record__auxtrace_mmap_read_snapshot(struct record *rec,
762	struct mmap *map)
763	{
764	int ret;
765
766	ret = auxtrace_mmap__read_snapshot(map, rec->itr, &rec->tool,
767	record__process_auxtrace,
768	rec->opts.auxtrace_snapshot_size);
769	if (ret < 0)
770	return ret;
771
772	if (ret)
773	rec->samples++;
774
775	return 0;
776	}
777
778	static int record__auxtrace_read_snapshot_all(struct record *rec)
779	{
780	int i;
781	int rc = 0;
782
783	for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
784	struct mmap *map = &rec->evlist->mmap[i];
785
786	if (!map->auxtrace_mmap.base)
787	continue;
788
789	if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
790	rc = -1;
791	goto out;
792	}
793	}
794	out:
795	return rc;
796	}
797
798	static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
799	{
800	pr_debug("Recording AUX area tracing snapshot\n");
801	if (record__auxtrace_read_snapshot_all(rec) < 0) {
802	trigger_error(&auxtrace_snapshot_trigger);
803	} else {
804	if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
805	trigger_error(&auxtrace_snapshot_trigger);
806	else
807	trigger_ready(&auxtrace_snapshot_trigger);
808	}
809	}
810
811	static int record__auxtrace_snapshot_exit(struct record *rec)
812	{
813	if (trigger_is_error(&auxtrace_snapshot_trigger))
814	return 0;
815
816	if (!auxtrace_record__snapshot_started &&
817	auxtrace_record__snapshot_start(rec->itr))
818	return -1;
819
820	record__read_auxtrace_snapshot(rec, true);
821	if (trigger_is_error(&auxtrace_snapshot_trigger))
822	return -1;
823
824	return 0;
825	}
826
827	static int record__auxtrace_init(struct record *rec)
828	{
829	int err;
830
831	if ((rec->opts.auxtrace_snapshot_opts || rec->opts.auxtrace_sample_opts)
832	&& record__threads_enabled(rec)) {
833	pr_err("AUX area tracing options are not available in parallel streaming mode.\n");
834	return -EINVAL;
835	}
836
837	if (!rec->itr) {
838	rec->itr = auxtrace_record__init(rec->evlist, &err);
839	if (err)
840	return err;
841	}
842
843	err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
844	rec->opts.auxtrace_snapshot_opts);
845	if (err)
846	return err;
847
848	err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
849	rec->opts.auxtrace_sample_opts);
850	if (err)
851	return err;
852
853	auxtrace_regroup_aux_output(rec->evlist);
854
855	return auxtrace_parse_filters(rec->evlist);
856	}
857
858	#else
859
860	static inline
861	int record__auxtrace_mmap_read(struct record *rec __maybe_unused,
862	struct mmap *map __maybe_unused)
863	{
864	return 0;
865	}
866
867	static inline
868	void record__read_auxtrace_snapshot(struct record *rec __maybe_unused,
869	bool on_exit __maybe_unused)
870	{
871	}
872
873	static inline
874	int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused)
875	{
876	return 0;
877	}
878
879	static inline
880	int record__auxtrace_snapshot_exit(struct record *rec __maybe_unused)
881	{
882	return 0;
883	}
884
885	static int record__auxtrace_init(struct record *rec __maybe_unused)
886	{
887	return 0;
888	}
889
890	#endif
891
892	static int record__config_text_poke(struct evlist *evlist)
893	{
894	struct evsel *evsel;
895
896	/* Nothing to do if text poke is already configured */
897	evlist__for_each_entry(evlist, evsel) {
898	if (evsel->core.attr.text_poke)
899	return 0;
900	}
901
902	evsel = evlist__add_dummy_on_all_cpus(evlist);
903	if (!evsel)
904	return -ENOMEM;
905
906	evsel->core.attr.text_poke = 1;
907	evsel->core.attr.ksymbol = 1;
908	evsel->immediate = true;
909	evsel__set_sample_bit(evsel, TIME);
910
911	return 0;
912	}
913
914	static int record__config_off_cpu(struct record *rec)
915	{
916	return off_cpu_prepare(evlist: rec->evlist, target: &rec->opts.target, opts: &rec->opts);
917	}
918
919	static bool record__tracking_system_wide(struct record *rec)
920	{
921	struct evlist *evlist = rec->evlist;
922	struct evsel *evsel;
923
924	/*
925	* If non-dummy evsel exists, system_wide sideband is need to
926	* help parse sample information.
927	* For example, PERF_EVENT_MMAP event to help parse symbol,
928	* and PERF_EVENT_COMM event to help parse task executable name.
929	*/
930	evlist__for_each_entry(evlist, evsel) {
931	if (!evsel__is_dummy_event(evsel))
932	return true;
933	}
934
935	return false;
936	}
937
938	static int record__config_tracking_events(struct record *rec)
939	{
940	struct record_opts *opts = &rec->opts;
941	struct evlist *evlist = rec->evlist;
942	bool system_wide = false;
943	struct evsel *evsel;
944
945	/*
946	* For initial_delay, system wide or a hybrid system, we need to add
947	* tracking event so that we can track PERF_RECORD_MMAP to cover the
948	* delay of waiting or event synthesis.
949	*/
950	if (opts->target.initial_delay || target__has_cpu(target: &opts->target) ||
951	perf_pmus__num_core_pmus() > 1) {
952
953	/*
954	* User space tasks can migrate between CPUs, so when tracing
955	* selected CPUs, sideband for all CPUs is still needed.
956	*/
957	if (!!opts->target.cpu_list && record__tracking_system_wide(rec))
958	system_wide = true;
959
960	evsel = evlist__findnew_tracking_event(evlist, system_wide);
961	if (!evsel)
962	return -ENOMEM;
963
964	/*
965	* Enable the tracking event when the process is forked for
966	* initial_delay, immediately for system wide.
967	*/
968	if (opts->target.initial_delay && !evsel->immediate &&
969	!target__has_cpu(target: &opts->target))
970	evsel->core.attr.enable_on_exec = 1;
971	else
972	evsel->immediate = 1;
973	}
974
975	return 0;
976	}
977
978	static bool record__kcore_readable(struct machine *machine)
979	{
980	char kcore[PATH_MAX];
981	int fd;
982
983	scnprintf(buf: kcore, size: sizeof(kcore), fmt: "%s/proc/kcore", machine->root_dir);
984
985	fd = open(kcore, O_RDONLY);
986	if (fd < 0)
987	return false;
988
989	close(fd);
990
991	return true;
992	}
993
994	static int record__kcore_copy(struct machine *machine, struct perf_data *data)
995	{
996	char from_dir[PATH_MAX];
997	char kcore_dir[PATH_MAX];
998	int ret;
999
1000	snprintf(buf: from_dir, size: sizeof(from_dir), fmt: "%s/proc", machine->root_dir);
1001
1002	ret = perf_data__make_kcore_dir(data, buf: kcore_dir, buf_sz: sizeof(kcore_dir));
1003	if (ret)
1004	return ret;
1005
1006	return kcore_copy(from_dir, to_dir: kcore_dir);
1007	}
1008
1009	static void record__thread_data_init_pipes(struct record_thread *thread_data)
1010	{
1011	thread_data->pipes.msg[0] = -1;
1012	thread_data->pipes.msg[1] = -1;
1013	thread_data->pipes.ack[0] = -1;
1014	thread_data->pipes.ack[1] = -1;
1015	}
1016
1017	static int record__thread_data_open_pipes(struct record_thread *thread_data)
1018	{
1019	if (pipe(thread_data->pipes.msg))
1020	return -EINVAL;
1021
1022	if (pipe(thread_data->pipes.ack)) {
1023	close(thread_data->pipes.msg[0]);
1024	thread_data->pipes.msg[0] = -1;
1025	close(thread_data->pipes.msg[1]);
1026	thread_data->pipes.msg[1] = -1;
1027	return -EINVAL;
1028	}
1029
1030	pr_debug2("thread_data[%p]: msg=[%d,%d], ack=[%d,%d]\n", thread_data,
1031	thread_data->pipes.msg[0], thread_data->pipes.msg[1],
1032	thread_data->pipes.ack[0], thread_data->pipes.ack[1]);
1033
1034	return 0;
1035	}
1036
1037	static void record__thread_data_close_pipes(struct record_thread *thread_data)
1038	{
1039	if (thread_data->pipes.msg[0] != -1) {
1040	close(thread_data->pipes.msg[0]);
1041	thread_data->pipes.msg[0] = -1;
1042	}
1043	if (thread_data->pipes.msg[1] != -1) {
1044	close(thread_data->pipes.msg[1]);
1045	thread_data->pipes.msg[1] = -1;
1046	}
1047	if (thread_data->pipes.ack[0] != -1) {
1048	close(thread_data->pipes.ack[0]);
1049	thread_data->pipes.ack[0] = -1;
1050	}
1051	if (thread_data->pipes.ack[1] != -1) {
1052	close(thread_data->pipes.ack[1]);
1053	thread_data->pipes.ack[1] = -1;
1054	}
1055	}
1056
1057	static bool evlist__per_thread(struct evlist *evlist)
1058	{
1059	return cpu_map__is_dummy(cpus: evlist->core.user_requested_cpus);
1060	}
1061
1062	static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
1063	{
1064	int m, tm, nr_mmaps = evlist->core.nr_mmaps;
1065	struct mmap *mmap = evlist->mmap;
1066	struct mmap *overwrite_mmap = evlist->overwrite_mmap;
1067	struct perf_cpu_map *cpus = evlist->core.all_cpus;
1068	bool per_thread = evlist__per_thread(evlist);
1069
1070	if (per_thread)
1071	thread_data->nr_mmaps = nr_mmaps;
1072	else
1073	thread_data->nr_mmaps = bitmap_weight(src: thread_data->mask->maps.bits,
1074	nbits: thread_data->mask->maps.nbits);
1075	if (mmap) {
1076	thread_data->maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
1077	if (!thread_data->maps)
1078	return -ENOMEM;
1079	}
1080	if (overwrite_mmap) {
1081	thread_data->overwrite_maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
1082	if (!thread_data->overwrite_maps) {
1083	zfree(&thread_data->maps);
1084	return -ENOMEM;
1085	}
1086	}
1087	pr_debug2("thread_data[%p]: nr_mmaps=%d, maps=%p, ow_maps=%p\n", thread_data,
1088	thread_data->nr_mmaps, thread_data->maps, thread_data->overwrite_maps);
1089
1090	for (m = 0, tm = 0; m < nr_mmaps && tm < thread_data->nr_mmaps; m++) {
1091	if (per_thread ||
1092	test_bit(perf_cpu_map__cpu(cpus, m).cpu, thread_data->mask->maps.bits)) {
1093	if (thread_data->maps) {
1094	thread_data->maps[tm] = &mmap[m];
1095	pr_debug2("thread_data[%p]: cpu%d: maps[%d] -> mmap[%d]\n",
1096	thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
1097	}
1098	if (thread_data->overwrite_maps) {
1099	thread_data->overwrite_maps[tm] = &overwrite_mmap[m];
1100	pr_debug2("thread_data[%p]: cpu%d: ow_maps[%d] -> ow_mmap[%d]\n",
1101	thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
1102	}
1103	tm++;
1104	}
1105	}
1106
1107	return 0;
1108	}
1109
1110	static int record__thread_data_init_pollfd(struct record_thread *thread_data, struct evlist *evlist)
1111	{
1112	int f, tm, pos;
1113	struct mmap *map, *overwrite_map;
1114
1115	fdarray__init(&thread_data->pollfd, 64);
1116
1117	for (tm = 0; tm < thread_data->nr_mmaps; tm++) {
1118	map = thread_data->maps ? thread_data->maps[tm] : NULL;
1119	overwrite_map = thread_data->overwrite_maps ?
1120	thread_data->overwrite_maps[tm] : NULL;
1121
1122	for (f = 0; f < evlist->core.pollfd.nr; f++) {
1123	void *ptr = evlist->core.pollfd.priv[f].ptr;
1124
1125	if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
1126	pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
1127	&evlist->core.pollfd);
1128	if (pos < 0)
1129	return pos;
1130	pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
1131	thread_data, pos, evlist->core.pollfd.entries[f].fd);
1132	}
1133	}
1134	}
1135
1136	return 0;
1137	}
1138
1139	static void record__free_thread_data(struct record *rec)
1140	{
1141	int t;
1142	struct record_thread *thread_data = rec->thread_data;
1143
1144	if (thread_data == NULL)
1145	return;
1146
1147	for (t = 0; t < rec->nr_threads; t++) {
1148	record__thread_data_close_pipes(thread_data: &thread_data[t]);
1149	zfree(&thread_data[t].maps);
1150	zfree(&thread_data[t].overwrite_maps);
1151	fdarray__exit(&thread_data[t].pollfd);
1152	}
1153
1154	zfree(&rec->thread_data);
1155	}
1156
1157	static int record__map_thread_evlist_pollfd_indexes(struct record *rec,
1158	int evlist_pollfd_index,
1159	int thread_pollfd_index)
1160	{
1161	size_t x = rec->index_map_cnt;
1162
1163	if (realloc_array_as_needed(rec->index_map, rec->index_map_sz, x, NULL))
1164	return -ENOMEM;
1165	rec->index_map[x].evlist_pollfd_index = evlist_pollfd_index;
1166	rec->index_map[x].thread_pollfd_index = thread_pollfd_index;
1167	rec->index_map_cnt += 1;
1168	return 0;
1169	}
1170
1171	static int record__update_evlist_pollfd_from_thread(struct record *rec,
1172	struct evlist *evlist,
1173	struct record_thread *thread_data)
1174	{
1175	struct pollfd *e_entries = evlist->core.pollfd.entries;
1176	struct pollfd *t_entries = thread_data->pollfd.entries;
1177	int err = 0;
1178	size_t i;
1179
1180	for (i = 0; i < rec->index_map_cnt; i++) {
1181	int e_pos = rec->index_map[i].evlist_pollfd_index;
1182	int t_pos = rec->index_map[i].thread_pollfd_index;
1183
1184	if (e_entries[e_pos].fd != t_entries[t_pos].fd ||
1185	e_entries[e_pos].events != t_entries[t_pos].events) {
1186	pr_err("Thread and evlist pollfd index mismatch\n");
1187	err = -EINVAL;
1188	continue;
1189	}
1190	e_entries[e_pos].revents = t_entries[t_pos].revents;
1191	}
1192	return err;
1193	}
1194
1195	static int record__dup_non_perf_events(struct record *rec,
1196	struct evlist *evlist,
1197	struct record_thread *thread_data)
1198	{
1199	struct fdarray *fda = &evlist->core.pollfd;
1200	int i, ret;
1201
1202	for (i = 0; i < fda->nr; i++) {
1203	if (!(fda->priv[i].flags & fdarray_flag__non_perf_event))
1204	continue;
1205	ret = fdarray__dup_entry_from(&thread_data->pollfd, i, fda);
1206	if (ret < 0) {
1207	pr_err("Failed to duplicate descriptor in main thread pollfd\n");
1208	return ret;
1209	}
1210	pr_debug2("thread_data[%p]: pollfd[%d] <- non_perf_event fd=%d\n",
1211	thread_data, ret, fda->entries[i].fd);
1212	ret = record__map_thread_evlist_pollfd_indexes(rec, evlist_pollfd_index: i, thread_pollfd_index: ret);
1213	if (ret < 0) {
1214	pr_err("Failed to map thread and evlist pollfd indexes\n");
1215	return ret;
1216	}
1217	}
1218	return 0;
1219	}
1220
1221	static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
1222	{
1223	int t, ret;
1224	struct record_thread *thread_data;
1225
1226	rec->thread_data = zalloc(rec->nr_threads * sizeof(*(rec->thread_data)));
1227	if (!rec->thread_data) {
1228	pr_err("Failed to allocate thread data\n");
1229	return -ENOMEM;
1230	}
1231	thread_data = rec->thread_data;
1232
1233	for (t = 0; t < rec->nr_threads; t++)
1234	record__thread_data_init_pipes(thread_data: &thread_data[t]);
1235
1236	for (t = 0; t < rec->nr_threads; t++) {
1237	thread_data[t].rec = rec;
1238	thread_data[t].mask = &rec->thread_masks[t];
1239	ret = record__thread_data_init_maps(thread_data: &thread_data[t], evlist);
1240	if (ret) {
1241	pr_err("Failed to initialize thread[%d] maps\n", t);
1242	goto out_free;
1243	}
1244	ret = record__thread_data_init_pollfd(thread_data: &thread_data[t], evlist);
1245	if (ret) {
1246	pr_err("Failed to initialize thread[%d] pollfd\n", t);
1247	goto out_free;
1248	}
1249	if (t) {
1250	thread_data[t].tid = -1;
1251	ret = record__thread_data_open_pipes(thread_data: &thread_data[t]);
1252	if (ret) {
1253	pr_err("Failed to open thread[%d] communication pipes\n", t);
1254	goto out_free;
1255	}
1256	ret = fdarray__add(&thread_data[t].pollfd, thread_data[t].pipes.msg[0],
1257	POLLIN | POLLERR | POLLHUP, fdarray_flag__nonfilterable);
1258	if (ret < 0) {
1259	pr_err("Failed to add descriptor to thread[%d] pollfd\n", t);
1260	goto out_free;
1261	}
1262	thread_data[t].ctlfd_pos = ret;
1263	pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1264	thread_data, thread_data[t].ctlfd_pos,
1265	thread_data[t].pipes.msg[0]);
1266	} else {
1267	thread_data[t].tid = gettid();
1268
1269	ret = record__dup_non_perf_events(rec, evlist, thread_data: &thread_data[t]);
1270	if (ret < 0)
1271	goto out_free;
1272
1273	thread_data[t].ctlfd_pos = -1; /* Not used */
1274	}
1275	}
1276
1277	return 0;
1278
1279	out_free:
1280	record__free_thread_data(rec);
1281
1282	return ret;
1283	}
1284
1285	static int record__mmap_evlist(struct record *rec,
1286	struct evlist *evlist)
1287	{
1288	int i, ret;
1289	struct record_opts *opts = &rec->opts;
1290	bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
1291	opts->auxtrace_sample_mode;
1292	char msg[512];
1293
1294	if (opts->affinity != PERF_AFFINITY_SYS)
1295	cpu__setup_cpunode_map();
1296
1297	if (evlist__mmap_ex(evlist, pages: opts->mmap_pages,
1298	auxtrace_pages: opts->auxtrace_mmap_pages,
1299	auxtrace_overwrite,
1300	nr_cblocks: opts->nr_cblocks, affinity: opts->affinity,
1301	flush: opts->mmap_flush, comp_level: opts->comp_level) < 0) {
1302	if (errno == EPERM) {
1303	pr_err("Permission error mapping pages.\n"
1304	"Consider increasing "
1305	"/proc/sys/kernel/perf_event_mlock_kb,\n"
1306	"or try again with a smaller value of -m/--mmap_pages.\n"
1307	"(current value: %u,%u)\n",
1308	opts->mmap_pages, opts->auxtrace_mmap_pages);
1309	return -errno;
1310	} else {
1311	pr_err("failed to mmap with %d (%s)\n", errno,
1312	str_error_r(errno, msg, sizeof(msg)));
1313	if (errno)
1314	return -errno;
1315	else
1316	return -EINVAL;
1317	}
1318	}
1319
1320	if (evlist__initialize_ctlfd(evlist, ctl_fd: opts->ctl_fd, ctl_fd_ack: opts->ctl_fd_ack))
1321	return -1;
1322
1323	ret = record__alloc_thread_data(rec, evlist);
1324	if (ret)
1325	return ret;
1326
1327	if (record__threads_enabled(rec)) {
1328	ret = perf_data__create_dir(data: &rec->data, nr: evlist->core.nr_mmaps);
1329	if (ret) {
1330	pr_err("Failed to create data directory: %s\n", strerror(-ret));
1331	return ret;
1332	}
1333	for (i = 0; i < evlist->core.nr_mmaps; i++) {
1334	if (evlist->mmap)
1335	evlist->mmap[i].file = &rec->data.dir.files[i];
1336	if (evlist->overwrite_mmap)
1337	evlist->overwrite_mmap[i].file = &rec->data.dir.files[i];
1338	}
1339	}
1340
1341	return 0;
1342	}
1343
1344	static int record__mmap(struct record *rec)
1345	{
1346	return record__mmap_evlist(rec, evlist: rec->evlist);
1347	}
1348
1349	static int record__open(struct record *rec)
1350	{
1351	char msg[BUFSIZ];
1352	struct evsel *pos;
1353	struct evlist *evlist = rec->evlist;
1354	struct perf_session *session = rec->session;
1355	struct record_opts *opts = &rec->opts;
1356	int rc = 0;
1357
1358	evlist__config(evlist, opts, callchain: &callchain_param);
1359
1360	evlist__for_each_entry(evlist, pos) {
1361	try_again:
1362	if (evsel__open(evsel: pos, cpus: pos->core.cpus, threads: pos->core.threads) < 0) {
1363	if (evsel__fallback(pos, &opts->target, errno, msg, sizeof(msg))) {
1364	if (verbose > 0)
1365	ui__warning(format: "%s\n", msg);
1366	goto try_again;
1367	}
1368	if ((errno == EINVAL || errno == EBADF) &&
1369	pos->core.leader != &pos->core &&
1370	pos->weak_group) {
1371	pos = evlist__reset_weak_group(evlist, evsel: pos, close: true);
1372	goto try_again;
1373	}
1374	rc = -errno;
1375	evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
1376	ui__error(format: "%s\n", msg);
1377	goto out;
1378	}
1379
1380	pos->supported = true;
1381	}
1382
1383	if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
1384	pr_warning(
1385	"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
1386	"check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
1387	"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
1388	"file is not found in the buildid cache or in the vmlinux path.\n\n"
1389	"Samples in kernel modules won't be resolved at all.\n\n"
1390	"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
1391	"even with a suitable vmlinux or kallsyms file.\n\n");
1392	}
1393
1394	if (evlist__apply_filters(evlist, err_evsel: &pos)) {
1395	pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n",
1396	pos->filter ?: "BPF", evsel__name(pos), errno,
1397	str_error_r(errno, msg, sizeof(msg)));
1398	rc = -1;
1399	goto out;
1400	}
1401
1402	rc = record__mmap(rec);
1403	if (rc)
1404	goto out;
1405
1406	session->evlist = evlist;
1407	perf_session__set_id_hdr_size(session);
1408	out:
1409	return rc;
1410	}
1411
1412	static void set_timestamp_boundary(struct record *rec, u64 sample_time)
1413	{
1414	if (rec->evlist->first_sample_time == 0)
1415	rec->evlist->first_sample_time = sample_time;
1416
1417	if (sample_time)
1418	rec->evlist->last_sample_time = sample_time;
1419	}
1420
1421	static int process_sample_event(struct perf_tool *tool,
1422	union perf_event *event,
1423	struct perf_sample *sample,
1424	struct evsel *evsel,
1425	struct machine *machine)
1426	{
1427	struct record *rec = container_of(tool, struct record, tool);
1428
1429	set_timestamp_boundary(rec, sample_time: sample->time);
1430
1431	if (rec->buildid_all)
1432	return 0;
1433
1434	rec->samples++;
1435	return build_id__mark_dso_hit(tool, event, sample, evsel, machine);
1436	}
1437
1438	static int process_buildids(struct record *rec)
1439	{
1440	struct perf_session *session = rec->session;
1441
1442	if (perf_data__size(data: &rec->data) == 0)
1443	return 0;
1444
1445	/*
1446	* During this process, it'll load kernel map and replace the
1447	* dso->long_name to a real pathname it found. In this case
1448	* we prefer the vmlinux path like
1449	* /lib/modules/3.16.4/build/vmlinux
1450	*
1451	* rather than build-id path (in debug directory).
1452	* $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
1453	*/
1454	symbol_conf.ignore_vmlinux_buildid = true;
1455
1456	/*
1457	* If --buildid-all is given, it marks all DSO regardless of hits,
1458	* so no need to process samples. But if timestamp_boundary is enabled,
1459	* it still needs to walk on all samples to get the timestamps of
1460	* first/last samples.
1461	*/
1462	if (rec->buildid_all && !rec->timestamp_boundary)
1463	rec->tool.sample = NULL;
1464
1465	return perf_session__process_events(session);
1466	}
1467
1468	static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
1469	{
1470	int err;
1471	struct perf_tool *tool = data;
1472	/*
1473	*As for guest kernel when processing subcommand record&report,
1474	*we arrange module mmap prior to guest kernel mmap and trigger
1475	*a preload dso because default guest module symbols are loaded
1476	*from guest kallsyms instead of /lib/modules/XXX/XXX. This
1477	*method is used to avoid symbol missing when the first addr is
1478	*in module instead of in guest kernel.
1479	*/
1480	err = perf_event__synthesize_modules(tool, process: process_synthesized_event,
1481	machine);
1482	if (err < 0)
1483	pr_err("Couldn't record guest kernel [%d]'s reference"
1484	" relocation symbol.\n", machine->pid);
1485
1486	/*
1487	* We use _stext for guest kernel because guest kernel's /proc/kallsyms
1488	* have no _text sometimes.
1489	*/
1490	err = perf_event__synthesize_kernel_mmap(tool, process: process_synthesized_event,
1491	machine);
1492	if (err < 0)
1493	pr_err("Couldn't record guest kernel [%d]'s reference"
1494	" relocation symbol.\n", machine->pid);
1495	}
1496
1497	static struct perf_event_header finished_round_event = {
1498	.size = sizeof(struct perf_event_header),
1499	.type = PERF_RECORD_FINISHED_ROUND,
1500	};
1501
1502	static struct perf_event_header finished_init_event = {
1503	.size = sizeof(struct perf_event_header),
1504	.type = PERF_RECORD_FINISHED_INIT,
1505	};
1506
1507	static void record__adjust_affinity(struct record *rec, struct mmap *map)
1508	{
1509	if (rec->opts.affinity != PERF_AFFINITY_SYS &&
1510	!bitmap_equal(src1: thread->mask->affinity.bits, src2: map->affinity_mask.bits,
1511	nbits: thread->mask->affinity.nbits)) {
1512	bitmap_zero(dst: thread->mask->affinity.bits, nbits: thread->mask->affinity.nbits);
1513	bitmap_or(dst: thread->mask->affinity.bits, src1: thread->mask->affinity.bits,
1514	src2: map->affinity_mask.bits, nbits: thread->mask->affinity.nbits);
1515	sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
1516	(cpu_set_t *)thread->mask->affinity.bits);
1517	if (verbose == 2) {
1518	pr_debug("threads[%d]: running on cpu%d: ", thread->tid, sched_getcpu());
1519	mmap_cpu_mask__scnprintf(mask: &thread->mask->affinity, tag: "affinity");
1520	}
1521	}
1522	}
1523
1524	static size_t process_comp_header(void *record, size_t increment)
1525	{
1526	struct perf_record_compressed *event = record;
1527	size_t size = sizeof(*event);
1528
1529	if (increment) {
1530	event->header.size += increment;
1531	return increment;
1532	}
1533
1534	event->header.type = PERF_RECORD_COMPRESSED;
1535	event->header.size = size;
1536
1537	return size;
1538	}
1539
1540	static ssize_t zstd_compress(struct perf_session *session, struct mmap *map,
1541	void *dst, size_t dst_size, void *src, size_t src_size)
1542	{
1543	ssize_t compressed;
1544	size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed) - 1;
1545	struct zstd_data *zstd_data = &session->zstd_data;
1546
1547	if (map && map->file)
1548	zstd_data = &map->zstd_data;
1549
1550	compressed = zstd_compress_stream_to_records(zstd_data, dst, dst_size, src, src_size,
1551	max_record_size, process_comp_header);
1552	if (compressed < 0)
1553	return compressed;
1554
1555	if (map && map->file) {
1556	thread->bytes_transferred += src_size;
1557	thread->bytes_compressed += compressed;
1558	} else {
1559	session->bytes_transferred += src_size;
1560	session->bytes_compressed += compressed;
1561	}
1562
1563	return compressed;
1564	}
1565
1566	static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
1567	bool overwrite, bool synch)
1568	{
1569	u64 bytes_written = rec->bytes_written;
1570	int i;
1571	int rc = 0;
1572	int nr_mmaps;
1573	struct mmap **maps;
1574	int trace_fd = rec->data.file.fd;
1575	off_t off = 0;
1576
1577	if (!evlist)
1578	return 0;
1579
1580	nr_mmaps = thread->nr_mmaps;
1581	maps = overwrite ? thread->overwrite_maps : thread->maps;
1582
1583	if (!maps)
1584	return 0;
1585
1586	if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
1587	return 0;
1588
1589	if (record__aio_enabled(rec))
1590	off = record__aio_get_pos(trace_fd);
1591
1592	for (i = 0; i < nr_mmaps; i++) {
1593	u64 flush = 0;
1594	struct mmap *map = maps[i];
1595
1596	if (map->core.base) {
1597	record__adjust_affinity(rec, map);
1598	if (synch) {
1599	flush = map->core.flush;
1600	map->core.flush = 1;
1601	}
1602	if (!record__aio_enabled(rec)) {
1603	if (perf_mmap__push(md: map, to: rec, push: record__pushfn) < 0) {
1604	if (synch)
1605	map->core.flush = flush;
1606	rc = -1;
1607	goto out;
1608	}
1609	} else {
1610	if (record__aio_push(rec, map, off: &off) < 0) {
1611	record__aio_set_pos(trace_fd, pos: off);
1612	if (synch)
1613	map->core.flush = flush;
1614	rc = -1;
1615	goto out;
1616	}
1617	}
1618	if (synch)
1619	map->core.flush = flush;
1620	}
1621
1622	if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
1623	!rec->opts.auxtrace_sample_mode &&
1624	record__auxtrace_mmap_read(rec, map) != 0) {
1625	rc = -1;
1626	goto out;
1627	}
1628	}
1629
1630	if (record__aio_enabled(rec))
1631	record__aio_set_pos(trace_fd, pos: off);
1632
1633	/*
1634	* Mark the round finished in case we wrote
1635	* at least one event.
1636	*
1637	* No need for round events in directory mode,
1638	* because per-cpu maps and files have data
1639	* sorted by kernel.
1640	*/
1641	if (!record__threads_enabled(rec) && bytes_written != rec->bytes_written)
1642	rc = record__write(rec, NULL, bf: &finished_round_event, size: sizeof(finished_round_event));
1643
1644	if (overwrite)
1645	evlist__toggle_bkw_mmap(evlist, state: BKW_MMAP_EMPTY);
1646	out:
1647	return rc;
1648	}
1649
1650	static int record__mmap_read_all(struct record *rec, bool synch)
1651	{
1652	int err;
1653
1654	err = record__mmap_read_evlist(rec, evlist: rec->evlist, overwrite: false, synch);
1655	if (err)
1656	return err;
1657
1658	return record__mmap_read_evlist(rec, evlist: rec->evlist, overwrite: true, synch);
1659	}
1660
1661	static void record__thread_munmap_filtered(struct fdarray *fda, int fd,
1662	void *arg __maybe_unused)
1663	{
1664	struct perf_mmap *map = fda->priv[fd].ptr;
1665
1666	if (map)
1667	perf_mmap__put(map);
1668	}
1669
1670	static void *record__thread(void *arg)
1671	{
1672	enum thread_msg msg = THREAD_MSG__READY;
1673	bool terminate = false;
1674	struct fdarray *pollfd;
1675	int err, ctlfd_pos;
1676
1677	thread = arg;
1678	thread->tid = gettid();
1679
1680	err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1681	if (err == -1)
1682	pr_warning("threads[%d]: failed to notify on start: %s\n",
1683	thread->tid, strerror(errno));
1684
1685	pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
1686
1687	pollfd = &thread->pollfd;
1688	ctlfd_pos = thread->ctlfd_pos;
1689
1690	for (;;) {
1691	unsigned long long hits = thread->samples;
1692
1693	if (record__mmap_read_all(rec: thread->rec, synch: false) < 0 || terminate)
1694	break;
1695
1696	if (hits == thread->samples) {
1697
1698	err = fdarray__poll(pollfd, -1);
1699	/*
1700	* Propagate error, only if there's any. Ignore positive
1701	* number of returned events and interrupt error.
1702	*/
1703	if (err > 0 || (err < 0 && errno == EINTR))
1704	err = 0;
1705	thread->waking++;
1706
1707	if (fdarray__filter(pollfd, POLLERR | POLLHUP,
1708	record__thread_munmap_filtered, NULL) == 0)
1709	break;
1710	}
1711
1712	if (pollfd->entries[ctlfd_pos].revents & POLLHUP) {
1713	terminate = true;
1714	close(thread->pipes.msg[0]);
1715	thread->pipes.msg[0] = -1;
1716	pollfd->entries[ctlfd_pos].fd = -1;
1717	pollfd->entries[ctlfd_pos].events = 0;
1718	}
1719
1720	pollfd->entries[ctlfd_pos].revents = 0;
1721	}
1722	record__mmap_read_all(rec: thread->rec, synch: true);
1723
1724	err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1725	if (err == -1)
1726	pr_warning("threads[%d]: failed to notify on termination: %s\n",
1727	thread->tid, strerror(errno));
1728
1729	return NULL;
1730	}
1731
1732	static void record__init_features(struct record *rec)
1733	{
1734	struct perf_session *session = rec->session;
1735	int feat;
1736
1737	for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
1738	perf_header__set_feat(header: &session->header, feat);
1739
1740	if (rec->no_buildid)
1741	perf_header__clear_feat(header: &session->header, feat: HEADER_BUILD_ID);
1742
1743	#ifdef HAVE_LIBTRACEEVENT
1744	if (!have_tracepoints(&rec->evlist->core.entries))
1745	perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
1746	#endif
1747
1748	if (!rec->opts.branch_stack)
1749	perf_header__clear_feat(header: &session->header, feat: HEADER_BRANCH_STACK);
1750
1751	if (!rec->opts.full_auxtrace)
1752	perf_header__clear_feat(header: &session->header, feat: HEADER_AUXTRACE);
1753
1754	if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
1755	perf_header__clear_feat(header: &session->header, feat: HEADER_CLOCKID);
1756
1757	if (!rec->opts.use_clockid)
1758	perf_header__clear_feat(header: &session->header, feat: HEADER_CLOCK_DATA);
1759
1760	if (!record__threads_enabled(rec))
1761	perf_header__clear_feat(header: &session->header, feat: HEADER_DIR_FORMAT);
1762
1763	if (!record__comp_enabled(rec))
1764	perf_header__clear_feat(header: &session->header, feat: HEADER_COMPRESSED);
1765
1766	perf_header__clear_feat(header: &session->header, feat: HEADER_STAT);
1767	}
1768
1769	static void
1770	record__finish_output(struct record *rec)
1771	{
1772	int i;
1773	struct perf_data *data = &rec->data;
1774	int fd = perf_data__fd(data);
1775
1776	if (data->is_pipe) {
1777	/* Just to display approx. size */
1778	data->file.size = rec->bytes_written;
1779	return;
1780	}
1781
1782	rec->session->header.data_size += rec->bytes_written;
1783	data->file.size = lseek(perf_data__fd(data), 0, SEEK_CUR);
1784	if (record__threads_enabled(rec)) {
1785	for (i = 0; i < data->dir.nr; i++)
1786	data->dir.files[i].size = lseek(data->dir.files[i].fd, 0, SEEK_CUR);
1787	}
1788
1789	if (!rec->no_buildid) {
1790	process_buildids(rec);
1791
1792	if (rec->buildid_all)
1793	dsos__hit_all(session: rec->session);
1794	}
1795	perf_session__write_header(session: rec->session, evlist: rec->evlist, fd, at_exit: true);
1796
1797	return;
1798	}
1799
1800	static int record__synthesize_workload(struct record *rec, bool tail)
1801	{
1802	int err;
1803	struct perf_thread_map *thread_map;
1804	bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
1805
1806	if (rec->opts.tail_synthesize != tail)
1807	return 0;
1808
1809	thread_map = thread_map__new_by_tid(tid: rec->evlist->workload.pid);
1810	if (thread_map == NULL)
1811	return -1;
1812
1813	err = perf_event__synthesize_thread_map(tool: &rec->tool, threads: thread_map,
1814	process: process_synthesized_event,
1815	machine: &rec->session->machines.host,
1816	needs_mmap,
1817	mmap_data: rec->opts.sample_address);
1818	perf_thread_map__put(thread_map);
1819	return err;
1820	}
1821
1822	static int write_finished_init(struct record *rec, bool tail)
1823	{
1824	if (rec->opts.tail_synthesize != tail)
1825	return 0;
1826
1827	return record__write(rec, NULL, bf: &finished_init_event, size: sizeof(finished_init_event));
1828	}
1829
1830	static int record__synthesize(struct record *rec, bool tail);
1831
1832	static int
1833	record__switch_output(struct record *rec, bool at_exit)
1834	{
1835	struct perf_data *data = &rec->data;
1836	char *new_filename = NULL;
1837	int fd, err;
1838
1839	/* Same Size: "2015122520103046"*/
1840	char timestamp[] = "InvalidTimestamp";
1841
1842	record__aio_mmap_read_sync(rec);
1843
1844	write_finished_init(rec, tail: true);
1845
1846	record__synthesize(rec, tail: true);
1847	if (target__none(target: &rec->opts.target))
1848	record__synthesize_workload(rec, tail: true);
1849
1850	rec->samples = 0;
1851	record__finish_output(rec);
1852	err = fetch_current_timestamp(buf: timestamp, sz: sizeof(timestamp));
1853	if (err) {
1854	pr_err("Failed to get current timestamp\n");
1855	return -EINVAL;
1856	}
1857
1858	fd = perf_data__switch(data, postfix: timestamp,
1859	pos: rec->session->header.data_offset,
1860	at_exit, new_filepath: &new_filename);
1861	if (fd >= 0 && !at_exit) {
1862	rec->bytes_written = 0;
1863	rec->session->header.data_size = 0;
1864	}
1865
1866	if (!quiet) {
1867	fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
1868	data->path, timestamp);
1869	}
1870
1871	if (rec->switch_output.num_files) {
1872	int n = rec->switch_output.cur_file + 1;
1873
1874	if (n >= rec->switch_output.num_files)
1875	n = 0;
1876	rec->switch_output.cur_file = n;
1877	if (rec->switch_output.filenames[n]) {
1878	remove(rec->switch_output.filenames[n]);
1879	zfree(&rec->switch_output.filenames[n]);
1880	}
1881	rec->switch_output.filenames[n] = new_filename;
1882	} else {
1883	free(new_filename);
1884	}
1885
1886	/* Output tracking events */
1887	if (!at_exit) {
1888	record__synthesize(rec, tail: false);
1889
1890	/*
1891	* In 'perf record --switch-output' without -a,
1892	* record__synthesize() in record__switch_output() won't
1893	* generate tracking events because there's no thread_map
1894	* in evlist. Which causes newly created perf.data doesn't
1895	* contain map and comm information.
1896	* Create a fake thread_map and directly call
1897	* perf_event__synthesize_thread_map() for those events.
1898	*/
1899	if (target__none(target: &rec->opts.target))
1900	record__synthesize_workload(rec, tail: false);
1901	write_finished_init(rec, tail: false);
1902	}
1903	return fd;
1904	}
1905
1906	static void __record__save_lost_samples(struct record *rec, struct evsel *evsel,
1907	struct perf_record_lost_samples *lost,
1908	int cpu_idx, int thread_idx, u64 lost_count,
1909	u16 misc_flag)
1910	{
1911	struct perf_sample_id *sid;
1912	struct perf_sample sample = {};
1913	int id_hdr_size;
1914
1915	lost->lost = lost_count;
1916	if (evsel->core.ids) {
1917	sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
1918	sample.id = sid->id;
1919	}
1920
1921	id_hdr_size = perf_event__synthesize_id_sample(array: (void *)(lost + 1),
1922	type: evsel->core.attr.sample_type, sample: &sample);
1923	lost->header.size = sizeof(*lost) + id_hdr_size;
1924	lost->header.misc = misc_flag;
1925	record__write(rec, NULL, bf: lost, size: lost->header.size);
1926	}
1927
1928	static void record__read_lost_samples(struct record *rec)
1929	{
1930	struct perf_session *session = rec->session;
1931	struct perf_record_lost_samples *lost = NULL;
1932	struct evsel *evsel;
1933
1934	/* there was an error during record__open */
1935	if (session->evlist == NULL)
1936	return;
1937
1938	evlist__for_each_entry(session->evlist, evsel) {
1939	struct xyarray *xy = evsel->core.sample_id;
1940	u64 lost_count;
1941
1942	if (xy == NULL || evsel->core.fd == NULL)
1943	continue;
1944	if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
1945	xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
1946	pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
1947	continue;
1948	}
1949
1950	for (int x = 0; x < xyarray__max_x(xy); x++) {
1951	for (int y = 0; y < xyarray__max_y(xy); y++) {
1952	struct perf_counts_values count;
1953
1954	if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
1955	pr_debug("read LOST count failed\n");
1956	goto out;
1957	}
1958
1959	if (count.lost) {
1960	if (!lost) {
1961	lost = zalloc(sizeof(*lost) +
1962	session->machines.host.id_hdr_size);
1963	if (!lost) {
1964	pr_debug("Memory allocation failed\n");
1965	return;
1966	}
1967	lost->header.type = PERF_RECORD_LOST_SAMPLES;
1968	}
1969	__record__save_lost_samples(rec, evsel, lost,
1970	cpu_idx: x, thread_idx: y, lost_count: count.lost, misc_flag: 0);
1971	}
1972	}
1973	}
1974
1975	lost_count = perf_bpf_filter__lost_count(evsel);
1976	if (lost_count) {
1977	if (!lost) {
1978	lost = zalloc(sizeof(*lost) +
1979	session->machines.host.id_hdr_size);
1980	if (!lost) {
1981	pr_debug("Memory allocation failed\n");
1982	return;
1983	}
1984	lost->header.type = PERF_RECORD_LOST_SAMPLES;
1985	}
1986	__record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count,
1987	PERF_RECORD_MISC_LOST_SAMPLES_BPF);
1988	}
1989	}
1990	out:
1991	free(lost);
1992	}
1993
1994	static volatile sig_atomic_t workload_exec_errno;
1995
1996	/*
1997	* evlist__prepare_workload will send a SIGUSR1
1998	* if the fork fails, since we asked by setting its
1999	* want_signal to true.
2000	*/
2001	static void workload_exec_failed_signal(int signo __maybe_unused,
2002	siginfo_t *info,
2003	void *ucontext __maybe_unused)
2004	{
2005	workload_exec_errno = info->si_value.sival_int;
2006	done = 1;
2007	child_finished = 1;
2008	}
2009
2010	static void snapshot_sig_handler(int sig);
2011	static void alarm_sig_handler(int sig);
2012
2013	static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
2014	{
2015	if (evlist) {
2016	if (evlist->mmap && evlist->mmap[0].core.base)
2017	return evlist->mmap[0].core.base;
2018	if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
2019	return evlist->overwrite_mmap[0].core.base;
2020	}
2021	return NULL;
2022	}
2023
2024	static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
2025	{
2026	const struct perf_event_mmap_page *pc = evlist__pick_pc(evlist: rec->evlist);
2027	if (pc)
2028	return pc;
2029	return NULL;
2030	}
2031
2032	static int record__synthesize(struct record *rec, bool tail)
2033	{
2034	struct perf_session *session = rec->session;
2035	struct machine *machine = &session->machines.host;
2036	struct perf_data *data = &rec->data;
2037	struct record_opts *opts = &rec->opts;
2038	struct perf_tool *tool = &rec->tool;
2039	int err = 0;
2040	event_op f = process_synthesized_event;
2041
2042	if (rec->opts.tail_synthesize != tail)
2043	return 0;
2044
2045	if (data->is_pipe) {
2046	err = perf_event__synthesize_for_pipe(tool, session, data,
2047	process: process_synthesized_event);
2048	if (err < 0)
2049	goto out;
2050
2051	rec->bytes_written += err;
2052	}
2053
2054	err = perf_event__synth_time_conv(pc: record__pick_pc(rec), tool,
2055	process: process_synthesized_event, machine);
2056	if (err)
2057	goto out;
2058
2059	/* Synthesize id_index before auxtrace_info */
2060	err = perf_event__synthesize_id_index(tool,
2061	process: process_synthesized_event,
2062	evlist: session->evlist, machine);
2063	if (err)
2064	goto out;
2065
2066	if (rec->opts.full_auxtrace) {
2067	err = perf_event__synthesize_auxtrace_info(itr: rec->itr, tool,
2068	session, process: process_synthesized_event);
2069	if (err)
2070	goto out;
2071	}
2072
2073	if (!evlist__exclude_kernel(evlist: rec->evlist)) {
2074	err = perf_event__synthesize_kernel_mmap(tool, process: process_synthesized_event,
2075	machine);
2076	WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
2077	"Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2078	"Check /proc/kallsyms permission or run as root.\n");
2079
2080	err = perf_event__synthesize_modules(tool, process: process_synthesized_event,
2081	machine);
2082	WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
2083	"Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2084	"Check /proc/modules permission or run as root.\n");
2085	}
2086
2087	if (perf_guest) {
2088	machines__process_guests(machines: &session->machines,
2089	process: perf_event__synthesize_guest_os, data: tool);
2090	}
2091
2092	err = perf_event__synthesize_extra_attr(tool: &rec->tool,
2093	evsel_list: rec->evlist,
2094	process: process_synthesized_event,
2095	is_pipe: data->is_pipe);
2096	if (err)
2097	goto out;
2098
2099	err = perf_event__synthesize_thread_map2(tool: &rec->tool, threads: rec->evlist->core.threads,
2100	process: process_synthesized_event,
2101	NULL);
2102	if (err < 0) {
2103	pr_err("Couldn't synthesize thread map.\n");
2104	return err;
2105	}
2106
2107	err = perf_event__synthesize_cpu_map(tool: &rec->tool, cpus: rec->evlist->core.all_cpus,
2108	process: process_synthesized_event, NULL);
2109	if (err < 0) {
2110	pr_err("Couldn't synthesize cpu map.\n");
2111	return err;
2112	}
2113
2114	err = perf_event__synthesize_bpf_events(session, process: process_synthesized_event,
2115	machine, opts);
2116	if (err < 0) {
2117	pr_warning("Couldn't synthesize bpf events.\n");
2118	err = 0;
2119	}
2120
2121	if (rec->opts.synth & PERF_SYNTH_CGROUP) {
2122	err = perf_event__synthesize_cgroups(tool, process: process_synthesized_event,
2123	machine);
2124	if (err < 0) {
2125	pr_warning("Couldn't synthesize cgroup events.\n");
2126	err = 0;
2127	}
2128	}
2129
2130	if (rec->opts.nr_threads_synthesize > 1) {
2131	mutex_init(&synth_lock);
2132	perf_set_multithreaded();
2133	f = process_locked_synthesized_event;
2134	}
2135
2136	if (rec->opts.synth & PERF_SYNTH_TASK) {
2137	bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
2138
2139	err = __machine__synthesize_threads(machine, tool, target: &opts->target,
2140	threads: rec->evlist->core.threads,
2141	process: f, needs_mmap, data_mmap: opts->sample_address,
2142	nr_threads_synthesize: rec->opts.nr_threads_synthesize);
2143	}
2144
2145	if (rec->opts.nr_threads_synthesize > 1) {
2146	perf_set_singlethreaded();
2147	mutex_destroy(lock: &synth_lock);
2148	}
2149
2150	out:
2151	return err;
2152	}
2153
2154	static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
2155	{
2156	struct record *rec = data;
2157	pthread_kill(rec->thread_id, SIGUSR2);
2158	return 0;
2159	}
2160
2161	static int record__setup_sb_evlist(struct record *rec)
2162	{
2163	struct record_opts *opts = &rec->opts;
2164
2165	if (rec->sb_evlist != NULL) {
2166	/*
2167	* We get here if --switch-output-event populated the
2168	* sb_evlist, so associate a callback that will send a SIGUSR2
2169	* to the main thread.
2170	*/
2171	evlist__set_cb(evlist: rec->sb_evlist, cb: record__process_signal_event, data: rec);
2172	rec->thread_id = pthread_self();
2173	}
2174	#ifdef HAVE_LIBBPF_SUPPORT
2175	if (!opts->no_bpf_event) {
2176	if (rec->sb_evlist == NULL) {
2177	rec->sb_evlist = evlist__new();
2178
2179	if (rec->sb_evlist == NULL) {
2180	pr_err("Couldn't create side band evlist.\n.");
2181	return -1;
2182	}
2183	}
2184
2185	if (evlist__add_bpf_sb_event(rec->sb_evlist, &rec->session->header.env)) {
2186	pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
2187	return -1;
2188	}
2189	}
2190	#endif
2191	if (evlist__start_sb_thread(evlist: rec->sb_evlist, target: &rec->opts.target)) {
2192	pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
2193	opts->no_bpf_event = true;
2194	}
2195
2196	return 0;
2197	}
2198
2199	static int record__init_clock(struct record *rec)
2200	{
2201	struct perf_session *session = rec->session;
2202	struct timespec ref_clockid;
2203	struct timeval ref_tod;
2204	u64 ref;
2205
2206	if (!rec->opts.use_clockid)
2207	return 0;
2208
2209	if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
2210	session->header.env.clock.clockid_res_ns = rec->opts.clockid_res_ns;
2211
2212	session->header.env.clock.clockid = rec->opts.clockid;
2213
2214	if (gettimeofday(&ref_tod, NULL) != 0) {
2215	pr_err("gettimeofday failed, cannot set reference time.\n");
2216	return -1;
2217	}
2218
2219	if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
2220	pr_err("clock_gettime failed, cannot set reference time.\n");
2221	return -1;
2222	}
2223
2224	ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
2225	(u64) ref_tod.tv_usec * NSEC_PER_USEC;
2226
2227	session->header.env.clock.tod_ns = ref;
2228
2229	ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
2230	(u64) ref_clockid.tv_nsec;
2231
2232	session->header.env.clock.clockid_ns = ref;
2233	return 0;
2234	}
2235
2236	static void hit_auxtrace_snapshot_trigger(struct record *rec)
2237	{
2238	if (trigger_is_ready(t: &auxtrace_snapshot_trigger)) {
2239	trigger_hit(t: &auxtrace_snapshot_trigger);
2240	auxtrace_record__snapshot_started = 1;
2241	if (auxtrace_record__snapshot_start(itr: rec->itr))
2242	trigger_error(t: &auxtrace_snapshot_trigger);
2243	}
2244	}
2245
2246	static int record__terminate_thread(struct record_thread *thread_data)
2247	{
2248	int err;
2249	enum thread_msg ack = THREAD_MSG__UNDEFINED;
2250	pid_t tid = thread_data->tid;
2251
2252	close(thread_data->pipes.msg[1]);
2253	thread_data->pipes.msg[1] = -1;
2254	err = read(thread_data->pipes.ack[0], &ack, sizeof(ack));
2255	if (err > 0)
2256	pr_debug2("threads[%d]: sent %s\n", tid, thread_msg_tags[ack]);
2257	else
2258	pr_warning("threads[%d]: failed to receive termination notification from %d\n",
2259	thread->tid, tid);
2260
2261	return 0;
2262	}
2263
2264	static int record__start_threads(struct record *rec)
2265	{
2266	int t, tt, err, ret = 0, nr_threads = rec->nr_threads;
2267	struct record_thread *thread_data = rec->thread_data;
2268	sigset_t full, mask;
2269	pthread_t handle;
2270	pthread_attr_t attrs;
2271
2272	thread = &thread_data[0];
2273
2274	if (!record__threads_enabled(rec))
2275	return 0;
2276
2277	sigfillset(set: &full);
2278	if (sigprocmask(SIG_SETMASK, &full, &mask)) {
2279	pr_err("Failed to block signals on threads start: %s\n", strerror(errno));
2280	return -1;
2281	}
2282
2283	pthread_attr_init(&attrs);
2284	pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);
2285
2286	for (t = 1; t < nr_threads; t++) {
2287	enum thread_msg msg = THREAD_MSG__UNDEFINED;
2288
2289	#ifdef HAVE_PTHREAD_ATTR_SETAFFINITY_NP
2290	pthread_attr_setaffinity_np(&attrs,
2291	MMAP_CPU_MASK_BYTES(&(thread_data[t].mask->affinity)),
2292	(cpu_set_t *)(thread_data[t].mask->affinity.bits));
2293	#endif
2294	if (pthread_create(&handle, &attrs, record__thread, &thread_data[t])) {
2295	for (tt = 1; tt < t; tt++)
2296	record__terminate_thread(thread_data: &thread_data[t]);
2297	pr_err("Failed to start threads: %s\n", strerror(errno));
2298	ret = -1;
2299	goto out_err;
2300	}
2301
2302	err = read(thread_data[t].pipes.ack[0], &msg, sizeof(msg));
2303	if (err > 0)
2304	pr_debug2("threads[%d]: sent %s\n", rec->thread_data[t].tid,
2305	thread_msg_tags[msg]);
2306	else
2307	pr_warning("threads[%d]: failed to receive start notification from %d\n",
2308	thread->tid, rec->thread_data[t].tid);
2309	}
2310
2311	sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
2312	(cpu_set_t *)thread->mask->affinity.bits);
2313
2314	pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
2315
2316	out_err:
2317	pthread_attr_destroy(&attrs);
2318
2319	if (sigprocmask(SIG_SETMASK, &mask, NULL)) {
2320	pr_err("Failed to unblock signals on threads start: %s\n", strerror(errno));
2321	ret = -1;
2322	}
2323
2324	return ret;
2325	}
2326
2327	static int record__stop_threads(struct record *rec)
2328	{
2329	int t;
2330	struct record_thread *thread_data = rec->thread_data;
2331
2332	for (t = 1; t < rec->nr_threads; t++)
2333	record__terminate_thread(thread_data: &thread_data[t]);
2334
2335	for (t = 0; t < rec->nr_threads; t++) {
2336	rec->samples += thread_data[t].samples;
2337	if (!record__threads_enabled(rec))
2338	continue;
2339	rec->session->bytes_transferred += thread_data[t].bytes_transferred;
2340	rec->session->bytes_compressed += thread_data[t].bytes_compressed;
2341	pr_debug("threads[%d]: samples=%lld, wakes=%ld, ", thread_data[t].tid,
2342	thread_data[t].samples, thread_data[t].waking);
2343	if (thread_data[t].bytes_transferred && thread_data[t].bytes_compressed)
2344	pr_debug("transferred=%" PRIu64 ", compressed=%" PRIu64 "\n",
2345	thread_data[t].bytes_transferred, thread_data[t].bytes_compressed);
2346	else
2347	pr_debug("written=%" PRIu64 "\n", thread_data[t].bytes_written);
2348	}
2349
2350	return 0;
2351	}
2352
2353	static unsigned long record__waking(struct record *rec)
2354	{
2355	int t;
2356	unsigned long waking = 0;
2357	struct record_thread *thread_data = rec->thread_data;
2358
2359	for (t = 0; t < rec->nr_threads; t++)
2360	waking += thread_data[t].waking;
2361
2362	return waking;
2363	}
2364
2365	static int __cmd_record(struct record *rec, int argc, const char **argv)
2366	{
2367	int err;
2368	int status = 0;
2369	const bool forks = argc > 0;
2370	struct perf_tool *tool = &rec->tool;
2371	struct record_opts *opts = &rec->opts;
2372	struct perf_data *data = &rec->data;
2373	struct perf_session *session;
2374	bool disabled = false, draining = false;
2375	int fd;
2376	float ratio = 0;
2377	enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
2378
2379	atexit(record__sig_exit);
2380	signal(SIGCHLD, sig_handler);
2381	signal(SIGINT, sig_handler);
2382	signal(SIGTERM, sig_handler);
2383	signal(SIGSEGV, sigsegv_handler);
2384
2385	if (rec->opts.record_namespaces)
2386	tool->namespace_events = true;
2387
2388	if (rec->opts.record_cgroup) {
2389	#ifdef HAVE_FILE_HANDLE
2390	tool->cgroup_events = true;
2391	#else
2392	pr_err("cgroup tracking is not supported\n");
2393	return -1;
2394	#endif
2395	}
2396
2397	if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
2398	signal(SIGUSR2, snapshot_sig_handler);
2399	if (rec->opts.auxtrace_snapshot_mode)
2400	trigger_on(t: &auxtrace_snapshot_trigger);
2401	if (rec->switch_output.enabled)
2402	trigger_on(t: &switch_output_trigger);
2403	} else {
2404	signal(SIGUSR2, SIG_IGN);
2405	}
2406
2407	session = perf_session__new(data, tool);
2408	if (IS_ERR(ptr: session)) {
2409	pr_err("Perf session creation failed.\n");
2410	return PTR_ERR(ptr: session);
2411	}
2412
2413	if (record__threads_enabled(rec)) {
2414	if (perf_data__is_pipe(data: &rec->data)) {
2415	pr_err("Parallel trace streaming is not available in pipe mode.\n");
2416	return -1;
2417	}
2418	if (rec->opts.full_auxtrace) {
2419	pr_err("Parallel trace streaming is not available in AUX area tracing mode.\n");
2420	return -1;
2421	}
2422	}
2423
2424	fd = perf_data__fd(data);
2425	rec->session = session;
2426
2427	if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
2428	pr_err("Compression initialization failed.\n");
2429	return -1;
2430	}
2431	#ifdef HAVE_EVENTFD_SUPPORT
2432	done_fd = eventfd(0, EFD_NONBLOCK);
2433	if (done_fd < 0) {
2434	pr_err("Failed to create wakeup eventfd, error: %m\n");
2435	status = -1;
2436	goto out_delete_session;
2437	}
2438	err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
2439	if (err < 0) {
2440	pr_err("Failed to add wakeup eventfd to poll list\n");
2441	status = err;
2442	goto out_delete_session;
2443	}
2444	#endif // HAVE_EVENTFD_SUPPORT
2445
2446	session->header.env.comp_type = PERF_COMP_ZSTD;
2447	session->header.env.comp_level = rec->opts.comp_level;
2448
2449	if (rec->opts.kcore &&
2450	!record__kcore_readable(machine: &session->machines.host)) {
2451	pr_err("ERROR: kcore is not readable.\n");
2452	return -1;
2453	}
2454
2455	if (record__init_clock(rec))
2456	return -1;
2457
2458	record__init_features(rec);
2459
2460	if (forks) {
2461	err = evlist__prepare_workload(evlist: rec->evlist, target: &opts->target, argv, pipe_output: data->is_pipe,
2462	exec_error: workload_exec_failed_signal);
2463	if (err < 0) {
2464	pr_err("Couldn't run the workload!\n");
2465	status = err;
2466	goto out_delete_session;
2467	}
2468	}
2469
2470	/*
2471	* If we have just single event and are sending data
2472	* through pipe, we need to force the ids allocation,
2473	* because we synthesize event name through the pipe
2474	* and need the id for that.
2475	*/
2476	if (data->is_pipe && rec->evlist->core.nr_entries == 1)
2477	rec->opts.sample_id = true;
2478
2479	if (rec->timestamp_filename && perf_data__is_pipe(data)) {
2480	rec->timestamp_filename = false;
2481	pr_warning("WARNING: --timestamp-filename option is not available in pipe mode.\n");
2482	}
2483
2484	evlist__uniquify_name(evlist: rec->evlist);
2485
2486	/* Debug message used by test scripts */
2487	pr_debug3("perf record opening and mmapping events\n");
2488	if (record__open(rec) != 0) {
2489	err = -1;
2490	goto out_free_threads;
2491	}
2492	/* Debug message used by test scripts */
2493	pr_debug3("perf record done opening and mmapping events\n");
2494	session->header.env.comp_mmap_len = session->evlist->core.mmap_len;
2495
2496	if (rec->opts.kcore) {
2497	err = record__kcore_copy(machine: &session->machines.host, data);
2498	if (err) {
2499	pr_err("ERROR: Failed to copy kcore\n");
2500	goto out_free_threads;
2501	}
2502	}
2503
2504	/*
2505	* Normally perf_session__new would do this, but it doesn't have the
2506	* evlist.
2507	*/
2508	if (rec->tool.ordered_events && !evlist__sample_id_all(evlist: rec->evlist)) {
2509	pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
2510	rec->tool.ordered_events = false;
2511	}
2512
2513	if (evlist__nr_groups(evlist: rec->evlist) == 0)
2514	perf_header__clear_feat(header: &session->header, feat: HEADER_GROUP_DESC);
2515
2516	if (data->is_pipe) {
2517	err = perf_header__write_pipe(fd);
2518	if (err < 0)
2519	goto out_free_threads;
2520	} else {
2521	err = perf_session__write_header(session, evlist: rec->evlist, fd, at_exit: false);
2522	if (err < 0)
2523	goto out_free_threads;
2524	}
2525
2526	err = -1;
2527	if (!rec->no_buildid
2528	&& !perf_header__has_feat(header: &session->header, feat: HEADER_BUILD_ID)) {
2529	pr_err("Couldn't generate buildids. "
2530	"Use --no-buildid to profile anyway.\n");
2531	goto out_free_threads;
2532	}
2533
2534	err = record__setup_sb_evlist(rec);
2535	if (err)
2536	goto out_free_threads;
2537
2538	err = record__synthesize(rec, tail: false);
2539	if (err < 0)
2540	goto out_free_threads;
2541
2542	if (rec->realtime_prio) {
2543	struct sched_param param;
2544
2545	param.sched_priority = rec->realtime_prio;
2546	if (sched_setscheduler(0, SCHED_FIFO, &param)) {
2547	pr_err("Could not set realtime priority.\n");
2548	err = -1;
2549	goto out_free_threads;
2550	}
2551	}
2552
2553	if (record__start_threads(rec))
2554	goto out_free_threads;
2555
2556	/*
2557	* When perf is starting the traced process, all the events
2558	* (apart from group members) have enable_on_exec=1 set,
2559	* so don't spoil it by prematurely enabling them.
2560	*/
2561	if (!target__none(target: &opts->target) && !opts->target.initial_delay)
2562	evlist__enable(evlist: rec->evlist);
2563
2564	/*
2565	* Let the child rip
2566	*/
2567	if (forks) {
2568	struct machine *machine = &session->machines.host;
2569	union perf_event *event;
2570	pid_t tgid;
2571
2572	event = malloc(sizeof(event->comm) + machine->id_hdr_size);
2573	if (event == NULL) {
2574	err = -ENOMEM;
2575	goto out_child;
2576	}
2577
2578	/*
2579	* Some H/W events are generated before COMM event
2580	* which is emitted during exec(), so perf script
2581	* cannot see a correct process name for those events.
2582	* Synthesize COMM event to prevent it.
2583	*/
2584	tgid = perf_event__synthesize_comm(tool, event,
2585	pid: rec->evlist->workload.pid,
2586	process: process_synthesized_event,
2587	machine);
2588	free(event);
2589
2590	if (tgid == -1)
2591	goto out_child;
2592
2593	event = malloc(sizeof(event->namespaces) +
2594	(NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
2595	machine->id_hdr_size);
2596	if (event == NULL) {
2597	err = -ENOMEM;
2598	goto out_child;
2599	}
2600
2601	/*
2602	* Synthesize NAMESPACES event for the command specified.
2603	*/
2604	perf_event__synthesize_namespaces(tool, event,
2605	pid: rec->evlist->workload.pid,
2606	tgid, process: process_synthesized_event,
2607	machine);
2608	free(event);
2609
2610	evlist__start_workload(evlist: rec->evlist);
2611	}
2612
2613	if (opts->target.initial_delay) {
2614	pr_info(EVLIST_DISABLED_MSG);
2615	if (opts->target.initial_delay > 0) {
2616	usleep(opts->target.initial_delay * USEC_PER_MSEC);
2617	evlist__enable(evlist: rec->evlist);
2618	pr_info(EVLIST_ENABLED_MSG);
2619	}
2620	}
2621
2622	err = event_enable_timer__start(eet: rec->evlist->eet);
2623	if (err)
2624	goto out_child;
2625
2626	/* Debug message used by test scripts */
2627	pr_debug3("perf record has started\n");
2628	fflush(stderr);
2629
2630	trigger_ready(t: &auxtrace_snapshot_trigger);
2631	trigger_ready(t: &switch_output_trigger);
2632	perf_hooks__invoke_record_start();
2633
2634	/*
2635	* Must write FINISHED_INIT so it will be seen after all other
2636	* synthesized user events, but before any regular events.
2637	*/
2638	err = write_finished_init(rec, tail: false);
2639	if (err < 0)
2640	goto out_child;
2641
2642	for (;;) {
2643	unsigned long long hits = thread->samples;
2644
2645	/*
2646	* rec->evlist->bkw_mmap_state is possible to be
2647	* BKW_MMAP_EMPTY here: when done == true and
2648	* hits != rec->samples in previous round.
2649	*
2650	* evlist__toggle_bkw_mmap ensure we never
2651	* convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
2652	*/
2653	if (trigger_is_hit(t: &switch_output_trigger) || done || draining)
2654	evlist__toggle_bkw_mmap(evlist: rec->evlist, state: BKW_MMAP_DATA_PENDING);
2655
2656	if (record__mmap_read_all(rec, synch: false) < 0) {
2657	trigger_error(t: &auxtrace_snapshot_trigger);
2658	trigger_error(t: &switch_output_trigger);
2659	err = -1;
2660	goto out_child;
2661	}
2662
2663	if (auxtrace_record__snapshot_started) {
2664	auxtrace_record__snapshot_started = 0;
2665	if (!trigger_is_error(t: &auxtrace_snapshot_trigger))
2666	record__read_auxtrace_snapshot(rec, on_exit: false);
2667	if (trigger_is_error(t: &auxtrace_snapshot_trigger)) {
2668	pr_err("AUX area tracing snapshot failed\n");
2669	err = -1;
2670	goto out_child;
2671	}
2672	}
2673
2674	if (trigger_is_hit(t: &switch_output_trigger)) {
2675	/*
2676	* If switch_output_trigger is hit, the data in
2677	* overwritable ring buffer should have been collected,
2678	* so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
2679	*
2680	* If SIGUSR2 raise after or during record__mmap_read_all(),
2681	* record__mmap_read_all() didn't collect data from
2682	* overwritable ring buffer. Read again.
2683	*/
2684	if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
2685	continue;
2686	trigger_ready(t: &switch_output_trigger);
2687
2688	/*
2689	* Reenable events in overwrite ring buffer after
2690	* record__mmap_read_all(): we should have collected
2691	* data from it.
2692	*/
2693	evlist__toggle_bkw_mmap(evlist: rec->evlist, state: BKW_MMAP_RUNNING);
2694
2695	if (!quiet)
2696	fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
2697	record__waking(rec));
2698	thread->waking = 0;
2699	fd = record__switch_output(rec, at_exit: false);
2700	if (fd < 0) {
2701	pr_err("Failed to switch to new file\n");
2702	trigger_error(t: &switch_output_trigger);
2703	err = fd;
2704	goto out_child;
2705	}
2706
2707	/* re-arm the alarm */
2708	if (rec->switch_output.time)
2709	alarm(rec->switch_output.time);
2710	}
2711
2712	if (hits == thread->samples) {
2713	if (done || draining)
2714	break;
2715	err = fdarray__poll(&thread->pollfd, -1);
2716	/*
2717	* Propagate error, only if there's any. Ignore positive
2718	* number of returned events and interrupt error.
2719	*/
2720	if (err > 0 || (err < 0 && errno == EINTR))
2721	err = 0;
2722	thread->waking++;
2723
2724	if (fdarray__filter(&thread->pollfd, POLLERR | POLLHUP,
2725	record__thread_munmap_filtered, NULL) == 0)
2726	draining = true;
2727
2728	err = record__update_evlist_pollfd_from_thread(rec, evlist: rec->evlist, thread_data: thread);
2729	if (err)
2730	goto out_child;
2731	}
2732
2733	if (evlist__ctlfd_process(evlist: rec->evlist, cmd: &cmd) > 0) {
2734	switch (cmd) {
2735	case EVLIST_CTL_CMD_SNAPSHOT:
2736	hit_auxtrace_snapshot_trigger(rec);
2737	evlist__ctlfd_ack(evlist: rec->evlist);
2738	break;
2739	case EVLIST_CTL_CMD_STOP:
2740	done = 1;
2741	break;
2742	case EVLIST_CTL_CMD_ACK:
2743	case EVLIST_CTL_CMD_UNSUPPORTED:
2744	case EVLIST_CTL_CMD_ENABLE:
2745	case EVLIST_CTL_CMD_DISABLE:
2746	case EVLIST_CTL_CMD_EVLIST:
2747	case EVLIST_CTL_CMD_PING:
2748	default:
2749	break;
2750	}
2751	}
2752
2753	err = event_enable_timer__process(eet: rec->evlist->eet);
2754	if (err < 0)
2755	goto out_child;
2756	if (err) {
2757	err = 0;
2758	done = 1;
2759	}
2760
2761	/*
2762	* When perf is starting the traced process, at the end events
2763	* die with the process and we wait for that. Thus no need to
2764	* disable events in this case.
2765	*/
2766	if (done && !disabled && !target__none(target: &opts->target)) {
2767	trigger_off(t: &auxtrace_snapshot_trigger);
2768	evlist__disable(evlist: rec->evlist);
2769	disabled = true;
2770	}
2771	}
2772
2773	trigger_off(t: &auxtrace_snapshot_trigger);
2774	trigger_off(t: &switch_output_trigger);
2775
2776	if (opts->auxtrace_snapshot_on_exit)
2777	record__auxtrace_snapshot_exit(rec);
2778
2779	if (forks && workload_exec_errno) {
2780	char msg[STRERR_BUFSIZE], strevsels[2048];
2781	const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));
2782
2783	evlist__scnprintf_evsels(evlist: rec->evlist, size: sizeof(strevsels), bf: strevsels);
2784
2785	pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
2786	strevsels, argv[0], emsg);
2787	err = -1;
2788	goto out_child;
2789	}
2790
2791	if (!quiet)
2792	fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n",
2793	record__waking(rec));
2794
2795	write_finished_init(rec, tail: true);
2796
2797	if (target__none(target: &rec->opts.target))
2798	record__synthesize_workload(rec, tail: true);
2799
2800	out_child:
2801	record__stop_threads(rec);
2802	record__mmap_read_all(rec, synch: true);
2803	out_free_threads:
2804	record__free_thread_data(rec);
2805	evlist__finalize_ctlfd(evlist: rec->evlist);
2806	record__aio_mmap_read_sync(rec);
2807
2808	if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
2809	ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
2810	session->header.env.comp_ratio = ratio + 0.5;
2811	}
2812
2813	if (forks) {
2814	int exit_status;
2815
2816	if (!child_finished)
2817	kill(rec->evlist->workload.pid, SIGTERM);
2818
2819	wait(&exit_status);
2820
2821	if (err < 0)
2822	status = err;
2823	else if (WIFEXITED(exit_status))
2824	status = WEXITSTATUS(exit_status);
2825	else if (WIFSIGNALED(exit_status))
2826	signr = WTERMSIG(exit_status);
2827	} else
2828	status = err;
2829
2830	if (rec->off_cpu)
2831	rec->bytes_written += off_cpu_write(session: rec->session);
2832
2833	record__read_lost_samples(rec);
2834	record__synthesize(rec, tail: true);
2835	/* this will be recalculated during process_buildids() */
2836	rec->samples = 0;
2837
2838	if (!err) {
2839	if (!rec->timestamp_filename) {
2840	record__finish_output(rec);
2841	} else {
2842	fd = record__switch_output(rec, at_exit: true);
2843	if (fd < 0) {
2844	status = fd;
2845	goto out_delete_session;
2846	}
2847	}
2848	}
2849
2850	perf_hooks__invoke_record_end();
2851
2852	if (!err && !quiet) {
2853	char samples[128];
2854	const char *postfix = rec->timestamp_filename ?
2855	".<timestamp>" : "";
2856
2857	if (rec->samples && !rec->opts.full_auxtrace)
2858	scnprintf(samples, sizeof(samples),
2859	" (%" PRIu64 " samples)", rec->samples);
2860	else
2861	samples[0] = '\0';
2862
2863	fprintf(stderr, "[ perf record: Captured and wrote %.3f MB %s%s%s",
2864	perf_data__size(data) / 1024.0 / 1024.0,
2865	data->path, postfix, samples);
2866	if (ratio) {
2867	fprintf(stderr, ", compressed (original %.3f MB, ratio is %.3f)",
2868	rec->session->bytes_transferred / 1024.0 / 1024.0,
2869	ratio);
2870	}
2871	fprintf(stderr, " ]\n");
2872	}
2873
2874	out_delete_session:
2875	#ifdef HAVE_EVENTFD_SUPPORT
2876	if (done_fd >= 0) {
2877	fd = done_fd;
2878	done_fd = -1;
2879
2880	close(fd);
2881	}
2882	#endif
2883	zstd_fini(&session->zstd_data);
2884	perf_session__delete(session);
2885
2886	if (!opts->no_bpf_event)
2887	evlist__stop_sb_thread(evlist: rec->sb_evlist);
2888	return status;
2889	}
2890
2891	static void callchain_debug(struct callchain_param *callchain)
2892	{
2893	static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" };
2894
2895	pr_debug("callchain: type %s\n", str[callchain->record_mode]);
2896
2897	if (callchain->record_mode == CALLCHAIN_DWARF)
2898	pr_debug("callchain: stack dump size %d\n",
2899	callchain->dump_size);
2900	}
2901
2902	int record_opts__parse_callchain(struct record_opts *record,
2903	struct callchain_param *callchain,
2904	const char *arg, bool unset)
2905	{
2906	int ret;
2907	callchain->enabled = !unset;
2908
2909	/* --no-call-graph */
2910	if (unset) {
2911	callchain->record_mode = CALLCHAIN_NONE;
2912	pr_debug("callchain: disabled\n");
2913	return 0;
2914	}
2915
2916	ret = parse_callchain_record_opt(arg, param: callchain);
2917	if (!ret) {
2918	/* Enable data address sampling for DWARF unwind. */
2919	if (callchain->record_mode == CALLCHAIN_DWARF)
2920	record->sample_address = true;
2921	callchain_debug(callchain);
2922	}
2923
2924	return ret;
2925	}
2926
2927	int record_parse_callchain_opt(const struct option *opt,
2928	const char *arg,
2929	int unset)
2930	{
2931	return record_opts__parse_callchain(record: opt->value, callchain: &callchain_param, arg, unset);
2932	}
2933
2934	int record_callchain_opt(const struct option *opt,
2935	const char *arg __maybe_unused,
2936	int unset __maybe_unused)
2937	{
2938	struct callchain_param *callchain = opt->value;
2939
2940	callchain->enabled = true;
2941
2942	if (callchain->record_mode == CALLCHAIN_NONE)
2943	callchain->record_mode = CALLCHAIN_FP;
2944
2945	callchain_debug(callchain);
2946	return 0;
2947	}
2948
2949	static int perf_record_config(const char *var, const char *value, void *cb)
2950	{
2951	struct record *rec = cb;
2952
2953	if (!strcmp(var, "record.build-id")) {
2954	if (!strcmp(value, "cache"))
2955	rec->no_buildid_cache = false;
2956	else if (!strcmp(value, "no-cache"))
2957	rec->no_buildid_cache = true;
2958	else if (!strcmp(value, "skip"))
2959	rec->no_buildid = true;
2960	else if (!strcmp(value, "mmap"))
2961	rec->buildid_mmap = true;
2962	else
2963	return -1;
2964	return 0;
2965	}
2966	if (!strcmp(var, "record.call-graph")) {
2967	var = "call-graph.record-mode";
2968	return perf_default_config(var, value, cb);
2969	}
2970	#ifdef HAVE_AIO_SUPPORT
2971	if (!strcmp(var, "record.aio")) {
2972	rec->opts.nr_cblocks = strtol(value, NULL, 0);
2973	if (!rec->opts.nr_cblocks)
2974	rec->opts.nr_cblocks = nr_cblocks_default;
2975	}
2976	#endif
2977	if (!strcmp(var, "record.debuginfod")) {
2978	rec->debuginfod.urls = strdup(value);
2979	if (!rec->debuginfod.urls)
2980	return -ENOMEM;
2981	rec->debuginfod.set = true;
2982	}
2983
2984	return 0;
2985	}
2986
2987	static int record__parse_event_enable_time(const struct option *opt, const char *str, int unset)
2988	{
2989	struct record *rec = (struct record *)opt->value;
2990
2991	return evlist__parse_event_enable_time(evlist: rec->evlist, opts: &rec->opts, str, unset);
2992	}
2993
2994	static int record__parse_affinity(const struct option *opt, const char *str, int unset)
2995	{
2996	struct record_opts *opts = (struct record_opts *)opt->value;
2997
2998	if (unset || !str)
2999	return 0;
3000
3001	if (!strcasecmp(s1: str, s2: "node"))
3002	opts->affinity = PERF_AFFINITY_NODE;
3003	else if (!strcasecmp(s1: str, s2: "cpu"))
3004	opts->affinity = PERF_AFFINITY_CPU;
3005
3006	return 0;
3007	}
3008
3009	static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits)
3010	{
3011	mask->nbits = nr_bits;
3012	mask->bits = bitmap_zalloc(mask->nbits);
3013	if (!mask->bits)
3014	return -ENOMEM;
3015
3016	return 0;
3017	}
3018
3019	static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask)
3020	{
3021	bitmap_free(bitmap: mask->bits);
3022	mask->nbits = 0;
3023	}
3024
3025	static int record__thread_mask_alloc(struct thread_mask *mask, int nr_bits)
3026	{
3027	int ret;
3028
3029	ret = record__mmap_cpu_mask_alloc(mask: &mask->maps, nr_bits);
3030	if (ret) {
3031	mask->affinity.bits = NULL;
3032	return ret;
3033	}
3034
3035	ret = record__mmap_cpu_mask_alloc(mask: &mask->affinity, nr_bits);
3036	if (ret) {
3037	record__mmap_cpu_mask_free(mask: &mask->maps);
3038	mask->maps.bits = NULL;
3039	}
3040
3041	return ret;
3042	}
3043
3044	static void record__thread_mask_free(struct thread_mask *mask)
3045	{
3046	record__mmap_cpu_mask_free(mask: &mask->maps);
3047	record__mmap_cpu_mask_free(mask: &mask->affinity);
3048	}
3049
3050	static int record__parse_threads(const struct option *opt, const char *str, int unset)
3051	{
3052	int s;
3053	struct record_opts *opts = opt->value;
3054
3055	if (unset || !str || !strlen(str)) {
3056	opts->threads_spec = THREAD_SPEC__CPU;
3057	} else {
3058	for (s = 1; s < THREAD_SPEC__MAX; s++) {
3059	if (s == THREAD_SPEC__USER) {
3060	opts->threads_user_spec = strdup(str);
3061	if (!opts->threads_user_spec)
3062	return -ENOMEM;
3063	opts->threads_spec = THREAD_SPEC__USER;
3064	break;
3065	}
3066	if (!strncasecmp(s1: str, s2: thread_spec_tags[s], strlen(thread_spec_tags[s]))) {
3067	opts->threads_spec = s;
3068	break;
3069	}
3070	}
3071	}
3072
3073	if (opts->threads_spec == THREAD_SPEC__USER)
3074	pr_debug("threads_spec: %s\n", opts->threads_user_spec);
3075	else
3076	pr_debug("threads_spec: %s\n", thread_spec_tags[opts->threads_spec]);
3077
3078	return 0;
3079	}
3080
3081	static int parse_output_max_size(const struct option *opt,
3082	const char *str, int unset)
3083	{
3084	unsigned long *s = (unsigned long *)opt->value;
3085	static struct parse_tag tags_size[] = {
3086	{ .tag = 'B', .mult = 1 },
3087	{ .tag = 'K', .mult = 1 << 10 },
3088	{ .tag = 'M', .mult = 1 << 20 },
3089	{ .tag = 'G', .mult = 1 << 30 },
3090	{ .tag = 0 },
3091	};
3092	unsigned long val;
3093
3094	if (unset) {
3095	*s = 0;
3096	return 0;
3097	}
3098
3099	val = parse_tag_value(str, tags: tags_size);
3100	if (val != (unsigned long) -1) {
3101	*s = val;
3102	return 0;
3103	}
3104
3105	return -1;
3106	}
3107
3108	static int record__parse_mmap_pages(const struct option *opt,
3109	const char *str,
3110	int unset __maybe_unused)
3111	{
3112	struct record_opts *opts = opt->value;
3113	char *s, *p;
3114	unsigned int mmap_pages;
3115	int ret;
3116
3117	if (!str)
3118	return -EINVAL;
3119
3120	s = strdup(str);
3121	if (!s)
3122	return -ENOMEM;
3123
3124	p = strchr(s, ',');
3125	if (p)
3126	*p = '\0';
3127
3128	if (*s) {
3129	ret = __evlist__parse_mmap_pages(mmap_pages: &mmap_pages, str: s);
3130	if (ret)
3131	goto out_free;
3132	opts->mmap_pages = mmap_pages;
3133	}
3134
3135	if (!p) {
3136	ret = 0;
3137	goto out_free;
3138	}
3139
3140	ret = __evlist__parse_mmap_pages(mmap_pages: &mmap_pages, str: p + 1);
3141	if (ret)
3142	goto out_free;
3143
3144	opts->auxtrace_mmap_pages = mmap_pages;
3145
3146	out_free:
3147	free(s);
3148	return ret;
3149	}
3150
3151	void __weak arch__add_leaf_frame_record_opts(struct record_opts *opts __maybe_unused)
3152	{
3153	}
3154
3155	static int parse_control_option(const struct option *opt,
3156	const char *str,
3157	int unset __maybe_unused)
3158	{
3159	struct record_opts *opts = opt->value;
3160
3161	return evlist__parse_control(str, ctl_fd: &opts->ctl_fd, ctl_fd_ack: &opts->ctl_fd_ack, ctl_fd_close: &opts->ctl_fd_close);
3162	}
3163
3164	static void switch_output_size_warn(struct record *rec)
3165	{
3166	u64 wakeup_size = evlist__mmap_size(pages: rec->opts.mmap_pages);
3167	struct switch_output *s = &rec->switch_output;
3168
3169	wakeup_size /= 2;
3170
3171	if (s->size < wakeup_size) {
3172	char buf[100];
3173
3174	unit_number__scnprintf(buf, size: sizeof(buf), n: wakeup_size);
3175	pr_warning("WARNING: switch-output data size lower than "
3176	"wakeup kernel buffer size (%s) "
3177	"expect bigger perf.data sizes\n", buf);
3178	}
3179	}
3180
3181	static int switch_output_setup(struct record *rec)
3182	{
3183	struct switch_output *s = &rec->switch_output;
3184	static struct parse_tag tags_size[] = {
3185	{ .tag = 'B', .mult = 1 },
3186	{ .tag = 'K', .mult = 1 << 10 },
3187	{ .tag = 'M', .mult = 1 << 20 },
3188	{ .tag = 'G', .mult = 1 << 30 },
3189	{ .tag = 0 },
3190	};
3191	static struct parse_tag tags_time[] = {
3192	{ .tag = 's', .mult = 1 },
3193	{ .tag = 'm', .mult = 60 },
3194	{ .tag = 'h', .mult = 60*60 },
3195	{ .tag = 'd', .mult = 60*60*24 },
3196	{ .tag = 0 },
3197	};
3198	unsigned long val;
3199
3200	/*
3201	* If we're using --switch-output-events, then we imply its
3202	* --switch-output=signal, as we'll send a SIGUSR2 from the side band
3203	* thread to its parent.
3204	*/
3205	if (rec->switch_output_event_set) {
3206	if (record__threads_enabled(rec)) {
3207	pr_warning("WARNING: --switch-output-event option is not available in parallel streaming mode.\n");
3208	return 0;
3209	}
3210	goto do_signal;
3211	}
3212
3213	if (!s->set)
3214	return 0;
3215
3216	if (record__threads_enabled(rec)) {
3217	pr_warning("WARNING: --switch-output option is not available in parallel streaming mode.\n");
3218	return 0;
3219	}
3220
3221	if (!strcmp(s->str, "signal")) {
3222	do_signal:
3223	s->signal = true;
3224	pr_debug("switch-output with SIGUSR2 signal\n");
3225	goto enabled;
3226	}
3227
3228	val = parse_tag_value(str: s->str, tags: tags_size);
3229	if (val != (unsigned long) -1) {
3230	s->size = val;
3231	pr_debug("switch-output with %s size threshold\n", s->str);
3232	goto enabled;
3233	}
3234
3235	val = parse_tag_value(str: s->str, tags: tags_time);
3236	if (val != (unsigned long) -1) {
3237	s->time = val;
3238	pr_debug("switch-output with %s time threshold (%lu seconds)\n",
3239	s->str, s->time);
3240	goto enabled;
3241	}
3242
3243	return -1;
3244
3245	enabled:
3246	rec->timestamp_filename = true;
3247	s->enabled = true;
3248
3249	if (s->size && !rec->opts.no_buffering)
3250	switch_output_size_warn(rec);
3251
3252	return 0;
3253	}
3254
3255	static const char * const __record_usage[] = {
3256	"perf record [<options>] [<command>]",
3257	"perf record [<options>] -- <command> [<options>]",
3258	NULL
3259	};
3260	const char * const *record_usage = __record_usage;
3261
3262	static int build_id__process_mmap(struct perf_tool *tool, union perf_event *event,
3263	struct perf_sample *sample, struct machine *machine)
3264	{
3265	/*
3266	* We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3267	* no need to add them twice.
3268	*/
3269	if (!(event->header.misc & PERF_RECORD_MISC_USER))
3270	return 0;
3271	return perf_event__process_mmap(tool, event, sample, machine);
3272	}
3273
3274	static int build_id__process_mmap2(struct perf_tool *tool, union perf_event *event,
3275	struct perf_sample *sample, struct machine *machine)
3276	{
3277	/*
3278	* We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3279	* no need to add them twice.
3280	*/
3281	if (!(event->header.misc & PERF_RECORD_MISC_USER))
3282	return 0;
3283
3284	return perf_event__process_mmap2(tool, event, sample, machine);
3285	}
3286
3287	static int process_timestamp_boundary(struct perf_tool *tool,
3288	union perf_event *event __maybe_unused,
3289	struct perf_sample *sample,
3290	struct machine *machine __maybe_unused)
3291	{
3292	struct record *rec = container_of(tool, struct record, tool);
3293
3294	set_timestamp_boundary(rec, sample_time: sample->time);
3295	return 0;
3296	}
3297
3298	static int parse_record_synth_option(const struct option *opt,
3299	const char *str,
3300	int unset __maybe_unused)
3301	{
3302	struct record_opts *opts = opt->value;
3303	char *p = strdup(str);
3304
3305	if (p == NULL)
3306	return -1;
3307
3308	opts->synth = parse_synth_opt(str: p);
3309	free(p);
3310
3311	if (opts->synth < 0) {
3312	pr_err("Invalid synth option: %s\n", str);
3313	return -1;
3314	}
3315	return 0;
3316	}
3317
3318	/*
3319	* XXX Ideally would be local to cmd_record() and passed to a record__new
3320	* because we need to have access to it in record__exit, that is called
3321	* after cmd_record() exits, but since record_options need to be accessible to
3322	* builtin-script, leave it here.
3323	*
3324	* At least we don't ouch it in all the other functions here directly.
3325	*
3326	* Just say no to tons of global variables, sigh.
3327	*/
3328	static struct record record = {
3329	.opts = {
3330	.sample_time = true,
3331	.mmap_pages = UINT_MAX,
3332	.user_freq = UINT_MAX,
3333	.user_interval = ULLONG_MAX,
3334	.freq = 4000,
3335	.target = {
3336	.uses_mmap = true,
3337	.default_per_cpu = true,
3338	},
3339	.mmap_flush = MMAP_FLUSH_DEFAULT,
3340	.nr_threads_synthesize = 1,
3341	.ctl_fd = -1,
3342	.ctl_fd_ack = -1,
3343	.synth = PERF_SYNTH_ALL,
3344	},
3345	.tool = {
3346	.sample = process_sample_event,
3347	.fork = perf_event__process_fork,
3348	.exit = perf_event__process_exit,
3349	.comm = perf_event__process_comm,
3350	.namespaces = perf_event__process_namespaces,
3351	.mmap = build_id__process_mmap,
3352	.mmap2 = build_id__process_mmap2,
3353	.itrace_start = process_timestamp_boundary,
3354	.aux = process_timestamp_boundary,
3355	.ordered_events = true,
3356	},
3357	};
3358
3359	const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
3360	"\n\t\t\t\tDefault: fp";
3361
3362	static bool dry_run;
3363
3364	static struct parse_events_option_args parse_events_option_args = {
3365	.evlistp = &record.evlist,
3366	};
3367
3368	static struct parse_events_option_args switch_output_parse_events_option_args = {
3369	.evlistp = &record.sb_evlist,
3370	};
3371
3372	/*
3373	* XXX Will stay a global variable till we fix builtin-script.c to stop messing
3374	* with it and switch to use the library functions in perf_evlist that came
3375	* from builtin-record.c, i.e. use record_opts,
3376	* evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
3377	* using pipes, etc.
3378	*/
3379	static struct option __record_options[] = {
3380	OPT_CALLBACK('e', "event", &parse_events_option_args, "event",
3381	"event selector. use 'perf list' to list available events",
3382	parse_events_option),
3383	OPT_CALLBACK(0, "filter", &record.evlist, "filter",
3384	"event filter", parse_filter),
3385	OPT_CALLBACK_NOOPT(0, "exclude-perf", &record.evlist,
3386	NULL, "don't record events from perf itself",
3387	exclude_perf),
3388	OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
3389	"record events on existing process id"),
3390	OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
3391	"record events on existing thread id"),
3392	OPT_INTEGER('r', "realtime", &record.realtime_prio,
3393	"collect data with this RT SCHED_FIFO priority"),
3394	OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
3395	"collect data without buffering"),
3396	OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
3397	"collect raw sample records from all opened counters"),
3398	OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
3399	"system-wide collection from all CPUs"),
3400	OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
3401	"list of cpus to monitor"),
3402	OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
3403	OPT_STRING('o', "output", &record.data.path, "file",
3404	"output file name"),
3405	OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
3406	&record.opts.no_inherit_set,
3407	"child tasks do not inherit counters"),
3408	OPT_BOOLEAN(0, "tail-synthesize", &record.opts.tail_synthesize,
3409	"synthesize non-sample events at the end of output"),
3410	OPT_BOOLEAN(0, "overwrite", &record.opts.overwrite, "use overwrite mode"),
3411	OPT_BOOLEAN(0, "no-bpf-event", &record.opts.no_bpf_event, "do not record bpf events"),
3412	OPT_BOOLEAN(0, "strict-freq", &record.opts.strict_freq,
3413	"Fail if the specified frequency can't be used"),
3414	OPT_CALLBACK('F', "freq", &record.opts, "freq or 'max'",
3415	"profile at this frequency",
3416	record__parse_freq),
3417	OPT_CALLBACK('m', "mmap-pages", &record.opts, "pages[,pages]",
3418	"number of mmap data pages and AUX area tracing mmap pages",
3419	record__parse_mmap_pages),
3420	OPT_CALLBACK(0, "mmap-flush", &record.opts, "number",
3421	"Minimal number of bytes that is extracted from mmap data pages (default: 1)",
3422	record__mmap_flush_parse),
3423	OPT_CALLBACK_NOOPT('g', NULL, &callchain_param,
3424	NULL, "enables call-graph recording" ,
3425	&record_callchain_opt),
3426	OPT_CALLBACK(0, "call-graph", &record.opts,
3427	"record_mode[,record_size]", record_callchain_help,
3428	&record_parse_callchain_opt),
3429	OPT_INCR('v', "verbose", &verbose,
3430	"be more verbose (show counter open errors, etc)"),
3431	OPT_BOOLEAN('q', "quiet", &quiet, "don't print any warnings or messages"),
3432	OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
3433	"per thread counts"),
3434	OPT_BOOLEAN('d', "data", &record.opts.sample_address, "Record the sample addresses"),
3435	OPT_BOOLEAN(0, "phys-data", &record.opts.sample_phys_addr,
3436	"Record the sample physical addresses"),
3437	OPT_BOOLEAN(0, "data-page-size", &record.opts.sample_data_page_size,
3438	"Record the sampled data address data page size"),
3439	OPT_BOOLEAN(0, "code-page-size", &record.opts.sample_code_page_size,
3440	"Record the sampled code address (ip) page size"),
3441	OPT_BOOLEAN(0, "sample-cpu", &record.opts.sample_cpu, "Record the sample cpu"),
3442	OPT_BOOLEAN(0, "sample-identifier", &record.opts.sample_identifier,
3443	"Record the sample identifier"),
3444	OPT_BOOLEAN_SET('T', "timestamp", &record.opts.sample_time,
3445	&record.opts.sample_time_set,
3446	"Record the sample timestamps"),
3447	OPT_BOOLEAN_SET('P', "period", &record.opts.period, &record.opts.period_set,
3448	"Record the sample period"),
3449	OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
3450	"don't sample"),
3451	OPT_BOOLEAN_SET('N', "no-buildid-cache", &record.no_buildid_cache,
3452	&record.no_buildid_cache_set,
3453	"do not update the buildid cache"),
3454	OPT_BOOLEAN_SET('B', "no-buildid", &record.no_buildid,
3455	&record.no_buildid_set,
3456	"do not collect buildids in perf.data"),
3457	OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
3458	"monitor event in cgroup name only",
3459	parse_cgroups),
3460	OPT_CALLBACK('D', "delay", &record, "ms",
3461	"ms to wait before starting measurement after program start (-1: start with events disabled), "
3462	"or ranges of time to enable events e.g. '-D 10-20,30-40'",
3463	record__parse_event_enable_time),
3464	OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
3465	OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
3466	"user to profile"),
3467
3468	OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
3469	"branch any", "sample any taken branches",
3470	parse_branch_stack),
3471
3472	OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
3473	"branch filter mask", "branch stack filter modes",
3474	parse_branch_stack),
3475	OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
3476	"sample by weight (on special events only)"),
3477	OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
3478	"sample transaction flags (special events only)"),
3479	OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
3480	"use per-thread mmaps"),
3481	OPT_CALLBACK_OPTARG('I', "intr-regs", &record.opts.sample_intr_regs, NULL, "any register",
3482	"sample selected machine registers on interrupt,"
3483	" use '-I?' to list register names", parse_intr_regs),
3484	OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register",
3485	"sample selected machine registers on interrupt,"
3486	" use '--user-regs=?' to list register names", parse_user_regs),
3487	OPT_BOOLEAN(0, "running-time", &record.opts.running_time,
3488	"Record running/enabled time of read (:S) events"),
3489	OPT_CALLBACK('k', "clockid", &record.opts,
3490	"clockid", "clockid to use for events, see clock_gettime()",
3491	parse_clockid),
3492	OPT_STRING_OPTARG('S', "snapshot", &record.opts.auxtrace_snapshot_opts,
3493	"opts", "AUX area tracing Snapshot Mode", ""),
3494	OPT_STRING_OPTARG(0, "aux-sample", &record.opts.auxtrace_sample_opts,
3495	"opts", "sample AUX area", ""),
3496	OPT_UINTEGER(0, "proc-map-timeout", &proc_map_timeout,
3497	"per thread proc mmap processing timeout in ms"),
3498	OPT_BOOLEAN(0, "namespaces", &record.opts.record_namespaces,
3499	"Record namespaces events"),
3500	OPT_BOOLEAN(0, "all-cgroups", &record.opts.record_cgroup,
3501	"Record cgroup events"),
3502	OPT_BOOLEAN_SET(0, "switch-events", &record.opts.record_switch_events,
3503	&record.opts.record_switch_events_set,
3504	"Record context switch events"),
3505	OPT_BOOLEAN_FLAG(0, "all-kernel", &record.opts.all_kernel,
3506	"Configure all used events to run in kernel space.",
3507	PARSE_OPT_EXCLUSIVE),
3508	OPT_BOOLEAN_FLAG(0, "all-user", &record.opts.all_user,
3509	"Configure all used events to run in user space.",
3510	PARSE_OPT_EXCLUSIVE),
3511	OPT_BOOLEAN(0, "kernel-callchains", &record.opts.kernel_callchains,
3512	"collect kernel callchains"),
3513	OPT_BOOLEAN(0, "user-callchains", &record.opts.user_callchains,
3514	"collect user callchains"),
3515	OPT_STRING(0, "vmlinux", &symbol_conf.vmlinux_name,
3516	"file", "vmlinux pathname"),
3517	OPT_BOOLEAN(0, "buildid-all", &record.buildid_all,
3518	"Record build-id of all DSOs regardless of hits"),
3519	OPT_BOOLEAN(0, "buildid-mmap", &record.buildid_mmap,
3520	"Record build-id in map events"),
3521	OPT_BOOLEAN(0, "timestamp-filename", &record.timestamp_filename,
3522	"append timestamp to output filename"),
3523	OPT_BOOLEAN(0, "timestamp-boundary", &record.timestamp_boundary,
3524	"Record timestamp boundary (time of first/last samples)"),
3525	OPT_STRING_OPTARG_SET(0, "switch-output", &record.switch_output.str,
3526	&record.switch_output.set, "signal or size[BKMG] or time[smhd]",
3527	"Switch output when receiving SIGUSR2 (signal) or cross a size or time threshold",
3528	"signal"),
3529	OPT_CALLBACK_SET(0, "switch-output-event", &switch_output_parse_events_option_args,
3530	&record.switch_output_event_set, "switch output event",
3531	"switch output event selector. use 'perf list' to list available events",
3532	parse_events_option_new_evlist),
3533	OPT_INTEGER(0, "switch-max-files", &record.switch_output.num_files,
3534	"Limit number of switch output generated files"),
3535	OPT_BOOLEAN(0, "dry-run", &dry_run,
3536	"Parse options then exit"),
3537	#ifdef HAVE_AIO_SUPPORT
3538	OPT_CALLBACK_OPTARG(0, "aio", &record.opts,
3539	&nr_cblocks_default, "n", "Use <n> control blocks in asynchronous trace writing mode (default: 1, max: 4)",
3540	record__aio_parse),
3541	#endif
3542	OPT_CALLBACK(0, "affinity", &record.opts, "node|cpu",
3543	"Set affinity mask of trace reading thread to NUMA node cpu mask or cpu of processed mmap buffer",
3544	record__parse_affinity),
3545	#ifdef HAVE_ZSTD_SUPPORT
3546	OPT_CALLBACK_OPTARG('z', "compression-level", &record.opts, &comp_level_default, "n",
3547	"Compress records using specified level (default: 1 - fastest compression, 22 - greatest compression)",
3548	record__parse_comp_level),
3549	#endif
3550	OPT_CALLBACK(0, "max-size", &record.output_max_size,
3551	"size", "Limit the maximum size of the output file", parse_output_max_size),
3552	OPT_UINTEGER(0, "num-thread-synthesize",
3553	&record.opts.nr_threads_synthesize,
3554	"number of threads to run for event synthesis"),
3555	#ifdef HAVE_LIBPFM
3556	OPT_CALLBACK(0, "pfm-events", &record.evlist, "event",
3557	"libpfm4 event selector. use 'perf list' to list available events",
3558	parse_libpfm_events_option),
3559	#endif
3560	OPT_CALLBACK(0, "control", &record.opts, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
3561	"Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events,\n"
3562	"\t\t\t 'snapshot': AUX area tracing snapshot).\n"
3563	"\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
3564	"\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
3565	parse_control_option),
3566	OPT_CALLBACK(0, "synth", &record.opts, "no|all|task|mmap|cgroup",
3567	"Fine-tune event synthesis: default=all", parse_record_synth_option),
3568	OPT_STRING_OPTARG_SET(0, "debuginfod", &record.debuginfod.urls,
3569	&record.debuginfod.set, "debuginfod urls",
3570	"Enable debuginfod data retrieval from DEBUGINFOD_URLS or specified urls",
3571	"system"),
3572	OPT_CALLBACK_OPTARG(0, "threads", &record.opts, NULL, "spec",
3573	"write collected trace data into several data files using parallel threads",
3574	record__parse_threads),
3575	OPT_BOOLEAN(0, "off-cpu", &record.off_cpu, "Enable off-cpu analysis"),
3576	OPT_END()
3577	};
3578
3579	struct option *record_options = __record_options;
3580
3581	static int record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus)
3582	{
3583	struct perf_cpu cpu;
3584	int idx;
3585
3586	if (cpu_map__is_dummy(cpus))
3587	return 0;
3588
3589	perf_cpu_map__for_each_cpu_skip_any(cpu, idx, cpus) {
3590	/* Return ENODEV is input cpu is greater than max cpu */
3591	if ((unsigned long)cpu.cpu > mask->nbits)
3592	return -ENODEV;
3593	__set_bit(cpu.cpu, mask->bits);
3594	}
3595
3596	return 0;
3597	}
3598
3599	static int record__mmap_cpu_mask_init_spec(struct mmap_cpu_mask *mask, const char *mask_spec)
3600	{
3601	struct perf_cpu_map *cpus;
3602
3603	cpus = perf_cpu_map__new(mask_spec);
3604	if (!cpus)
3605	return -ENOMEM;
3606
3607	bitmap_zero(dst: mask->bits, nbits: mask->nbits);
3608	if (record__mmap_cpu_mask_init(mask, cpus))
3609	return -ENODEV;
3610
3611	perf_cpu_map__put(cpus);
3612
3613	return 0;
3614	}
3615
3616	static void record__free_thread_masks(struct record *rec, int nr_threads)
3617	{
3618	int t;
3619
3620	if (rec->thread_masks)
3621	for (t = 0; t < nr_threads; t++)
3622	record__thread_mask_free(mask: &rec->thread_masks[t]);
3623
3624	zfree(&rec->thread_masks);
3625	}
3626
3627	static int record__alloc_thread_masks(struct record *rec, int nr_threads, int nr_bits)
3628	{
3629	int t, ret;
3630
3631	rec->thread_masks = zalloc(nr_threads * sizeof(*(rec->thread_masks)));
3632	if (!rec->thread_masks) {
3633	pr_err("Failed to allocate thread masks\n");
3634	return -ENOMEM;
3635	}
3636
3637	for (t = 0; t < nr_threads; t++) {
3638	ret = record__thread_mask_alloc(mask: &rec->thread_masks[t], nr_bits);
3639	if (ret) {
3640	pr_err("Failed to allocate thread masks[%d]\n", t);
3641	goto out_free;
3642	}
3643	}
3644
3645	return 0;
3646
3647	out_free:
3648	record__free_thread_masks(rec, nr_threads);
3649
3650	return ret;
3651	}
3652
3653	static int record__init_thread_cpu_masks(struct record *rec, struct perf_cpu_map *cpus)
3654	{
3655	int t, ret, nr_cpus = perf_cpu_map__nr(cpus);
3656
3657	ret = record__alloc_thread_masks(rec, nr_threads: nr_cpus, nr_bits: cpu__max_cpu().cpu);
3658	if (ret)
3659	return ret;
3660
3661	rec->nr_threads = nr_cpus;
3662	pr_debug("nr_threads: %d\n", rec->nr_threads);
3663
3664	for (t = 0; t < rec->nr_threads; t++) {
3665	__set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].maps.bits);
3666	__set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].affinity.bits);
3667	if (verbose > 0) {
3668	pr_debug("thread_masks[%d]: ", t);
3669	mmap_cpu_mask__scnprintf(mask: &rec->thread_masks[t].maps, tag: "maps");
3670	pr_debug("thread_masks[%d]: ", t);
3671	mmap_cpu_mask__scnprintf(mask: &rec->thread_masks[t].affinity, tag: "affinity");
3672	}
3673	}
3674
3675	return 0;
3676	}
3677
3678	static int record__init_thread_masks_spec(struct record *rec, struct perf_cpu_map *cpus,
3679	const char **maps_spec, const char **affinity_spec,
3680	u32 nr_spec)
3681	{
3682	u32 s;
3683	int ret = 0, t = 0;
3684	struct mmap_cpu_mask cpus_mask;
3685	struct thread_mask thread_mask, full_mask, *thread_masks;
3686
3687	ret = record__mmap_cpu_mask_alloc(mask: &cpus_mask, nr_bits: cpu__max_cpu().cpu);
3688	if (ret) {
3689	pr_err("Failed to allocate CPUs mask\n");
3690	return ret;
3691	}
3692
3693	ret = record__mmap_cpu_mask_init(mask: &cpus_mask, cpus);
3694	if (ret) {
3695	pr_err("Failed to init cpu mask\n");
3696	goto out_free_cpu_mask;
3697	}
3698
3699	ret = record__thread_mask_alloc(mask: &full_mask, nr_bits: cpu__max_cpu().cpu);
3700	if (ret) {
3701	pr_err("Failed to allocate full mask\n");
3702	goto out_free_cpu_mask;
3703	}
3704
3705	ret = record__thread_mask_alloc(mask: &thread_mask, nr_bits: cpu__max_cpu().cpu);
3706	if (ret) {
3707	pr_err("Failed to allocate thread mask\n");
3708	goto out_free_full_and_cpu_masks;
3709	}
3710
3711	for (s = 0; s < nr_spec; s++) {
3712	ret = record__mmap_cpu_mask_init_spec(mask: &thread_mask.maps, mask_spec: maps_spec[s]);
3713	if (ret) {
3714	pr_err("Failed to initialize maps thread mask\n");
3715	goto out_free;
3716	}
3717	ret = record__mmap_cpu_mask_init_spec(mask: &thread_mask.affinity, mask_spec: affinity_spec[s]);
3718	if (ret) {
3719	pr_err("Failed to initialize affinity thread mask\n");
3720	goto out_free;
3721	}
3722
3723	/* ignore invalid CPUs but do not allow empty masks */
3724	if (!bitmap_and(dst: thread_mask.maps.bits, src1: thread_mask.maps.bits,
3725	src2: cpus_mask.bits, nbits: thread_mask.maps.nbits)) {
3726	pr_err("Empty maps mask: %s\n", maps_spec[s]);
3727	ret = -EINVAL;
3728	goto out_free;
3729	}
3730	if (!bitmap_and(dst: thread_mask.affinity.bits, src1: thread_mask.affinity.bits,
3731	src2: cpus_mask.bits, nbits: thread_mask.affinity.nbits)) {
3732	pr_err("Empty affinity mask: %s\n", affinity_spec[s]);
3733	ret = -EINVAL;
3734	goto out_free;
3735	}
3736
3737	/* do not allow intersection with other masks (full_mask) */
3738	if (bitmap_intersects(src1: thread_mask.maps.bits, src2: full_mask.maps.bits,
3739	nbits: thread_mask.maps.nbits)) {
3740	pr_err("Intersecting maps mask: %s\n", maps_spec[s]);
3741	ret = -EINVAL;
3742	goto out_free;
3743	}
3744	if (bitmap_intersects(src1: thread_mask.affinity.bits, src2: full_mask.affinity.bits,
3745	nbits: thread_mask.affinity.nbits)) {
3746	pr_err("Intersecting affinity mask: %s\n", affinity_spec[s]);
3747	ret = -EINVAL;
3748	goto out_free;
3749	}
3750
3751	bitmap_or(dst: full_mask.maps.bits, src1: full_mask.maps.bits,
3752	src2: thread_mask.maps.bits, nbits: full_mask.maps.nbits);
3753	bitmap_or(dst: full_mask.affinity.bits, src1: full_mask.affinity.bits,
3754	src2: thread_mask.affinity.bits, nbits: full_mask.maps.nbits);
3755
3756	thread_masks = realloc(rec->thread_masks, (t + 1) * sizeof(struct thread_mask));
3757	if (!thread_masks) {
3758	pr_err("Failed to reallocate thread masks\n");
3759	ret = -ENOMEM;
3760	goto out_free;
3761	}
3762	rec->thread_masks = thread_masks;
3763	rec->thread_masks[t] = thread_mask;
3764	if (verbose > 0) {
3765	pr_debug("thread_masks[%d]: ", t);
3766	mmap_cpu_mask__scnprintf(mask: &rec->thread_masks[t].maps, tag: "maps");
3767	pr_debug("thread_masks[%d]: ", t);
3768	mmap_cpu_mask__scnprintf(mask: &rec->thread_masks[t].affinity, tag: "affinity");
3769	}
3770	t++;
3771	ret = record__thread_mask_alloc(mask: &thread_mask, nr_bits: cpu__max_cpu().cpu);
3772	if (ret) {
3773	pr_err("Failed to allocate thread mask\n");
3774	goto out_free_full_and_cpu_masks;
3775	}
3776	}
3777	rec->nr_threads = t;
3778	pr_debug("nr_threads: %d\n", rec->nr_threads);
3779	if (!rec->nr_threads)
3780	ret = -EINVAL;
3781
3782	out_free:
3783	record__thread_mask_free(mask: &thread_mask);
3784	out_free_full_and_cpu_masks:
3785	record__thread_mask_free(mask: &full_mask);
3786	out_free_cpu_mask:
3787	record__mmap_cpu_mask_free(mask: &cpus_mask);
3788
3789	return ret;
3790	}
3791
3792	static int record__init_thread_core_masks(struct record *rec, struct perf_cpu_map *cpus)
3793	{
3794	int ret;
3795	struct cpu_topology *topo;
3796
3797	topo = cpu_topology__new();
3798	if (!topo) {
3799	pr_err("Failed to allocate CPU topology\n");
3800	return -ENOMEM;
3801	}
3802
3803	ret = record__init_thread_masks_spec(rec, cpus, maps_spec: topo->core_cpus_list,
3804	affinity_spec: topo->core_cpus_list, nr_spec: topo->core_cpus_lists);
3805	cpu_topology__delete(topo);
3806
3807	return ret;
3808	}
3809
3810	static int record__init_thread_package_masks(struct record *rec, struct perf_cpu_map *cpus)
3811	{
3812	int ret;
3813	struct cpu_topology *topo;
3814
3815	topo = cpu_topology__new();
3816	if (!topo) {
3817	pr_err("Failed to allocate CPU topology\n");
3818	return -ENOMEM;
3819	}
3820
3821	ret = record__init_thread_masks_spec(rec, cpus, maps_spec: topo->package_cpus_list,
3822	affinity_spec: topo->package_cpus_list, nr_spec: topo->package_cpus_lists);
3823	cpu_topology__delete(topo);
3824
3825	return ret;
3826	}
3827
3828	static int record__init_thread_numa_masks(struct record *rec, struct perf_cpu_map *cpus)
3829	{
3830	u32 s;
3831	int ret;
3832	const char **spec;
3833	struct numa_topology *topo;
3834
3835	topo = numa_topology__new();
3836	if (!topo) {
3837	pr_err("Failed to allocate NUMA topology\n");
3838	return -ENOMEM;
3839	}
3840
3841	spec = zalloc(topo->nr * sizeof(char *));
3842	if (!spec) {
3843	pr_err("Failed to allocate NUMA spec\n");
3844	ret = -ENOMEM;
3845	goto out_delete_topo;
3846	}
3847	for (s = 0; s < topo->nr; s++)
3848	spec[s] = topo->nodes[s].cpus;
3849
3850	ret = record__init_thread_masks_spec(rec, cpus, maps_spec: spec, affinity_spec: spec, nr_spec: topo->nr);
3851
3852	zfree(&spec);
3853
3854	out_delete_topo:
3855	numa_topology__delete(topo);
3856
3857	return ret;
3858	}
3859
3860	static int record__init_thread_user_masks(struct record *rec, struct perf_cpu_map *cpus)
3861	{
3862	int t, ret;
3863	u32 s, nr_spec = 0;
3864	char **maps_spec = NULL, **affinity_spec = NULL, **tmp_spec;
3865	char *user_spec, *spec, *spec_ptr, *mask, *mask_ptr, *dup_mask = NULL;
3866
3867	for (t = 0, user_spec = (char *)rec->opts.threads_user_spec; ; t++, user_spec = NULL) {
3868	spec = strtok_r(user_spec, ":", &spec_ptr);
3869	if (spec == NULL)
3870	break;
3871	pr_debug2("threads_spec[%d]: %s\n", t, spec);
3872	mask = strtok_r(spec, "/", &mask_ptr);
3873	if (mask == NULL)
3874	break;
3875	pr_debug2(" maps mask: %s\n", mask);
3876	tmp_spec = realloc(maps_spec, (nr_spec + 1) * sizeof(char *));
3877	if (!tmp_spec) {
3878	pr_err("Failed to reallocate maps spec\n");
3879	ret = -ENOMEM;
3880	goto out_free;
3881	}
3882	maps_spec = tmp_spec;
3883	maps_spec[nr_spec] = dup_mask = strdup(mask);
3884	if (!maps_spec[nr_spec]) {
3885	pr_err("Failed to allocate maps spec[%d]\n", nr_spec);
3886	ret = -ENOMEM;
3887	goto out_free;
3888	}
3889	mask = strtok_r(NULL, "/", &mask_ptr);
3890	if (mask == NULL) {
3891	pr_err("Invalid thread maps or affinity specs\n");
3892	ret = -EINVAL;
3893	goto out_free;
3894	}
3895	pr_debug2(" affinity mask: %s\n", mask);
3896	tmp_spec = realloc(affinity_spec, (nr_spec + 1) * sizeof(char *));
3897	if (!tmp_spec) {
3898	pr_err("Failed to reallocate affinity spec\n");
3899	ret = -ENOMEM;
3900	goto out_free;
3901	}
3902	affinity_spec = tmp_spec;
3903	affinity_spec[nr_spec] = strdup(mask);
3904	if (!affinity_spec[nr_spec]) {
3905	pr_err("Failed to allocate affinity spec[%d]\n", nr_spec);
3906	ret = -ENOMEM;
3907	goto out_free;
3908	}
3909	dup_mask = NULL;
3910	nr_spec++;
3911	}
3912
3913	ret = record__init_thread_masks_spec(rec, cpus, maps_spec: (const char **)maps_spec,
3914	affinity_spec: (const char **)affinity_spec, nr_spec);
3915
3916	out_free:
3917	free(dup_mask);
3918	for (s = 0; s < nr_spec; s++) {
3919	if (maps_spec)
3920	free(maps_spec[s]);
3921	if (affinity_spec)
3922	free(affinity_spec[s]);
3923	}
3924	free(affinity_spec);
3925	free(maps_spec);
3926
3927	return ret;
3928	}
3929
3930	static int record__init_thread_default_masks(struct record *rec, struct perf_cpu_map *cpus)
3931	{
3932	int ret;
3933
3934	ret = record__alloc_thread_masks(rec, nr_threads: 1, nr_bits: cpu__max_cpu().cpu);
3935	if (ret)
3936	return ret;
3937
3938	if (record__mmap_cpu_mask_init(mask: &rec->thread_masks->maps, cpus))
3939	return -ENODEV;
3940
3941	rec->nr_threads = 1;
3942
3943	return 0;
3944	}
3945
3946	static int record__init_thread_masks(struct record *rec)
3947	{
3948	int ret = 0;
3949	struct perf_cpu_map *cpus = rec->evlist->core.all_cpus;
3950
3951	if (!record__threads_enabled(rec))
3952	return record__init_thread_default_masks(rec, cpus);
3953
3954	if (evlist__per_thread(evlist: rec->evlist)) {
3955	pr_err("--per-thread option is mutually exclusive to parallel streaming mode.\n");
3956	return -EINVAL;
3957	}
3958
3959	switch (rec->opts.threads_spec) {
3960	case THREAD_SPEC__CPU:
3961	ret = record__init_thread_cpu_masks(rec, cpus);
3962	break;
3963	case THREAD_SPEC__CORE:
3964	ret = record__init_thread_core_masks(rec, cpus);
3965	break;
3966	case THREAD_SPEC__PACKAGE:
3967	ret = record__init_thread_package_masks(rec, cpus);
3968	break;
3969	case THREAD_SPEC__NUMA:
3970	ret = record__init_thread_numa_masks(rec, cpus);
3971	break;
3972	case THREAD_SPEC__USER:
3973	ret = record__init_thread_user_masks(rec, cpus);
3974	break;
3975	default:
3976	break;
3977	}
3978
3979	return ret;
3980	}
3981
3982	int cmd_record(int argc, const char **argv)
3983	{
3984	int err;
3985	struct record *rec = &record;
3986	char errbuf[BUFSIZ];
3987
3988	setlocale(LC_ALL, "");
3989
3990	#ifndef HAVE_BPF_SKEL
3991	# define set_nobuild(s, l, m, c) set_option_nobuild(record_options, s, l, m, c)
3992	set_nobuild('\0', "off-cpu", "no BUILD_BPF_SKEL=1", true);
3993	# undef set_nobuild
3994	#endif
3995
3996	/* Disable eager loading of kernel symbols that adds overhead to perf record. */
3997	symbol_conf.lazy_load_kernel_maps = true;
3998	rec->opts.affinity = PERF_AFFINITY_SYS;
3999
4000	rec->evlist = evlist__new();
4001	if (rec->evlist == NULL)
4002	return -ENOMEM;
4003
4004	err = perf_config(fn: perf_record_config, rec);
4005	if (err)
4006	return err;
4007
4008	argc = parse_options(argc, argv, record_options, record_usage,
4009	PARSE_OPT_STOP_AT_NON_OPTION);
4010	if (quiet)
4011	perf_quiet_option();
4012
4013	err = symbol__validate_sym_arguments();
4014	if (err)
4015	return err;
4016
4017	perf_debuginfod_setup(di: &record.debuginfod);
4018
4019	/* Make system wide (-a) the default target. */
4020	if (!argc && target__none(target: &rec->opts.target))
4021	rec->opts.target.system_wide = true;
4022
4023	if (nr_cgroups && !rec->opts.target.system_wide) {
4024	usage_with_options_msg(record_usage, record_options,
4025	"cgroup monitoring only available in system-wide mode");
4026
4027	}
4028
4029	if (rec->buildid_mmap) {
4030	if (!perf_can_record_build_id()) {
4031	pr_err("Failed: no support to record build id in mmap events, update your kernel.\n");
4032	err = -EINVAL;
4033	goto out_opts;
4034	}
4035	pr_debug("Enabling build id in mmap2 events.\n");
4036	/* Enable mmap build id synthesizing. */
4037	symbol_conf.buildid_mmap2 = true;
4038	/* Enable perf_event_attr::build_id bit. */
4039	rec->opts.build_id = true;
4040	/* Disable build id cache. */
4041	rec->no_buildid = true;
4042	}
4043
4044	if (rec->opts.record_cgroup && !perf_can_record_cgroup()) {
4045	pr_err("Kernel has no cgroup sampling support.\n");
4046	err = -EINVAL;
4047	goto out_opts;
4048	}
4049
4050	if (rec->opts.kcore)
4051	rec->opts.text_poke = true;
4052
4053	if (rec->opts.kcore || record__threads_enabled(rec))
4054	rec->data.is_dir = true;
4055
4056	if (record__threads_enabled(rec)) {
4057	if (rec->opts.affinity != PERF_AFFINITY_SYS) {
4058	pr_err("--affinity option is mutually exclusive to parallel streaming mode.\n");
4059	goto out_opts;
4060	}
4061	if (record__aio_enabled(rec)) {
4062	pr_err("Asynchronous streaming mode (--aio) is mutually exclusive to parallel streaming mode.\n");
4063	goto out_opts;
4064	}
4065	}
4066
4067	if (rec->opts.comp_level != 0) {
4068	pr_debug("Compression enabled, disabling build id collection at the end of the session.\n");
4069	rec->no_buildid = true;
4070	}
4071
4072	if (rec->opts.record_switch_events &&
4073	!perf_can_record_switch_events()) {
4074	ui__error(format: "kernel does not support recording context switch events\n");
4075	parse_options_usage(record_usage, record_options, "switch-events", 0);
4076	err = -EINVAL;
4077	goto out_opts;
4078	}
4079
4080	if (switch_output_setup(rec)) {
4081	parse_options_usage(record_usage, record_options, "switch-output", 0);
4082	err = -EINVAL;
4083	goto out_opts;
4084	}
4085
4086	if (rec->switch_output.time) {
4087	signal(SIGALRM, alarm_sig_handler);
4088	alarm(rec->switch_output.time);
4089	}
4090
4091	if (rec->switch_output.num_files) {
4092	rec->switch_output.filenames = calloc(rec->switch_output.num_files,
4093	sizeof(char *));
4094	if (!rec->switch_output.filenames) {
4095	err = -EINVAL;
4096	goto out_opts;
4097	}
4098	}
4099
4100	if (rec->timestamp_filename && record__threads_enabled(rec)) {
4101	rec->timestamp_filename = false;
4102	pr_warning("WARNING: --timestamp-filename option is not available in parallel streaming mode.\n");
4103	}
4104
4105	/*
4106	* Allow aliases to facilitate the lookup of symbols for address
4107	* filters. Refer to auxtrace_parse_filters().
4108	*/
4109	symbol_conf.allow_aliases = true;
4110
4111	symbol__init(NULL);
4112
4113	err = record__auxtrace_init(rec);
4114	if (err)
4115	goto out;
4116
4117	if (dry_run)
4118	goto out;
4119
4120	err = -ENOMEM;
4121
4122	if (rec->no_buildid_cache || rec->no_buildid) {
4123	disable_buildid_cache();
4124	} else if (rec->switch_output.enabled) {
4125	/*
4126	* In 'perf record --switch-output', disable buildid
4127	* generation by default to reduce data file switching
4128	* overhead. Still generate buildid if they are required
4129	* explicitly using
4130	*
4131	* perf record --switch-output --no-no-buildid \
4132	* --no-no-buildid-cache
4133	*
4134	* Following code equals to:
4135	*
4136	* if ((rec->no_buildid || !rec->no_buildid_set) &&
4137	* (rec->no_buildid_cache || !rec->no_buildid_cache_set))
4138	* disable_buildid_cache();
4139	*/
4140	bool disable = true;
4141
4142	if (rec->no_buildid_set && !rec->no_buildid)
4143	disable = false;
4144	if (rec->no_buildid_cache_set && !rec->no_buildid_cache)
4145	disable = false;
4146	if (disable) {
4147	rec->no_buildid = true;
4148	rec->no_buildid_cache = true;
4149	disable_buildid_cache();
4150	}
4151	}
4152
4153	if (record.opts.overwrite)
4154	record.opts.tail_synthesize = true;
4155
4156	if (rec->evlist->core.nr_entries == 0) {
4157	bool can_profile_kernel = perf_event_paranoid_check(max_level: 1);
4158
4159	err = parse_event(evlist: rec->evlist, str: can_profile_kernel ? "cycles:P" : "cycles:Pu");
4160	if (err)
4161	goto out;
4162	}
4163
4164	if (rec->opts.target.tid && !rec->opts.no_inherit_set)
4165	rec->opts.no_inherit = true;
4166
4167	err = target__validate(target: &rec->opts.target);
4168	if (err) {
4169	target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
4170	ui__warning(format: "%s\n", errbuf);
4171	}
4172
4173	err = target__parse_uid(target: &rec->opts.target);
4174	if (err) {
4175	int saved_errno = errno;
4176
4177	target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
4178	ui__error(format: "%s", errbuf);
4179
4180	err = -saved_errno;
4181	goto out;
4182	}
4183
4184	/* Enable ignoring missing threads when -u/-p option is defined. */
4185	rec->opts.ignore_missing_thread = rec->opts.target.uid != UINT_MAX || rec->opts.target.pid;
4186
4187	evlist__warn_user_requested_cpus(evlist: rec->evlist, cpu_list: rec->opts.target.cpu_list);
4188
4189	if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP)
4190	arch__add_leaf_frame_record_opts(opts: &rec->opts);
4191
4192	err = -ENOMEM;
4193	if (evlist__create_maps(evlist: rec->evlist, target: &rec->opts.target) < 0) {
4194	if (rec->opts.target.pid != NULL) {
4195	pr_err("Couldn't create thread/CPU maps: %s\n",
4196	errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf)));
4197	goto out;
4198	}
4199	else
4200	usage_with_options(record_usage, record_options);
4201	}
4202
4203	err = auxtrace_record__options(itr: rec->itr, evlist: rec->evlist, opts: &rec->opts);
4204	if (err)
4205	goto out;
4206
4207	/*
4208	* We take all buildids when the file contains
4209	* AUX area tracing data because we do not decode the
4210	* trace because it would take too long.
4211	*/
4212	if (rec->opts.full_auxtrace)
4213	rec->buildid_all = true;
4214
4215	if (rec->opts.text_poke) {
4216	err = record__config_text_poke(evlist: rec->evlist);
4217	if (err) {
4218	pr_err("record__config_text_poke failed, error %d\n", err);
4219	goto out;
4220	}
4221	}
4222
4223	if (rec->off_cpu) {
4224	err = record__config_off_cpu(rec);
4225	if (err) {
4226	pr_err("record__config_off_cpu failed, error %d\n", err);
4227	goto out;
4228	}
4229	}
4230
4231	if (record_opts__config(opts: &rec->opts)) {
4232	err = -EINVAL;
4233	goto out;
4234	}
4235
4236	err = record__config_tracking_events(rec);
4237	if (err) {
4238	pr_err("record__config_tracking_events failed, error %d\n", err);
4239	goto out;
4240	}
4241
4242	err = record__init_thread_masks(rec);
4243	if (err) {
4244	pr_err("Failed to initialize parallel data streaming masks\n");
4245	goto out;
4246	}
4247
4248	if (rec->opts.nr_cblocks > nr_cblocks_max)
4249	rec->opts.nr_cblocks = nr_cblocks_max;
4250	pr_debug("nr_cblocks: %d\n", rec->opts.nr_cblocks);
4251
4252	pr_debug("affinity: %s\n", affinity_tags[rec->opts.affinity]);
4253	pr_debug("mmap flush: %d\n", rec->opts.mmap_flush);
4254
4255	if (rec->opts.comp_level > comp_level_max)
4256	rec->opts.comp_level = comp_level_max;
4257	pr_debug("comp level: %d\n", rec->opts.comp_level);
4258
4259	err = __cmd_record(rec: &record, argc, argv);
4260	out:
4261	evlist__delete(evlist: rec->evlist);
4262	symbol__exit();
4263	auxtrace_record__free(itr: rec->itr);
4264	out_opts:
4265	record__free_thread_masks(rec, nr_threads: rec->nr_threads);
4266	rec->nr_threads = 0;
4267	evlist__close_control(ctl_fd: rec->opts.ctl_fd, ctl_fd_ack: rec->opts.ctl_fd_ack, ctl_fd_close: &rec->opts.ctl_fd_close);
4268	return err;
4269	}
4270
4271	static void snapshot_sig_handler(int sig __maybe_unused)
4272	{
4273	struct record *rec = &record;
4274
4275	hit_auxtrace_snapshot_trigger(rec);
4276
4277	if (switch_output_signal(rec))
4278	trigger_hit(t: &switch_output_trigger);
4279	}
4280
4281	static void alarm_sig_handler(int sig __maybe_unused)
4282	{
4283	struct record *rec = &record;
4284
4285	if (switch_output_time(rec))
4286	trigger_hit(t: &switch_output_trigger);
4287	}
4288