1	// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2	/* Copyright (C) 2017-2018 Netronome Systems, Inc. */
3
4	#ifndef _GNU_SOURCE
5	#define _GNU_SOURCE
6	#endif
7	#include <errno.h>
8	#include <fcntl.h>
9	#include <signal.h>
10	#include <stdarg.h>
11	#include <stdio.h>
12	#include <stdlib.h>
13	#include <string.h>
14	#include <time.h>
15	#include <unistd.h>
16	#include <net/if.h>
17	#include <sys/ioctl.h>
18	#include <sys/types.h>
19	#include <sys/stat.h>
20	#include <sys/syscall.h>
21	#include <dirent.h>
22
23	#include <linux/err.h>
24	#include <linux/perf_event.h>
25	#include <linux/sizes.h>
26	#include <linux/keyctl.h>
27
28	#include <bpf/bpf.h>
29	#include <bpf/btf.h>
30	#include <bpf/hashmap.h>
31	#include <bpf/libbpf.h>
32	#include <bpf/libbpf_internal.h>
33	#include <bpf/skel_internal.h>
34
35	#include "cfg.h"
36	#include "main.h"
37	#include "xlated_dumper.h"
38
39	#define BPF_METADATA_PREFIX "bpf_metadata_"
40	#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1)
41
42	enum dump_mode {
43	DUMP_JITED,
44	DUMP_XLATED,
45	};
46
47	static const bool attach_types[] = {
48	[BPF_SK_SKB_STREAM_PARSER] = true,
49	[BPF_SK_SKB_STREAM_VERDICT] = true,
50	[BPF_SK_SKB_VERDICT] = true,
51	[BPF_SK_MSG_VERDICT] = true,
52	[BPF_FLOW_DISSECTOR] = true,
53	[__MAX_BPF_ATTACH_TYPE] = false,
54	};
55
56	/* Textual representations traditionally used by the program and kept around
57	* for the sake of backwards compatibility.
58	*/
59	static const char * const attach_type_strings[] = {
60	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
61	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
62	[BPF_SK_SKB_VERDICT] = "skb_verdict",
63	[BPF_SK_MSG_VERDICT] = "msg_verdict",
64	[__MAX_BPF_ATTACH_TYPE] = NULL,
65	};
66
67	static struct hashmap *prog_table;
68
69	static enum bpf_attach_type parse_attach_type(const char *str)
70	{
71	enum bpf_attach_type type;
72
73	for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
74	if (attach_types[type]) {
75	const char *attach_type_str;
76
77	attach_type_str = libbpf_bpf_attach_type_str(type);
78	if (!strcmp(str, attach_type_str))
79	return type;
80	}
81
82	if (attach_type_strings[type] &&
83	is_prefix(pfx: str, str: attach_type_strings[type]))
84	return type;
85	}
86
87	return __MAX_BPF_ATTACH_TYPE;
88	}
89
90	static int prep_prog_info(struct bpf_prog_info *const info, enum dump_mode mode,
91	void **info_data, size_t *const info_data_sz)
92	{
93	struct bpf_prog_info holder = {};
94	size_t needed = 0;
95	void *ptr;
96
97	if (mode == DUMP_JITED) {
98	holder.jited_prog_len = info->jited_prog_len;
99	needed += info->jited_prog_len;
100	} else {
101	holder.xlated_prog_len = info->xlated_prog_len;
102	needed += info->xlated_prog_len;
103	}
104
105	holder.nr_jited_ksyms = info->nr_jited_ksyms;
106	needed += info->nr_jited_ksyms * sizeof(__u64);
107
108	holder.nr_jited_func_lens = info->nr_jited_func_lens;
109	needed += info->nr_jited_func_lens * sizeof(__u32);
110
111	holder.nr_func_info = info->nr_func_info;
112	holder.func_info_rec_size = info->func_info_rec_size;
113	needed += info->nr_func_info * info->func_info_rec_size;
114
115	holder.nr_line_info = info->nr_line_info;
116	holder.line_info_rec_size = info->line_info_rec_size;
117	needed += info->nr_line_info * info->line_info_rec_size;
118
119	holder.nr_jited_line_info = info->nr_jited_line_info;
120	holder.jited_line_info_rec_size = info->jited_line_info_rec_size;
121	needed += info->nr_jited_line_info * info->jited_line_info_rec_size;
122
123	if (needed > *info_data_sz) {
124	ptr = realloc(*info_data, needed);
125	if (!ptr)
126	return -1;
127
128	*info_data = ptr;
129	*info_data_sz = needed;
130	}
131	ptr = *info_data;
132
133	if (mode == DUMP_JITED) {
134	holder.jited_prog_insns = ptr_to_u64(ptr);
135	ptr += holder.jited_prog_len;
136	} else {
137	holder.xlated_prog_insns = ptr_to_u64(ptr);
138	ptr += holder.xlated_prog_len;
139	}
140
141	holder.jited_ksyms = ptr_to_u64(ptr);
142	ptr += holder.nr_jited_ksyms * sizeof(__u64);
143
144	holder.jited_func_lens = ptr_to_u64(ptr);
145	ptr += holder.nr_jited_func_lens * sizeof(__u32);
146
147	holder.func_info = ptr_to_u64(ptr);
148	ptr += holder.nr_func_info * holder.func_info_rec_size;
149
150	holder.line_info = ptr_to_u64(ptr);
151	ptr += holder.nr_line_info * holder.line_info_rec_size;
152
153	holder.jited_line_info = ptr_to_u64(ptr);
154	ptr += holder.nr_jited_line_info * holder.jited_line_info_rec_size;
155
156	*info = holder;
157	return 0;
158	}
159
160	static void print_boot_time(__u64 nsecs, char *buf, unsigned int size)
161	{
162	struct timespec real_time_ts, boot_time_ts;
163	time_t wallclock_secs;
164	struct tm load_tm;
165
166	buf[--size] = '\0';
167
168	if (clock_gettime(CLOCK_REALTIME, &real_time_ts) ||
169	clock_gettime(CLOCK_BOOTTIME, &boot_time_ts)) {
170	perror("Can't read clocks");
171	snprintf(buf, size, fmt: "%llu", nsecs / 1000000000);
172	return;
173	}
174
175	wallclock_secs = (real_time_ts.tv_sec - boot_time_ts.tv_sec) +
176	(real_time_ts.tv_nsec - boot_time_ts.tv_nsec + nsecs) /
177	1000000000;
178
179
180	if (!localtime_r(&wallclock_secs, &load_tm)) {
181	snprintf(buf, size, fmt: "%llu", nsecs / 1000000000);
182	return;
183	}
184
185	if (json_output)
186	strftime(buf, size, "%s", &load_tm);
187	else
188	strftime(buf, size, "%FT%T%z", &load_tm);
189	}
190
191	static void show_prog_maps(int fd, __u32 num_maps)
192	{
193	struct bpf_prog_info info = {};
194	__u32 len = sizeof(info);
195	__u32 map_ids[num_maps];
196	unsigned int i;
197	int err;
198
199	info.nr_map_ids = num_maps;
200	info.map_ids = ptr_to_u64(ptr: map_ids);
201
202	err = bpf_prog_get_info_by_fd(fd, &info, &len);
203	if (err || !info.nr_map_ids)
204	return;
205
206	if (json_output) {
207	jsonw_name(self: json_wtr, name: "map_ids");
208	jsonw_start_array(self: json_wtr);
209	for (i = 0; i < info.nr_map_ids; i++)
210	jsonw_uint(self: json_wtr, number: map_ids[i]);
211	jsonw_end_array(self: json_wtr);
212	} else {
213	printf(" map_ids ");
214	for (i = 0; i < info.nr_map_ids; i++)
215	printf("%u%s", map_ids[i],
216	i == info.nr_map_ids - 1 ? "" : ",");
217	}
218	}
219
220	static void *find_metadata(int prog_fd, struct bpf_map_info *map_info)
221	{
222	struct bpf_prog_info prog_info;
223	__u32 prog_info_len;
224	__u32 map_info_len;
225	void *value = NULL;
226	__u32 *map_ids;
227	int nr_maps;
228	int key = 0;
229	int map_fd;
230	int ret;
231	__u32 i;
232
233	memset(&prog_info, 0, sizeof(prog_info));
234	prog_info_len = sizeof(prog_info);
235	ret = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &prog_info_len);
236	if (ret)
237	return NULL;
238
239	if (!prog_info.nr_map_ids)
240	return NULL;
241
242	map_ids = calloc(prog_info.nr_map_ids, sizeof(__u32));
243	if (!map_ids)
244	return NULL;
245
246	nr_maps = prog_info.nr_map_ids;
247	memset(&prog_info, 0, sizeof(prog_info));
248	prog_info.nr_map_ids = nr_maps;
249	prog_info.map_ids = ptr_to_u64(ptr: map_ids);
250	prog_info_len = sizeof(prog_info);
251
252	ret = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &prog_info_len);
253	if (ret)
254	goto free_map_ids;
255
256	for (i = 0; i < prog_info.nr_map_ids; i++) {
257	map_fd = bpf_map_get_fd_by_id(map_ids[i]);
258	if (map_fd < 0)
259	goto free_map_ids;
260
261	memset(map_info, 0, sizeof(*map_info));
262	map_info_len = sizeof(*map_info);
263	ret = bpf_map_get_info_by_fd(map_fd, map_info, &map_info_len);
264	if (ret < 0) {
265	close(map_fd);
266	goto free_map_ids;
267	}
268
269	if (map_info->type != BPF_MAP_TYPE_ARRAY ||
270	map_info->key_size != sizeof(int) ||
271	map_info->max_entries != 1 ||
272	!map_info->btf_value_type_id ||
273	!strstr(map_info->name, ".rodata")) {
274	close(map_fd);
275	continue;
276	}
277
278	value = malloc(map_info->value_size);
279	if (!value) {
280	close(map_fd);
281	goto free_map_ids;
282	}
283
284	if (bpf_map_lookup_elem(map_fd, &key, value)) {
285	close(map_fd);
286	free(value);
287	value = NULL;
288	goto free_map_ids;
289	}
290
291	close(map_fd);
292	break;
293	}
294
295	free_map_ids:
296	free(map_ids);
297	return value;
298	}
299
300	static bool has_metadata_prefix(const char *s)
301	{
302	return strncmp(s, BPF_METADATA_PREFIX, BPF_METADATA_PREFIX_LEN) == 0;
303	}
304
305	static void show_prog_metadata(int fd, __u32 num_maps)
306	{
307	const struct btf_type *t_datasec, *t_var;
308	struct bpf_map_info map_info;
309	struct btf_var_secinfo *vsi;
310	bool printed_header = false;
311	unsigned int i, vlen;
312	void *value = NULL;
313	const char *name;
314	struct btf *btf;
315	int err;
316
317	if (!num_maps)
318	return;
319
320	memset(&map_info, 0, sizeof(map_info));
321	value = find_metadata(prog_fd: fd, map_info: &map_info);
322	if (!value)
323	return;
324
325	btf = btf__load_from_kernel_by_id(map_info.btf_id);
326	if (!btf)
327	goto out_free;
328
329	t_datasec = btf__type_by_id(btf, map_info.btf_value_type_id);
330	if (!btf_is_datasec(t_datasec))
331	goto out_free;
332
333	vlen = btf_vlen(t: t_datasec);
334	vsi = btf_var_secinfos(t_datasec);
335
336	/* We don't proceed to check the kinds of the elements of the DATASEC.
337	* The verifier enforces them to be BTF_KIND_VAR.
338	*/
339
340	if (json_output) {
341	struct btf_dumper d = {
342	.btf = btf,
343	.jw = json_wtr,
344	.is_plain_text = false,
345	};
346
347	for (i = 0; i < vlen; i++, vsi++) {
348	t_var = btf__type_by_id(btf, vsi->type);
349	name = btf__name_by_offset(btf, t_var->name_off);
350
351	if (!has_metadata_prefix(s: name))
352	continue;
353
354	if (!printed_header) {
355	jsonw_name(self: json_wtr, name: "metadata");
356	jsonw_start_object(self: json_wtr);
357	printed_header = true;
358	}
359
360	jsonw_name(self: json_wtr, name: name + BPF_METADATA_PREFIX_LEN);
361	err = btf_dumper_type(d: &d, type_id: t_var->type, data: value + vsi->offset);
362	if (err) {
363	p_err(fmt: "btf dump failed: %d", err);
364	break;
365	}
366	}
367	if (printed_header)
368	jsonw_end_object(self: json_wtr);
369	} else {
370	json_writer_t *btf_wtr;
371	struct btf_dumper d = {
372	.btf = btf,
373	.is_plain_text = true,
374	};
375
376	for (i = 0; i < vlen; i++, vsi++) {
377	t_var = btf__type_by_id(btf, vsi->type);
378	name = btf__name_by_offset(btf, t_var->name_off);
379
380	if (!has_metadata_prefix(s: name))
381	continue;
382
383	if (!printed_header) {
384	printf("\tmetadata:");
385
386	btf_wtr = jsonw_new(stdout);
387	if (!btf_wtr) {
388	p_err(fmt: "jsonw alloc failed");
389	goto out_free;
390	}
391	d.jw = btf_wtr,
392
393	printed_header = true;
394	}
395
396	printf("\n\t\t%s = ", name + BPF_METADATA_PREFIX_LEN);
397
398	jsonw_reset(self: btf_wtr);
399	err = btf_dumper_type(d: &d, type_id: t_var->type, data: value + vsi->offset);
400	if (err) {
401	p_err(fmt: "btf dump failed: %d", err);
402	break;
403	}
404	}
405	if (printed_header)
406	jsonw_destroy(self_p: &btf_wtr);
407	}
408
409	out_free:
410	btf__free(btf);
411	free(value);
412	}
413
414	static void print_prog_header_json(struct bpf_prog_info *info, int fd)
415	{
416	const char *prog_type_str;
417	char prog_name[MAX_PROG_FULL_NAME];
418
419	jsonw_uint_field(self: json_wtr, prop: "id", num: info->id);
420	prog_type_str = libbpf_bpf_prog_type_str(info->type);
421
422	if (prog_type_str)
423	jsonw_string_field(self: json_wtr, prop: "type", val: prog_type_str);
424	else
425	jsonw_uint_field(self: json_wtr, prop: "type", num: info->type);
426
427	if (*info->name) {
428	get_prog_full_name(prog_info: info, prog_fd: fd, name_buff: prog_name, buff_len: sizeof(prog_name));
429	jsonw_string_field(self: json_wtr, prop: "name", val: prog_name);
430	}
431
432	jsonw_name(self: json_wtr, name: "tag");
433	jsonw_printf(self: json_wtr, fmt: "\"" BPF_TAG_FMT "\"",
434	info->tag[0], info->tag[1], info->tag[2], info->tag[3],
435	info->tag[4], info->tag[5], info->tag[6], info->tag[7]);
436
437	jsonw_bool_field(self: json_wtr, prop: "gpl_compatible", value: info->gpl_compatible);
438	if (info->run_time_ns) {
439	jsonw_uint_field(self: json_wtr, prop: "run_time_ns", num: info->run_time_ns);
440	jsonw_uint_field(self: json_wtr, prop: "run_cnt", num: info->run_cnt);
441	}
442	if (info->recursion_misses)
443	jsonw_uint_field(self: json_wtr, prop: "recursion_misses", num: info->recursion_misses);
444	}
445
446	static void print_prog_json(struct bpf_prog_info *info, int fd, bool orphaned)
447	{
448	char *memlock;
449
450	jsonw_start_object(self: json_wtr);
451	print_prog_header_json(info, fd);
452	print_dev_json(ifindex: info->ifindex, ns_dev: info->netns_dev, ns_inode: info->netns_ino);
453
454	if (info->load_time) {
455	char buf[32];
456
457	print_boot_time(nsecs: info->load_time, buf, size: sizeof(buf));
458
459	/* Piggy back on load_time, since 0 uid is a valid one */
460	jsonw_name(self: json_wtr, name: "loaded_at");
461	jsonw_printf(self: json_wtr, fmt: "%s", buf);
462	jsonw_uint_field(self: json_wtr, prop: "uid", num: info->created_by_uid);
463	}
464
465	jsonw_bool_field(self: json_wtr, prop: "orphaned", value: orphaned);
466	jsonw_uint_field(self: json_wtr, prop: "bytes_xlated", num: info->xlated_prog_len);
467
468	if (info->jited_prog_len) {
469	jsonw_bool_field(self: json_wtr, prop: "jited", value: true);
470	jsonw_uint_field(self: json_wtr, prop: "bytes_jited", num: info->jited_prog_len);
471	} else {
472	jsonw_bool_field(self: json_wtr, prop: "jited", value: false);
473	}
474
475	memlock = get_fdinfo(fd, key: "memlock");
476	if (memlock)
477	jsonw_int_field(self: json_wtr, prop: "bytes_memlock", num: atoll(memlock));
478	free(memlock);
479
480	if (info->nr_map_ids)
481	show_prog_maps(fd, num_maps: info->nr_map_ids);
482
483	if (info->btf_id)
484	jsonw_int_field(self: json_wtr, prop: "btf_id", num: info->btf_id);
485
486	if (!hashmap__empty(map: prog_table)) {
487	struct hashmap_entry *entry;
488
489	jsonw_name(self: json_wtr, name: "pinned");
490	jsonw_start_array(self: json_wtr);
491	hashmap__for_each_key_entry(prog_table, entry, info->id)
492	jsonw_string(self: json_wtr, value: entry->pvalue);
493	jsonw_end_array(self: json_wtr);
494	}
495
496	emit_obj_refs_json(table: refs_table, id: info->id, json_wtr);
497
498	show_prog_metadata(fd, num_maps: info->nr_map_ids);
499
500	jsonw_end_object(self: json_wtr);
501	}
502
503	static void print_prog_header_plain(struct bpf_prog_info *info, int fd)
504	{
505	const char *prog_type_str;
506	char prog_name[MAX_PROG_FULL_NAME];
507
508	printf("%u: ", info->id);
509	prog_type_str = libbpf_bpf_prog_type_str(info->type);
510	if (prog_type_str)
511	printf("%s ", prog_type_str);
512	else
513	printf("type %u ", info->type);
514
515	if (*info->name) {
516	get_prog_full_name(prog_info: info, prog_fd: fd, name_buff: prog_name, buff_len: sizeof(prog_name));
517	printf("name %s ", prog_name);
518	}
519
520	printf("tag ");
521	fprint_hex(stdout, info->tag, BPF_TAG_SIZE, "");
522	print_dev_plain(ifindex: info->ifindex, ns_dev: info->netns_dev, ns_inode: info->netns_ino);
523	printf("%s", info->gpl_compatible ? " gpl" : "");
524	if (info->run_time_ns)
525	printf(" run_time_ns %llu run_cnt %llu",
526	info->run_time_ns, info->run_cnt);
527	if (info->recursion_misses)
528	printf(" recursion_misses %llu", info->recursion_misses);
529	printf("\n");
530	}
531
532	static void print_prog_plain(struct bpf_prog_info *info, int fd, bool orphaned)
533	{
534	char *memlock;
535
536	print_prog_header_plain(info, fd);
537
538	if (info->load_time) {
539	char buf[32];
540
541	print_boot_time(nsecs: info->load_time, buf, size: sizeof(buf));
542
543	/* Piggy back on load_time, since 0 uid is a valid one */
544	printf("\tloaded_at %s uid %u\n", buf, info->created_by_uid);
545	}
546
547	printf("\txlated %uB", info->xlated_prog_len);
548
549	if (info->jited_prog_len)
550	printf(" jited %uB", info->jited_prog_len);
551	else
552	printf(" not jited");
553
554	memlock = get_fdinfo(fd, key: "memlock");
555	if (memlock)
556	printf(" memlock %sB", memlock);
557	free(memlock);
558
559	if (orphaned)
560	printf(" orphaned");
561
562	if (info->nr_map_ids)
563	show_prog_maps(fd, num_maps: info->nr_map_ids);
564
565	if (!hashmap__empty(map: prog_table)) {
566	struct hashmap_entry *entry;
567
568	hashmap__for_each_key_entry(prog_table, entry, info->id)
569	printf("\n\tpinned %s", (char *)entry->pvalue);
570	}
571
572	if (info->btf_id)
573	printf("\n\tbtf_id %u", info->btf_id);
574
575	emit_obj_refs_plain(table: refs_table, id: info->id, prefix: "\n\tpids ");
576
577	printf("\n");
578
579	show_prog_metadata(fd, num_maps: info->nr_map_ids);
580	}
581
582	static int show_prog(int fd)
583	{
584	struct bpf_prog_info info = {};
585	__u32 len = sizeof(info);
586	int err;
587
588	err = bpf_prog_get_info_by_fd(fd, &info, &len);
589	if (err && err != -ENODEV) {
590	p_err(fmt: "can't get prog info: %s", strerror(errno));
591	return -1;
592	}
593
594	if (json_output)
595	print_prog_json(info: &info, fd, orphaned: err == -ENODEV);
596	else
597	print_prog_plain(info: &info, fd, orphaned: err == -ENODEV);
598
599	return 0;
600	}
601
602	static int do_show_subset(int argc, char **argv)
603	{
604	int *fds = NULL;
605	int nb_fds, i;
606	int err = -1;
607
608	fds = malloc(sizeof(int));
609	if (!fds) {
610	p_err(fmt: "mem alloc failed");
611	return -1;
612	}
613	nb_fds = prog_parse_fds(argc: &argc, argv: &argv, fds: &fds);
614	if (nb_fds < 1)
615	goto exit_free;
616
617	if (json_output && nb_fds > 1)
618	jsonw_start_array(self: json_wtr); /* root array */
619	for (i = 0; i < nb_fds; i++) {
620	err = show_prog(fd: fds[i]);
621	if (err) {
622	for (; i < nb_fds; i++)
623	close(fds[i]);
624	break;
625	}
626	close(fds[i]);
627	}
628	if (json_output && nb_fds > 1)
629	jsonw_end_array(self: json_wtr); /* root array */
630
631	exit_free:
632	free(fds);
633	return err;
634	}
635
636	static int do_show(int argc, char **argv)
637	{
638	__u32 id = 0;
639	int err;
640	int fd;
641
642	if (show_pinned) {
643	prog_table = hashmap__new(hash_fn_for_key_as_id,
644	equal_fn_for_key_as_id, NULL);
645	if (IS_ERR(ptr: prog_table)) {
646	p_err(fmt: "failed to create hashmap for pinned paths");
647	return -1;
648	}
649	build_pinned_obj_table(table: prog_table, type: BPF_OBJ_PROG);
650	}
651	build_obj_refs_table(table: &refs_table, type: BPF_OBJ_PROG);
652
653	if (argc == 2)
654	return do_show_subset(argc, argv);
655
656	if (argc)
657	return BAD_ARG();
658
659	if (json_output)
660	jsonw_start_array(self: json_wtr);
661	while (true) {
662	err = bpf_prog_get_next_id(id, &id);
663	if (err) {
664	if (errno == ENOENT) {
665	err = 0;
666	break;
667	}
668	p_err(fmt: "can't get next program: %s%s", strerror(errno),
669	errno == EINVAL ? " -- kernel too old?" : "");
670	err = -1;
671	break;
672	}
673
674	fd = bpf_prog_get_fd_by_id(id);
675	if (fd < 0) {
676	if (errno == ENOENT)
677	continue;
678	p_err(fmt: "can't get prog by id (%u): %s",
679	id, strerror(errno));
680	err = -1;
681	break;
682	}
683
684	err = show_prog(fd);
685	close(fd);
686	if (err)
687	break;
688	}
689
690	if (json_output)
691	jsonw_end_array(self: json_wtr);
692
693	delete_obj_refs_table(table: refs_table);
694
695	if (show_pinned)
696	delete_pinned_obj_table(table: prog_table);
697
698	return err;
699	}
700
701	static int
702	prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
703	char *filepath, bool opcodes, bool visual, bool linum)
704	{
705	struct bpf_prog_linfo *prog_linfo = NULL;
706	const char *disasm_opt = NULL;
707	struct dump_data dd = {};
708	void *func_info = NULL;
709	struct btf *btf = NULL;
710	char func_sig[1024];
711	unsigned char *buf;
712	__u32 member_len;
713	int fd, err = -1;
714	ssize_t n;
715
716	if (mode == DUMP_JITED) {
717	if (info->jited_prog_len == 0 || !info->jited_prog_insns) {
718	p_err(fmt: "error retrieving jit dump: no instructions returned or kernel.kptr_restrict set?");
719	return -1;
720	}
721	buf = u64_to_ptr(ptr: info->jited_prog_insns);
722	member_len = info->jited_prog_len;
723	} else { /* DUMP_XLATED */
724	if (info->xlated_prog_len == 0 || !info->xlated_prog_insns) {
725	p_err(fmt: "error retrieving insn dump: kernel.kptr_restrict set?");
726	return -1;
727	}
728	buf = u64_to_ptr(ptr: info->xlated_prog_insns);
729	member_len = info->xlated_prog_len;
730	}
731
732	if (info->btf_id) {
733	btf = btf__load_from_kernel_by_id(info->btf_id);
734	if (!btf) {
735	p_err(fmt: "failed to get btf");
736	return -1;
737	}
738	}
739
740	func_info = u64_to_ptr(ptr: info->func_info);
741
742	if (info->nr_line_info) {
743	prog_linfo = bpf_prog_linfo__new(info);
744	if (!prog_linfo)
745	p_info(fmt: "error in processing bpf_line_info. continue without it.");
746	}
747
748	if (filepath) {
749	fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600);
750	if (fd < 0) {
751	p_err(fmt: "can't open file %s: %s", filepath,
752	strerror(errno));
753	goto exit_free;
754	}
755
756	n = write(fd, buf, member_len);
757	close(fd);
758	if (n != (ssize_t)member_len) {
759	p_err(fmt: "error writing output file: %s",
760	n < 0 ? strerror(errno) : "short write");
761	goto exit_free;
762	}
763
764	if (json_output)
765	jsonw_null(self: json_wtr);
766	} else if (mode == DUMP_JITED) {
767	const char *name = NULL;
768
769	if (info->ifindex) {
770	name = ifindex_to_arch(ifindex: info->ifindex, ns_dev: info->netns_dev,
771	ns_ino: info->netns_ino, opt: &disasm_opt);
772	if (!name)
773	goto exit_free;
774	}
775
776	if (info->nr_jited_func_lens && info->jited_func_lens) {
777	struct kernel_sym *sym = NULL;
778	struct bpf_func_info *record;
779	char sym_name[SYM_MAX_NAME];
780	unsigned char *img = buf;
781	__u64 *ksyms = NULL;
782	__u32 *lens;
783	__u32 i;
784	if (info->nr_jited_ksyms) {
785	kernel_syms_load(dd: &dd);
786	ksyms = u64_to_ptr(ptr: info->jited_ksyms);
787	}
788
789	if (json_output)
790	jsonw_start_array(self: json_wtr);
791
792	lens = u64_to_ptr(ptr: info->jited_func_lens);
793	for (i = 0; i < info->nr_jited_func_lens; i++) {
794	if (ksyms) {
795	sym = kernel_syms_search(dd: &dd, key: ksyms[i]);
796	if (sym)
797	sprintf(buf: sym_name, fmt: "%s", sym->name);
798	else
799	sprintf(buf: sym_name, fmt: "0x%016llx", ksyms[i]);
800	} else {
801	strcpy(p: sym_name, q: "unknown");
802	}
803
804	if (func_info) {
805	record = func_info + i * info->func_info_rec_size;
806	btf_dumper_type_only(btf, func_type_id: record->type_id,
807	func_only: func_sig,
808	size: sizeof(func_sig));
809	}
810
811	if (json_output) {
812	jsonw_start_object(self: json_wtr);
813	if (func_info && func_sig[0] != '\0') {
814	jsonw_name(self: json_wtr, name: "proto");
815	jsonw_string(self: json_wtr, value: func_sig);
816	}
817	jsonw_name(self: json_wtr, name: "name");
818	jsonw_string(self: json_wtr, value: sym_name);
819	jsonw_name(self: json_wtr, name: "insns");
820	} else {
821	if (func_info && func_sig[0] != '\0')
822	printf("%s:\n", func_sig);
823	printf("%s:\n", sym_name);
824	}
825
826	if (ksyms) {
827	if (disasm_print_insn(image: img, len: lens[i], opcodes,
828	arch: name, disassembler_options: disasm_opt, btf,
829	prog_linfo, func_ksym: ksyms[i], func_idx: i,
830	linum))
831	goto exit_free;
832	} else {
833	if (disasm_print_insn(image: img, len: lens[i], opcodes,
834	arch: name, disassembler_options: disasm_opt, btf,
835	NULL, func_ksym: 0, func_idx: 0, linum: false))
836	goto exit_free;
837	}
838
839	img += lens[i];
840
841	if (json_output)
842	jsonw_end_object(self: json_wtr);
843	else
844	printf("\n");
845	}
846
847	if (json_output)
848	jsonw_end_array(self: json_wtr);
849	} else {
850	if (disasm_print_insn(image: buf, len: member_len, opcodes, arch: name,
851	disassembler_options: disasm_opt, btf, NULL, func_ksym: 0, func_idx: 0,
852	linum: false))
853	goto exit_free;
854	}
855	} else {
856	kernel_syms_load(dd: &dd);
857	dd.nr_jited_ksyms = info->nr_jited_ksyms;
858	dd.jited_ksyms = u64_to_ptr(ptr: info->jited_ksyms);
859	dd.btf = btf;
860	dd.func_info = func_info;
861	dd.finfo_rec_size = info->func_info_rec_size;
862	dd.prog_linfo = prog_linfo;
863
864	if (json_output)
865	dump_xlated_json(dd: &dd, buf, len: member_len, opcodes, linum);
866	else if (visual)
867	dump_xlated_cfg(dd: &dd, buf, len: member_len, opcodes, linum);
868	else
869	dump_xlated_plain(dd: &dd, buf, len: member_len, opcodes, linum);
870	kernel_syms_destroy(dd: &dd);
871	}
872
873	err = 0;
874
875	exit_free:
876	btf__free(btf);
877	bpf_prog_linfo__free(prog_linfo);
878	return err;
879	}
880
881	static int do_dump(int argc, char **argv)
882	{
883	struct bpf_prog_info info;
884	__u32 info_len = sizeof(info);
885	size_t info_data_sz = 0;
886	void *info_data = NULL;
887	char *filepath = NULL;
888	bool opcodes = false;
889	bool visual = false;
890	enum dump_mode mode;
891	bool linum = false;
892	int nb_fds, i = 0;
893	int *fds = NULL;
894	int err = -1;
895
896	if (is_prefix(pfx: *argv, str: "jited")) {
897	if (disasm_init())
898	return -1;
899	mode = DUMP_JITED;
900	} else if (is_prefix(pfx: *argv, str: "xlated")) {
901	mode = DUMP_XLATED;
902	} else {
903	p_err(fmt: "expected 'xlated' or 'jited', got: %s", *argv);
904	return -1;
905	}
906	NEXT_ARG();
907
908	if (argc < 2)
909	usage();
910
911	fds = malloc(sizeof(int));
912	if (!fds) {
913	p_err(fmt: "mem alloc failed");
914	return -1;
915	}
916	nb_fds = prog_parse_fds(argc: &argc, argv: &argv, fds: &fds);
917	if (nb_fds < 1)
918	goto exit_free;
919
920	while (argc) {
921	if (is_prefix(pfx: *argv, str: "file")) {
922	NEXT_ARG();
923	if (!argc) {
924	p_err(fmt: "expected file path");
925	goto exit_close;
926	}
927	if (nb_fds > 1) {
928	p_err(fmt: "several programs matched");
929	goto exit_close;
930	}
931
932	filepath = *argv;
933	NEXT_ARG();
934	} else if (is_prefix(pfx: *argv, str: "opcodes")) {
935	opcodes = true;
936	NEXT_ARG();
937	} else if (is_prefix(pfx: *argv, str: "visual")) {
938	if (nb_fds > 1) {
939	p_err(fmt: "several programs matched");
940	goto exit_close;
941	}
942
943	visual = true;
944	NEXT_ARG();
945	} else if (is_prefix(pfx: *argv, str: "linum")) {
946	linum = true;
947	NEXT_ARG();
948	} else {
949	usage();
950	goto exit_close;
951	}
952	}
953
954	if (filepath && (opcodes || visual || linum)) {
955	p_err(fmt: "'file' is not compatible with 'opcodes', 'visual', or 'linum'");
956	goto exit_close;
957	}
958	if (json_output && visual) {
959	p_err(fmt: "'visual' is not compatible with JSON output");
960	goto exit_close;
961	}
962
963	if (json_output && nb_fds > 1)
964	jsonw_start_array(self: json_wtr); /* root array */
965	for (i = 0; i < nb_fds; i++) {
966	memset(&info, 0, sizeof(info));
967
968	err = bpf_prog_get_info_by_fd(fds[i], &info, &info_len);
969	if (err) {
970	p_err("can't get prog info: %s", strerror(errno));
971	break;
972	}
973
974	err = prep_prog_info(info: &info, mode, info_data: &info_data, info_data_sz: &info_data_sz);
975	if (err) {
976	p_err(fmt: "can't grow prog info_data");
977	break;
978	}
979
980	err = bpf_prog_get_info_by_fd(fds[i], &info, &info_len);
981	if (err) {
982	p_err("can't get prog info: %s", strerror(errno));
983	break;
984	}
985
986	if (json_output && nb_fds > 1) {
987	jsonw_start_object(self: json_wtr); /* prog object */
988	print_prog_header_json(info: &info, fd: fds[i]);
989	jsonw_name(self: json_wtr, name: "insns");
990	} else if (nb_fds > 1) {
991	print_prog_header_plain(info: &info, fd: fds[i]);
992	}
993
994	err = prog_dump(info: &info, mode, filepath, opcodes, visual, linum);
995
996	if (json_output && nb_fds > 1)
997	jsonw_end_object(self: json_wtr); /* prog object */
998	else if (i != nb_fds - 1 && nb_fds > 1)
999	printf("\n");
1000
1001	if (err)
1002	break;
1003	close(fds[i]);
1004	}
1005	if (json_output && nb_fds > 1)
1006	jsonw_end_array(self: json_wtr); /* root array */
1007
1008	exit_close:
1009	for (; i < nb_fds; i++)
1010	close(fds[i]);
1011	exit_free:
1012	free(info_data);
1013	free(fds);
1014	return err;
1015	}
1016
1017	static int do_pin(int argc, char **argv)
1018	{
1019	int err;
1020
1021	err = do_pin_any(argc, argv, get_fd_by_id: prog_parse_fd);
1022	if (!err && json_output)
1023	jsonw_null(self: json_wtr);
1024	return err;
1025	}
1026
1027	struct map_replace {
1028	int idx;
1029	int fd;
1030	char *name;
1031	};
1032
1033	static int map_replace_compar(const void *p1, const void *p2)
1034	{
1035	const struct map_replace *a = p1, *b = p2;
1036
1037	return a->idx - b->idx;
1038	}
1039
1040	static int parse_attach_detach_args(int argc, char **argv, int *progfd,
1041	enum bpf_attach_type *attach_type,
1042	int *mapfd)
1043	{
1044	if (!REQ_ARGS(3))
1045	return -EINVAL;
1046
1047	*progfd = prog_parse_fd(argc: &argc, argv: &argv);
1048	if (*progfd < 0)
1049	return *progfd;
1050
1051	*attach_type = parse_attach_type(str: *argv);
1052	if (*attach_type == __MAX_BPF_ATTACH_TYPE) {
1053	p_err(fmt: "invalid attach/detach type");
1054	return -EINVAL;
1055	}
1056
1057	if (*attach_type == BPF_FLOW_DISSECTOR) {
1058	*mapfd = 0;
1059	return 0;
1060	}
1061
1062	NEXT_ARG();
1063	if (!REQ_ARGS(2))
1064	return -EINVAL;
1065
1066	*mapfd = map_parse_fd(argc: &argc, argv: &argv, open_flags: 0);
1067	if (*mapfd < 0)
1068	return *mapfd;
1069
1070	return 0;
1071	}
1072
1073	static int do_attach(int argc, char **argv)
1074	{
1075	enum bpf_attach_type attach_type;
1076	int err, progfd;
1077	int mapfd;
1078
1079	err = parse_attach_detach_args(argc, argv,
1080	progfd: &progfd, attach_type: &attach_type, mapfd: &mapfd);
1081	if (err)
1082	return err;
1083
1084	err = bpf_prog_attach(progfd, mapfd, attach_type, 0);
1085	if (err) {
1086	p_err(fmt: "failed prog attach to map");
1087	return -EINVAL;
1088	}
1089
1090	if (json_output)
1091	jsonw_null(self: json_wtr);
1092	return 0;
1093	}
1094
1095	static int do_detach(int argc, char **argv)
1096	{
1097	enum bpf_attach_type attach_type;
1098	int err, progfd;
1099	int mapfd;
1100
1101	err = parse_attach_detach_args(argc, argv,
1102	progfd: &progfd, attach_type: &attach_type, mapfd: &mapfd);
1103	if (err)
1104	return err;
1105
1106	err = bpf_prog_detach2(progfd, mapfd, attach_type);
1107	if (err) {
1108	p_err(fmt: "failed prog detach from map");
1109	return -EINVAL;
1110	}
1111
1112	if (json_output)
1113	jsonw_null(self: json_wtr);
1114	return 0;
1115	}
1116
1117	enum prog_tracelog_mode {
1118	TRACE_STDOUT,
1119	TRACE_STDERR,
1120	};
1121
1122	static int
1123	prog_tracelog_stream(int prog_fd, enum prog_tracelog_mode mode)
1124	{
1125	FILE *file = mode == TRACE_STDOUT ? stdout : stderr;
1126	int stream_id = mode == TRACE_STDOUT ? 1 : 2;
1127	char buf[512];
1128	int ret;
1129
1130	ret = 0;
1131	do {
1132	ret = bpf_prog_stream_read(prog_fd, stream_id, buf, sizeof(buf), NULL);
1133	if (ret > 0)
1134	fwrite(buf, sizeof(buf[0]), ret, file);
1135	} while (ret > 0);
1136
1137	fflush(file);
1138	return ret ? -1 : 0;
1139	}
1140
1141	static int do_tracelog_any(int argc, char **argv)
1142	{
1143	enum prog_tracelog_mode mode;
1144	int fd;
1145
1146	if (argc == 0)
1147	return do_tracelog(argc, arg: argv);
1148	if (!is_prefix(pfx: *argv, str: "stdout") && !is_prefix(pfx: *argv, str: "stderr"))
1149	usage();
1150	mode = is_prefix(pfx: *argv, str: "stdout") ? TRACE_STDOUT : TRACE_STDERR;
1151	NEXT_ARG();
1152
1153	if (!REQ_ARGS(2))
1154	return -1;
1155
1156	fd = prog_parse_fd(argc: &argc, argv: &argv);
1157	if (fd < 0)
1158	return -1;
1159
1160	return prog_tracelog_stream(prog_fd: fd, mode);
1161	}
1162
1163	static int check_single_stdin(char *file_data_in, char *file_ctx_in)
1164	{
1165	if (file_data_in && file_ctx_in &&
1166	!strcmp(file_data_in, "-") && !strcmp(file_ctx_in, "-")) {
1167	p_err(fmt: "cannot use standard input for both data_in and ctx_in");
1168	return -1;
1169	}
1170
1171	return 0;
1172	}
1173
1174	static int get_run_data(const char *fname, void **data_ptr, unsigned int *size)
1175	{
1176	size_t block_size = 256;
1177	size_t buf_size = block_size;
1178	size_t nb_read = 0;
1179	void *tmp;
1180	FILE *f;
1181
1182	if (!fname) {
1183	*data_ptr = NULL;
1184	*size = 0;
1185	return 0;
1186	}
1187
1188	if (!strcmp(fname, "-"))
1189	f = stdin;
1190	else
1191	f = fopen(fname, "r");
1192	if (!f) {
1193	p_err("failed to open %s: %s", fname, strerror(errno));
1194	return -1;
1195	}
1196
1197	*data_ptr = malloc(block_size);
1198	if (!*data_ptr) {
1199	p_err("failed to allocate memory for data_in/ctx_in: %s",
1200	strerror(errno));
1201	goto err_fclose;
1202	}
1203
1204	while ((nb_read += fread(*data_ptr + nb_read, 1, block_size, f))) {
1205	if (feof(f))
1206	break;
1207	if (ferror(f)) {
1208	p_err("failed to read data_in/ctx_in from %s: %s",
1209	fname, strerror(errno));
1210	goto err_free;
1211	}
1212	if (nb_read > buf_size - block_size) {
1213	if (buf_size == UINT32_MAX) {
1214	p_err("data_in/ctx_in is too long (max: %u)",
1215	UINT32_MAX);
1216	goto err_free;
1217	}
1218	/* No space for fread()-ing next chunk; realloc() */
1219	buf_size *= 2;
1220	tmp = realloc(*data_ptr, buf_size);
1221	if (!tmp) {
1222	p_err("failed to reallocate data_in/ctx_in: %s",
1223	strerror(errno));
1224	goto err_free;
1225	}
1226	*data_ptr = tmp;
1227	}
1228	}
1229	if (f != stdin)
1230	fclose(f);
1231
1232	*size = nb_read;
1233	return 0;
1234
1235	err_free:
1236	free(*data_ptr);
1237	*data_ptr = NULL;
1238	err_fclose:
1239	if (f != stdin)
1240	fclose(f);
1241	return -1;
1242	}
1243
1244	static void hex_print(void *data, unsigned int size, FILE *f)
1245	{
1246	size_t i, j;
1247	char c;
1248
1249	for (i = 0; i < size; i += 16) {
1250	/* Row offset */
1251	fprintf(f, "%07zx\t", i);
1252
1253	/* Hexadecimal values */
1254	for (j = i; j < i + 16 && j < size; j++)
1255	fprintf(f, "%02x%s", *(uint8_t *)(data + j),
1256	j % 2 ? " " : "");
1257	for (; j < i + 16; j++)
1258	fprintf(f, " %s", j % 2 ? " " : "");
1259
1260	/* ASCII values (if relevant), '.' otherwise */
1261	fprintf(f, "| ");
1262	for (j = i; j < i + 16 && j < size; j++) {
1263	c = *(char *)(data + j);
1264	if (c < ' ' || c > '~')
1265	c = '.';
1266	fprintf(f, "%c%s", c, j == i + 7 ? " " : "");
1267	}
1268
1269	fprintf(f, "\n");
1270	}
1271	}
1272
1273	static int
1274	print_run_output(void *data, unsigned int size, const char *fname,
1275	const char *json_key)
1276	{
1277	size_t nb_written;
1278	FILE *f;
1279
1280	if (!fname)
1281	return 0;
1282
1283	if (!strcmp(fname, "-")) {
1284	f = stdout;
1285	if (json_output) {
1286	jsonw_name(self: json_wtr, name: json_key);
1287	print_data_json(data, len: size);
1288	} else {
1289	hex_print(data, size, f);
1290	}
1291	return 0;
1292	}
1293
1294	f = fopen(fname, "w");
1295	if (!f) {
1296	p_err("failed to open %s: %s", fname, strerror(errno));
1297	return -1;
1298	}
1299
1300	nb_written = fwrite(data, 1, size, f);
1301	fclose(f);
1302	if (nb_written != size) {
1303	p_err("failed to write output data/ctx: %s", strerror(errno));
1304	return -1;
1305	}
1306
1307	return 0;
1308	}
1309
1310	static int alloc_run_data(void **data_ptr, unsigned int size_out)
1311	{
1312	*data_ptr = calloc(size_out, 1);
1313	if (!*data_ptr) {
1314	p_err("failed to allocate memory for output data/ctx: %s",
1315	strerror(errno));
1316	return -1;
1317	}
1318
1319	return 0;
1320	}
1321
1322	static int do_run(int argc, char **argv)
1323	{
1324	char *data_fname_in = NULL, *data_fname_out = NULL;
1325	char *ctx_fname_in = NULL, *ctx_fname_out = NULL;
1326	const unsigned int default_size = SZ_32K;
1327	void *data_in = NULL, *data_out = NULL;
1328	void *ctx_in = NULL, *ctx_out = NULL;
1329	unsigned int repeat = 1;
1330	int fd, err;
1331	LIBBPF_OPTS(bpf_test_run_opts, test_attr);
1332
1333	if (!REQ_ARGS(4))
1334	return -1;
1335
1336	fd = prog_parse_fd(argc: &argc, argv: &argv);
1337	if (fd < 0)
1338	return -1;
1339
1340	while (argc) {
1341	if (detect_common_prefix(arg: *argv, "data_in", "data_out",
1342	"data_size_out", NULL))
1343	return -1;
1344	if (detect_common_prefix(arg: *argv, "ctx_in", "ctx_out",
1345	"ctx_size_out", NULL))
1346	return -1;
1347
1348	if (is_prefix(pfx: *argv, str: "data_in")) {
1349	NEXT_ARG();
1350	if (!REQ_ARGS(1))
1351	return -1;
1352
1353	data_fname_in = GET_ARG();
1354	if (check_single_stdin(file_data_in: data_fname_in, file_ctx_in: ctx_fname_in))
1355	return -1;
1356	} else if (is_prefix(pfx: *argv, str: "data_out")) {
1357	NEXT_ARG();
1358	if (!REQ_ARGS(1))
1359	return -1;
1360
1361	data_fname_out = GET_ARG();
1362	} else if (is_prefix(pfx: *argv, str: "data_size_out")) {
1363	char *endptr;
1364
1365	NEXT_ARG();
1366	if (!REQ_ARGS(1))
1367	return -1;
1368
1369	test_attr.data_size_out = strtoul(*argv, &endptr, 0);
1370	if (*endptr) {
1371	p_err(fmt: "can't parse %s as output data size",
1372	*argv);
1373	return -1;
1374	}
1375	NEXT_ARG();
1376	} else if (is_prefix(pfx: *argv, str: "ctx_in")) {
1377	NEXT_ARG();
1378	if (!REQ_ARGS(1))
1379	return -1;
1380
1381	ctx_fname_in = GET_ARG();
1382	if (check_single_stdin(file_data_in: data_fname_in, file_ctx_in: ctx_fname_in))
1383	return -1;
1384	} else if (is_prefix(pfx: *argv, str: "ctx_out")) {
1385	NEXT_ARG();
1386	if (!REQ_ARGS(1))
1387	return -1;
1388
1389	ctx_fname_out = GET_ARG();
1390	} else if (is_prefix(pfx: *argv, str: "ctx_size_out")) {
1391	char *endptr;
1392
1393	NEXT_ARG();
1394	if (!REQ_ARGS(1))
1395	return -1;
1396
1397	test_attr.ctx_size_out = strtoul(*argv, &endptr, 0);
1398	if (*endptr) {
1399	p_err(fmt: "can't parse %s as output context size",
1400	*argv);
1401	return -1;
1402	}
1403	NEXT_ARG();
1404	} else if (is_prefix(pfx: *argv, str: "repeat")) {
1405	char *endptr;
1406
1407	NEXT_ARG();
1408	if (!REQ_ARGS(1))
1409	return -1;
1410
1411	repeat = strtoul(*argv, &endptr, 0);
1412	if (*endptr) {
1413	p_err(fmt: "can't parse %s as repeat number",
1414	*argv);
1415	return -1;
1416	}
1417	NEXT_ARG();
1418	} else {
1419	p_err(fmt: "expected no more arguments, 'data_in', 'data_out', 'data_size_out', 'ctx_in', 'ctx_out', 'ctx_size_out' or 'repeat', got: '%s'?",
1420	*argv);
1421	return -1;
1422	}
1423	}
1424
1425	err = get_run_data(data_fname_in, &data_in, &test_attr.data_size_in);
1426	if (err)
1427	return -1;
1428
1429	if (data_in) {
1430	if (!test_attr.data_size_out)
1431	test_attr.data_size_out = default_size;
1432	err = alloc_run_data(&data_out, test_attr.data_size_out);
1433	if (err)
1434	goto free_data_in;
1435	}
1436
1437	err = get_run_data(ctx_fname_in, &ctx_in, &test_attr.ctx_size_in);
1438	if (err)
1439	goto free_data_out;
1440
1441	if (ctx_in) {
1442	if (!test_attr.ctx_size_out)
1443	test_attr.ctx_size_out = default_size;
1444	err = alloc_run_data(&ctx_out, test_attr.ctx_size_out);
1445	if (err)
1446	goto free_ctx_in;
1447	}
1448
1449	test_attr.repeat = repeat;
1450	test_attr.data_in = data_in;
1451	test_attr.data_out = data_out;
1452	test_attr.ctx_in = ctx_in;
1453	test_attr.ctx_out = ctx_out;
1454
1455	err = bpf_prog_test_run_opts(fd, &test_attr);
1456	if (err) {
1457	p_err("failed to run program: %s", strerror(errno));
1458	goto free_ctx_out;
1459	}
1460
1461	err = 0;
1462
1463	if (json_output)
1464	jsonw_start_object(self: json_wtr); /* root */
1465
1466	/* Do not exit on errors occurring when printing output data/context,
1467	* we still want to print return value and duration for program run.
1468	*/
1469	if (test_attr.data_size_out)
1470	err += print_run_output(test_attr.data_out,
1471	test_attr.data_size_out,
1472	data_fname_out, "data_out");
1473	if (test_attr.ctx_size_out)
1474	err += print_run_output(test_attr.ctx_out,
1475	test_attr.ctx_size_out,
1476	ctx_fname_out, "ctx_out");
1477
1478	if (json_output) {
1479	jsonw_uint_field(json_wtr, "retval", test_attr.retval);
1480	jsonw_uint_field(json_wtr, "duration", test_attr.duration);
1481	jsonw_end_object(self: json_wtr); /* root */
1482	} else {
1483	fprintf(stdout, "Return value: %u, duration%s: %uns\n",
1484	test_attr.retval,
1485	repeat > 1 ? " (average)" : "", test_attr.duration);
1486	}
1487
1488	free_ctx_out:
1489	free(ctx_out);
1490	free_ctx_in:
1491	free(ctx_in);
1492	free_data_out:
1493	free(data_out);
1494	free_data_in:
1495	free(data_in);
1496
1497	return err;
1498	}
1499
1500	static int
1501	get_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
1502	enum bpf_attach_type *expected_attach_type)
1503	{
1504	libbpf_print_fn_t print_backup;
1505	int ret;
1506
1507	ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type);
1508	if (!ret)
1509	return ret;
1510
1511	/* libbpf_prog_type_by_name() failed, let's re-run with debug level */
1512	print_backup = libbpf_set_print(print_all_levels);
1513	ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type);
1514	libbpf_set_print(print_backup);
1515
1516	return ret;
1517	}
1518
1519	static int
1520	auto_attach_program(struct bpf_program *prog, const char *path)
1521	{
1522	struct bpf_link *link;
1523	int err;
1524
1525	link = bpf_program__attach(prog);
1526	if (!link) {
1527	p_info(fmt: "Program %s does not support autoattach, falling back to pinning",
1528	bpf_program__name(prog));
1529	return bpf_obj_pin(bpf_program__fd(prog), path);
1530	}
1531
1532	err = bpf_link__pin(link, path);
1533	bpf_link__destroy(link);
1534	return err;
1535	}
1536
1537	static int
1538	auto_attach_programs(struct bpf_object *obj, const char *path)
1539	{
1540	struct bpf_program *prog;
1541	char buf[PATH_MAX];
1542	int err;
1543
1544	bpf_object__for_each_program(prog, obj) {
1545	err = pathname_concat(buf, buf_sz: sizeof(buf), path, name: bpf_program__name(prog));
1546	if (err)
1547	goto err_unpin_programs;
1548
1549	err = auto_attach_program(prog, path: buf);
1550	if (err)
1551	goto err_unpin_programs;
1552	}
1553
1554	return 0;
1555
1556	err_unpin_programs:
1557	while ((prog = bpf_object__prev_program(obj, prog))) {
1558	if (pathname_concat(buf, buf_sz: sizeof(buf), path, name: bpf_program__name(prog)))
1559	continue;
1560
1561	bpf_program__unpin(prog, buf);
1562	}
1563
1564	return err;
1565	}
1566
1567	static int load_with_options(int argc, char **argv, bool first_prog_only)
1568	{
1569	enum bpf_prog_type common_prog_type = BPF_PROG_TYPE_UNSPEC;
1570	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
1571	.relaxed_maps = relaxed_maps,
1572	);
1573	enum bpf_attach_type expected_attach_type;
1574	struct map_replace *map_replace = NULL;
1575	struct bpf_program *prog = NULL, *pos;
1576	unsigned int old_map_fds = 0;
1577	const char *pinmaps = NULL;
1578	__u32 xdpmeta_ifindex = 0;
1579	__u32 offload_ifindex = 0;
1580	bool auto_attach = false;
1581	struct bpf_object *obj;
1582	struct bpf_map *map;
1583	const char *pinfile;
1584	unsigned int i, j;
1585	const char *file;
1586	int idx, err;
1587
1588
1589	if (!REQ_ARGS(2))
1590	return -1;
1591	file = GET_ARG();
1592	pinfile = GET_ARG();
1593
1594	while (argc) {
1595	if (is_prefix(pfx: *argv, str: "type")) {
1596	NEXT_ARG();
1597
1598	if (common_prog_type != BPF_PROG_TYPE_UNSPEC) {
1599	p_err(fmt: "program type already specified");
1600	goto err_free_reuse_maps;
1601	}
1602	if (!REQ_ARGS(1))
1603	goto err_free_reuse_maps;
1604
1605	err = libbpf_prog_type_by_name(*argv, &common_prog_type,
1606	&expected_attach_type);
1607	if (err < 0) {
1608	/* Put a '/' at the end of type to appease libbpf */
1609	char *type = malloc(strlen(*argv) + 2);
1610
1611	if (!type) {
1612	p_err(fmt: "mem alloc failed");
1613	goto err_free_reuse_maps;
1614	}
1615	*type = 0;
1616	strcat(p: type, q: *argv);
1617	strcat(p: type, q: "/");
1618
1619	err = get_prog_type_by_name(name: type, prog_type: &common_prog_type,
1620	expected_attach_type: &expected_attach_type);
1621	free(type);
1622	if (err < 0)
1623	goto err_free_reuse_maps;
1624	}
1625
1626	NEXT_ARG();
1627	} else if (is_prefix(pfx: *argv, str: "map")) {
1628	void *new_map_replace;
1629	char *endptr, *name;
1630	int fd;
1631
1632	NEXT_ARG();
1633
1634	if (!REQ_ARGS(4))
1635	goto err_free_reuse_maps;
1636
1637	if (is_prefix(pfx: *argv, str: "idx")) {
1638	NEXT_ARG();
1639
1640	idx = strtoul(*argv, &endptr, 0);
1641	if (*endptr) {
1642	p_err(fmt: "can't parse %s as IDX", *argv);
1643	goto err_free_reuse_maps;
1644	}
1645	name = NULL;
1646	} else if (is_prefix(pfx: *argv, str: "name")) {
1647	NEXT_ARG();
1648
1649	name = *argv;
1650	idx = -1;
1651	} else {
1652	p_err(fmt: "expected 'idx' or 'name', got: '%s'?",
1653	*argv);
1654	goto err_free_reuse_maps;
1655	}
1656	NEXT_ARG();
1657
1658	fd = map_parse_fd(argc: &argc, argv: &argv, open_flags: 0);
1659	if (fd < 0)
1660	goto err_free_reuse_maps;
1661
1662	new_map_replace = libbpf_reallocarray(map_replace,
1663	old_map_fds + 1,
1664	sizeof(*map_replace));
1665	if (!new_map_replace) {
1666	p_err(fmt: "mem alloc failed");
1667	goto err_free_reuse_maps;
1668	}
1669	map_replace = new_map_replace;
1670
1671	map_replace[old_map_fds].idx = idx;
1672	map_replace[old_map_fds].name = name;
1673	map_replace[old_map_fds].fd = fd;
1674	old_map_fds++;
1675	} else if (is_prefix(pfx: *argv, str: "dev")) {
1676	p_info(fmt: "Warning: 'bpftool prog load [...] dev <ifname>' syntax is deprecated.\n"
1677	"Going further, please use 'offload_dev <ifname>' to offload program to device.\n"
1678	"For applications using XDP hints only, use 'xdpmeta_dev <ifname>'.");
1679	goto offload_dev;
1680	} else if (is_prefix(pfx: *argv, str: "offload_dev")) {
1681	offload_dev:
1682	NEXT_ARG();
1683
1684	if (offload_ifindex) {
1685	p_err(fmt: "offload_dev already specified");
1686	goto err_free_reuse_maps;
1687	} else if (xdpmeta_ifindex) {
1688	p_err(fmt: "xdpmeta_dev and offload_dev are mutually exclusive");
1689	goto err_free_reuse_maps;
1690	}
1691	if (!REQ_ARGS(1))
1692	goto err_free_reuse_maps;
1693
1694	offload_ifindex = if_nametoindex(*argv);
1695	if (!offload_ifindex) {
1696	p_err("unrecognized netdevice '%s': %s",
1697	*argv, strerror(errno));
1698	goto err_free_reuse_maps;
1699	}
1700	NEXT_ARG();
1701	} else if (is_prefix(pfx: *argv, str: "xdpmeta_dev")) {
1702	NEXT_ARG();
1703
1704	if (xdpmeta_ifindex) {
1705	p_err(fmt: "xdpmeta_dev already specified");
1706	goto err_free_reuse_maps;
1707	} else if (offload_ifindex) {
1708	p_err(fmt: "xdpmeta_dev and offload_dev are mutually exclusive");
1709	goto err_free_reuse_maps;
1710	}
1711	if (!REQ_ARGS(1))
1712	goto err_free_reuse_maps;
1713
1714	xdpmeta_ifindex = if_nametoindex(*argv);
1715	if (!xdpmeta_ifindex) {
1716	p_err("unrecognized netdevice '%s': %s",
1717	*argv, strerror(errno));
1718	goto err_free_reuse_maps;
1719	}
1720	NEXT_ARG();
1721	} else if (is_prefix(pfx: *argv, str: "pinmaps")) {
1722	NEXT_ARG();
1723
1724	if (!REQ_ARGS(1))
1725	goto err_free_reuse_maps;
1726
1727	pinmaps = GET_ARG();
1728	} else if (is_prefix(pfx: *argv, str: "autoattach")) {
1729	auto_attach = true;
1730	NEXT_ARG();
1731	} else if (is_prefix(pfx: *argv, str: "kernel_btf")) {
1732	NEXT_ARG();
1733
1734	if (!REQ_ARGS(1))
1735	goto err_free_reuse_maps;
1736
1737	open_opts.btf_custom_path = GET_ARG();
1738	} else {
1739	p_err(fmt: "expected no more arguments, "
1740	"'type', 'map', 'offload_dev', 'xdpmeta_dev', 'pinmaps', "
1741	"'autoattach', or 'kernel_btf', got: '%s'?",
1742	*argv);
1743	goto err_free_reuse_maps;
1744	}
1745	}
1746
1747	set_max_rlimit();
1748
1749	if (verifier_logs)
1750	/* log_level1 + log_level2 + stats, but not stable UAPI */
1751	open_opts.kernel_log_level = 1 + 2 + 4;
1752
1753	obj = bpf_object__open_file(file, &open_opts);
1754	if (!obj) {
1755	p_err(fmt: "failed to open object file");
1756	goto err_free_reuse_maps;
1757	}
1758
1759	bpf_object__for_each_program(pos, obj) {
1760	enum bpf_prog_type prog_type = common_prog_type;
1761
1762	if (prog_type == BPF_PROG_TYPE_UNSPEC) {
1763	const char *sec_name = bpf_program__section_name(pos);
1764
1765	err = get_prog_type_by_name(name: sec_name, prog_type: &prog_type,
1766	expected_attach_type: &expected_attach_type);
1767	if (err < 0)
1768	goto err_close_obj;
1769	}
1770
1771	if (prog_type == BPF_PROG_TYPE_XDP && xdpmeta_ifindex) {
1772	bpf_program__set_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY);
1773	bpf_program__set_ifindex(pos, xdpmeta_ifindex);
1774	} else {
1775	bpf_program__set_ifindex(pos, offload_ifindex);
1776	}
1777	if (bpf_program__type(pos) != prog_type)
1778	bpf_program__set_type(pos, prog_type);
1779	bpf_program__set_expected_attach_type(pos, expected_attach_type);
1780	}
1781
1782	qsort(map_replace, old_map_fds, sizeof(*map_replace),
1783	map_replace_compar);
1784
1785	/* After the sort maps by name will be first on the list, because they
1786	* have idx == -1. Resolve them.
1787	*/
1788	j = 0;
1789	while (j < old_map_fds && map_replace[j].name) {
1790	i = 0;
1791	bpf_object__for_each_map(map, obj) {
1792	if (!strcmp(bpf_map__name(map), map_replace[j].name)) {
1793	map_replace[j].idx = i;
1794	break;
1795	}
1796	i++;
1797	}
1798	if (map_replace[j].idx == -1) {
1799	p_err(fmt: "unable to find map '%s'", map_replace[j].name);
1800	goto err_close_obj;
1801	}
1802	j++;
1803	}
1804	/* Resort if any names were resolved */
1805	if (j)
1806	qsort(map_replace, old_map_fds, sizeof(*map_replace),
1807	map_replace_compar);
1808
1809	/* Set ifindex and name reuse */
1810	j = 0;
1811	idx = 0;
1812	bpf_object__for_each_map(map, obj) {
1813	if (bpf_map__type(map) != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
1814	bpf_map__set_ifindex(map, offload_ifindex);
1815
1816	if (j < old_map_fds && idx == map_replace[j].idx) {
1817	err = bpf_map__reuse_fd(map, map_replace[j++].fd);
1818	if (err) {
1819	p_err(fmt: "unable to set up map reuse: %d", err);
1820	goto err_close_obj;
1821	}
1822
1823	/* Next reuse wants to apply to the same map */
1824	if (j < old_map_fds && map_replace[j].idx == idx) {
1825	p_err(fmt: "replacement for map idx %d specified more than once",
1826	idx);
1827	goto err_close_obj;
1828	}
1829	}
1830
1831	idx++;
1832	}
1833	if (j < old_map_fds) {
1834	p_err(fmt: "map idx '%d' not used", map_replace[j].idx);
1835	goto err_close_obj;
1836	}
1837
1838	err = bpf_object__load(obj);
1839	if (err) {
1840	p_err(fmt: "failed to load object file");
1841	goto err_close_obj;
1842	}
1843
1844	if (first_prog_only)
1845	err = mount_bpffs_for_file(file_name: pinfile);
1846	else
1847	err = create_and_mount_bpffs_dir(dir_name: pinfile);
1848	if (err)
1849	goto err_close_obj;
1850
1851	if (first_prog_only) {
1852	prog = bpf_object__next_program(obj, NULL);
1853	if (!prog) {
1854	p_err(fmt: "object file doesn't contain any bpf program");
1855	goto err_close_obj;
1856	}
1857
1858	if (auto_attach)
1859	err = auto_attach_program(prog, path: pinfile);
1860	else
1861	err = bpf_obj_pin(bpf_program__fd(prog), pinfile);
1862	if (err) {
1863	p_err(fmt: "failed to pin program %s",
1864	bpf_program__section_name(prog));
1865	goto err_close_obj;
1866	}
1867	} else {
1868	if (auto_attach)
1869	err = auto_attach_programs(obj, path: pinfile);
1870	else
1871	err = bpf_object__pin_programs(obj, pinfile);
1872	if (err) {
1873	p_err(fmt: "failed to pin all programs");
1874	goto err_close_obj;
1875	}
1876	}
1877
1878	if (pinmaps) {
1879	err = create_and_mount_bpffs_dir(dir_name: pinmaps);
1880	if (err)
1881	goto err_unpin;
1882
1883	err = bpf_object__pin_maps(obj, pinmaps);
1884	if (err) {
1885	p_err(fmt: "failed to pin all maps");
1886	goto err_unpin;
1887	}
1888	}
1889
1890	if (json_output)
1891	jsonw_null(self: json_wtr);
1892
1893	bpf_object__close(obj);
1894	for (i = 0; i < old_map_fds; i++)
1895	close(map_replace[i].fd);
1896	free(map_replace);
1897
1898	return 0;
1899
1900	err_unpin:
1901	if (first_prog_only)
1902	unlink(pinfile);
1903	else
1904	bpf_object__unpin_programs(obj, pinfile);
1905	err_close_obj:
1906	bpf_object__close(obj);
1907	err_free_reuse_maps:
1908	for (i = 0; i < old_map_fds; i++)
1909	close(map_replace[i].fd);
1910	free(map_replace);
1911	return -1;
1912	}
1913
1914	static int count_open_fds(void)
1915	{
1916	DIR *dp = opendir("/proc/self/fd");
1917	struct dirent *de;
1918	int cnt = -3;
1919
1920	if (!dp)
1921	return -1;
1922
1923	while ((de = readdir(dp)))
1924	cnt++;
1925
1926	closedir(dp);
1927	return cnt;
1928	}
1929
1930	static int try_loader(struct gen_loader_opts *gen)
1931	{
1932	struct bpf_load_and_run_opts opts = {};
1933	struct bpf_loader_ctx *ctx;
1934	char sig_buf[MAX_SIG_SIZE];
1935	__u8 prog_sha[SHA256_DIGEST_LENGTH];
1936	int ctx_sz = sizeof(*ctx) + 64 * max(sizeof(struct bpf_map_desc),
1937	sizeof(struct bpf_prog_desc));
1938	int log_buf_sz = (1u << 24) - 1;
1939	int err, fds_before, fd_delta;
1940	char *log_buf = NULL;
1941
1942	ctx = alloca(ctx_sz);
1943	memset(ctx, 0, ctx_sz);
1944	ctx->sz = ctx_sz;
1945	if (verifier_logs) {
1946	ctx->log_level = 1 + 2 + 4;
1947	ctx->log_size = log_buf_sz;
1948	log_buf = malloc(log_buf_sz);
1949	if (!log_buf)
1950	return -ENOMEM;
1951	ctx->log_buf = (long) log_buf;
1952	}
1953	opts.ctx = ctx;
1954	opts.data = gen->data;
1955	opts.data_sz = gen->data_sz;
1956	opts.insns = gen->insns;
1957	opts.insns_sz = gen->insns_sz;
1958	fds_before = count_open_fds();
1959
1960	if (sign_progs) {
1961	opts.excl_prog_hash = prog_sha;
1962	opts.excl_prog_hash_sz = sizeof(prog_sha);
1963	opts.signature = sig_buf;
1964	opts.signature_sz = MAX_SIG_SIZE;
1965	opts.keyring_id = KEY_SPEC_SESSION_KEYRING;
1966
1967	err = bpftool_prog_sign(opts: &opts);
1968	if (err < 0) {
1969	p_err(fmt: "failed to sign program");
1970	goto out;
1971	}
1972
1973	err = register_session_key(key_der_path: cert_path);
1974	if (err < 0) {
1975	p_err(fmt: "failed to add session key");
1976	goto out;
1977	}
1978	}
1979	err = bpf_load_and_run(&opts);
1980	fd_delta = count_open_fds() - fds_before;
1981	if (err < 0 || verifier_logs) {
1982	fprintf(stderr, "err %d\n%s\n%s", err, opts.errstr, log_buf);
1983	if (fd_delta && err < 0)
1984	fprintf(stderr, "loader prog leaked %d FDs\n",
1985	fd_delta);
1986	}
1987	out:
1988	free(log_buf);
1989	return err;
1990	}
1991
1992	static int do_loader(int argc, char **argv)
1993	{
1994	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts);
1995	DECLARE_LIBBPF_OPTS(gen_loader_opts, gen);
1996	struct bpf_object *obj;
1997	const char *file;
1998	int err = 0;
1999
2000	if (!REQ_ARGS(1))
2001	return -1;
2002	file = GET_ARG();
2003
2004	if (verifier_logs)
2005	/* log_level1 + log_level2 + stats, but not stable UAPI */
2006	open_opts.kernel_log_level = 1 + 2 + 4;
2007
2008	obj = bpf_object__open_file(file, &open_opts);
2009	if (!obj) {
2010	err = -1;
2011	p_err(fmt: "failed to open object file");
2012	goto err_close_obj;
2013	}
2014
2015	if (sign_progs)
2016	gen.gen_hash = true;
2017
2018	err = bpf_object__gen_loader(obj, &gen);
2019	if (err)
2020	goto err_close_obj;
2021
2022	err = bpf_object__load(obj);
2023	if (err) {
2024	p_err(fmt: "failed to load object file");
2025	goto err_close_obj;
2026	}
2027
2028	if (verifier_logs) {
2029	struct dump_data dd = {};
2030
2031	kernel_syms_load(dd: &dd);
2032	dump_xlated_plain(&dd, (void *)gen.insns, gen.insns_sz, false, false);
2033	kernel_syms_destroy(dd: &dd);
2034	}
2035	err = try_loader(&gen);
2036	err_close_obj:
2037	bpf_object__close(obj);
2038	return err;
2039	}
2040
2041	static int do_load(int argc, char **argv)
2042	{
2043	if (use_loader)
2044	return do_loader(argc, argv);
2045	return load_with_options(argc, argv, first_prog_only: true);
2046	}
2047
2048	static int do_loadall(int argc, char **argv)
2049	{
2050	return load_with_options(argc, argv, first_prog_only: false);
2051	}
2052
2053	#ifdef BPFTOOL_WITHOUT_SKELETONS
2054
2055	static int do_profile(int argc, char **argv)
2056	{
2057	p_err("bpftool prog profile command is not supported. Please build bpftool with clang >= 10.0.0");
2058	return 0;
2059	}
2060
2061	#else /* BPFTOOL_WITHOUT_SKELETONS */
2062
2063	#include "profiler.skel.h"
2064
2065	struct profile_metric {
2066	const char *name;
2067	struct bpf_perf_event_value val;
2068	struct perf_event_attr attr;
2069	bool selected;
2070
2071	/* calculate ratios like instructions per cycle */
2072	const int ratio_metric; /* 0 for N/A, 1 for index 0 (cycles) */
2073	const char *ratio_desc;
2074	const float ratio_mul;
2075	} metrics[] = {
2076	{
2077	.name = "cycles",
2078	.attr = {
2079	.type = PERF_TYPE_HARDWARE,
2080	.config = PERF_COUNT_HW_CPU_CYCLES,
2081	.exclude_user = 1,
2082	},
2083	},
2084	{
2085	.name = "instructions",
2086	.attr = {
2087	.type = PERF_TYPE_HARDWARE,
2088	.config = PERF_COUNT_HW_INSTRUCTIONS,
2089	.exclude_user = 1,
2090	},
2091	.ratio_metric = 1,
2092	.ratio_desc = "insns per cycle",
2093	.ratio_mul = 1.0,
2094	},
2095	{
2096	.name = "l1d_loads",
2097	.attr = {
2098	.type = PERF_TYPE_HW_CACHE,
2099	.config =
2100	PERF_COUNT_HW_CACHE_L1D |
2101	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2102	(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
2103	.exclude_user = 1,
2104	},
2105	},
2106	{
2107	.name = "llc_misses",
2108	.attr = {
2109	.type = PERF_TYPE_HW_CACHE,
2110	.config =
2111	PERF_COUNT_HW_CACHE_LL |
2112	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2113	(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2114	.exclude_user = 1
2115	},
2116	.ratio_metric = 2,
2117	.ratio_desc = "LLC misses per million insns",
2118	.ratio_mul = 1e6,
2119	},
2120	{
2121	.name = "itlb_misses",
2122	.attr = {
2123	.type = PERF_TYPE_HW_CACHE,
2124	.config =
2125	PERF_COUNT_HW_CACHE_ITLB |
2126	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2127	(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2128	.exclude_user = 1
2129	},
2130	.ratio_metric = 2,
2131	.ratio_desc = "itlb misses per million insns",
2132	.ratio_mul = 1e6,
2133	},
2134	{
2135	.name = "dtlb_misses",
2136	.attr = {
2137	.type = PERF_TYPE_HW_CACHE,
2138	.config =
2139	PERF_COUNT_HW_CACHE_DTLB |
2140	(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2141	(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2142	.exclude_user = 1
2143	},
2144	.ratio_metric = 2,
2145	.ratio_desc = "dtlb misses per million insns",
2146	.ratio_mul = 1e6,
2147	},
2148	};
2149
2150	static __u64 profile_total_count;
2151
2152	#define MAX_NUM_PROFILE_METRICS 4
2153
2154	static int profile_parse_metrics(int argc, char **argv)
2155	{
2156	unsigned int metric_cnt;
2157	int selected_cnt = 0;
2158	unsigned int i;
2159
2160	metric_cnt = ARRAY_SIZE(metrics);
2161
2162	while (argc > 0) {
2163	for (i = 0; i < metric_cnt; i++) {
2164	if (is_prefix(pfx: argv[0], str: metrics[i].name)) {
2165	if (!metrics[i].selected)
2166	selected_cnt++;
2167	metrics[i].selected = true;
2168	break;
2169	}
2170	}
2171	if (i == metric_cnt) {
2172	p_err(fmt: "unknown metric %s", argv[0]);
2173	return -1;
2174	}
2175	NEXT_ARG();
2176	}
2177	if (selected_cnt > MAX_NUM_PROFILE_METRICS) {
2178	p_err(fmt: "too many (%d) metrics, please specify no more than %d metrics at a time",
2179	selected_cnt, MAX_NUM_PROFILE_METRICS);
2180	return -1;
2181	}
2182	return selected_cnt;
2183	}
2184
2185	static void profile_read_values(struct profiler_bpf *obj)
2186	{
2187	__u32 m, cpu, num_cpu = obj->rodata->num_cpu;
2188	int reading_map_fd, count_map_fd;
2189	__u64 counts[num_cpu];
2190	__u32 key = 0;
2191	int err;
2192
2193	reading_map_fd = bpf_map__fd(obj->maps.accum_readings);
2194	count_map_fd = bpf_map__fd(obj->maps.counts);
2195	if (reading_map_fd < 0 || count_map_fd < 0) {
2196	p_err(fmt: "failed to get fd for map");
2197	return;
2198	}
2199
2200	err = bpf_map_lookup_elem(count_map_fd, &key, counts);
2201	if (err) {
2202	p_err("failed to read count_map: %s", strerror(errno));
2203	return;
2204	}
2205
2206	profile_total_count = 0;
2207	for (cpu = 0; cpu < num_cpu; cpu++)
2208	profile_total_count += counts[cpu];
2209
2210	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2211	struct bpf_perf_event_value values[num_cpu];
2212
2213	if (!metrics[m].selected)
2214	continue;
2215
2216	err = bpf_map_lookup_elem(reading_map_fd, &key, values);
2217	if (err) {
2218	p_err("failed to read reading_map: %s",
2219	strerror(errno));
2220	return;
2221	}
2222	for (cpu = 0; cpu < num_cpu; cpu++) {
2223	metrics[m].val.counter += values[cpu].counter;
2224	metrics[m].val.enabled += values[cpu].enabled;
2225	metrics[m].val.running += values[cpu].running;
2226	}
2227	key++;
2228	}
2229	}
2230
2231	static void profile_print_readings_json(void)
2232	{
2233	__u32 m;
2234
2235	jsonw_start_array(self: json_wtr);
2236	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2237	if (!metrics[m].selected)
2238	continue;
2239	jsonw_start_object(self: json_wtr);
2240	jsonw_string_field(self: json_wtr, prop: "metric", val: metrics[m].name);
2241	jsonw_lluint_field(self: json_wtr, prop: "run_cnt", num: profile_total_count);
2242	jsonw_lluint_field(self: json_wtr, prop: "value", num: metrics[m].val.counter);
2243	jsonw_lluint_field(self: json_wtr, prop: "enabled", num: metrics[m].val.enabled);
2244	jsonw_lluint_field(self: json_wtr, prop: "running", num: metrics[m].val.running);
2245
2246	jsonw_end_object(self: json_wtr);
2247	}
2248	jsonw_end_array(self: json_wtr);
2249	}
2250
2251	static void profile_print_readings_plain(void)
2252	{
2253	__u32 m;
2254
2255	printf("\n%18llu %-20s\n", profile_total_count, "run_cnt");
2256	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2257	struct bpf_perf_event_value *val = &metrics[m].val;
2258	int r;
2259
2260	if (!metrics[m].selected)
2261	continue;
2262	printf("%18llu %-20s", val->counter, metrics[m].name);
2263
2264	r = metrics[m].ratio_metric - 1;
2265	if (r >= 0 && metrics[r].selected &&
2266	metrics[r].val.counter > 0) {
2267	printf("# %8.2f %-30s",
2268	val->counter * metrics[m].ratio_mul /
2269	metrics[r].val.counter,
2270	metrics[m].ratio_desc);
2271	} else {
2272	printf("%-41s", "");
2273	}
2274
2275	if (val->enabled > val->running)
2276	printf("(%4.2f%%)",
2277	val->running * 100.0 / val->enabled);
2278	printf("\n");
2279	}
2280	}
2281
2282	static void profile_print_readings(void)
2283	{
2284	if (json_output)
2285	profile_print_readings_json();
2286	else
2287	profile_print_readings_plain();
2288	}
2289
2290	static char *profile_target_name(int tgt_fd)
2291	{
2292	struct bpf_func_info func_info = {};
2293	struct bpf_prog_info info = {};
2294	__u32 info_len = sizeof(info);
2295	const struct btf_type *t;
2296	__u32 func_info_rec_size;
2297	struct btf *btf = NULL;
2298	char *name = NULL;
2299	int err;
2300
2301	err = bpf_prog_get_info_by_fd(tgt_fd, &info, &info_len);
2302	if (err) {
2303	p_err(fmt: "failed to get info for prog FD %d", tgt_fd);
2304	goto out;
2305	}
2306
2307	if (info.btf_id == 0) {
2308	p_err(fmt: "prog FD %d doesn't have valid btf", tgt_fd);
2309	goto out;
2310	}
2311
2312	func_info_rec_size = info.func_info_rec_size;
2313	if (info.nr_func_info == 0) {
2314	p_err(fmt: "found 0 func_info for prog FD %d", tgt_fd);
2315	goto out;
2316	}
2317
2318	memset(&info, 0, sizeof(info));
2319	info.nr_func_info = 1;
2320	info.func_info_rec_size = func_info_rec_size;
2321	info.func_info = ptr_to_u64(ptr: &func_info);
2322
2323	err = bpf_prog_get_info_by_fd(tgt_fd, &info, &info_len);
2324	if (err) {
2325	p_err(fmt: "failed to get func_info for prog FD %d", tgt_fd);
2326	goto out;
2327	}
2328
2329	btf = btf__load_from_kernel_by_id(info.btf_id);
2330	if (!btf) {
2331	p_err(fmt: "failed to load btf for prog FD %d", tgt_fd);
2332	goto out;
2333	}
2334
2335	t = btf__type_by_id(btf, func_info.type_id);
2336	if (!t) {
2337	p_err(fmt: "btf %u doesn't have type %u",
2338	info.btf_id, func_info.type_id);
2339	goto out;
2340	}
2341	name = strdup(btf__name_by_offset(btf, t->name_off));
2342	out:
2343	btf__free(btf);
2344	return name;
2345	}
2346
2347	static struct profiler_bpf *profile_obj;
2348	static int profile_tgt_fd = -1;
2349	static char *profile_tgt_name;
2350	static int *profile_perf_events;
2351	static int profile_perf_event_cnt;
2352
2353	static void profile_close_perf_events(struct profiler_bpf *obj)
2354	{
2355	int i;
2356
2357	for (i = profile_perf_event_cnt - 1; i >= 0; i--)
2358	close(profile_perf_events[i]);
2359
2360	free(profile_perf_events);
2361	profile_perf_event_cnt = 0;
2362	}
2363
2364	static int profile_open_perf_event(int mid, int cpu, int map_fd)
2365	{
2366	int pmu_fd;
2367
2368	pmu_fd = syscall(__NR_perf_event_open, &metrics[mid].attr,
2369	-1 /*pid*/, cpu, -1 /*group_fd*/, 0);
2370	if (pmu_fd < 0) {
2371	if (errno == ENODEV) {
2372	p_info(fmt: "cpu %d may be offline, skip %s profiling.",
2373	cpu, metrics[mid].name);
2374	profile_perf_event_cnt++;
2375	return 0;
2376	}
2377	return -1;
2378	}
2379
2380	if (bpf_map_update_elem(map_fd,
2381	&profile_perf_event_cnt,
2382	&pmu_fd, BPF_ANY) ||
2383	ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) {
2384	close(pmu_fd);
2385	return -1;
2386	}
2387
2388	profile_perf_events[profile_perf_event_cnt++] = pmu_fd;
2389	return 0;
2390	}
2391
2392	static int profile_open_perf_events(struct profiler_bpf *obj)
2393	{
2394	unsigned int cpu, m;
2395	int map_fd;
2396
2397	profile_perf_events = calloc(
2398	obj->rodata->num_cpu * obj->rodata->num_metric, sizeof(int));
2399	if (!profile_perf_events) {
2400	p_err("failed to allocate memory for perf_event array: %s",
2401	strerror(errno));
2402	return -1;
2403	}
2404	map_fd = bpf_map__fd(obj->maps.events);
2405	if (map_fd < 0) {
2406	p_err(fmt: "failed to get fd for events map");
2407	return -1;
2408	}
2409
2410	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2411	if (!metrics[m].selected)
2412	continue;
2413	for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) {
2414	if (profile_open_perf_event(mid: m, cpu, map_fd)) {
2415	p_err(fmt: "failed to create event %s on cpu %u",
2416	metrics[m].name, cpu);
2417	return -1;
2418	}
2419	}
2420	}
2421	return 0;
2422	}
2423
2424	static void profile_print_and_cleanup(void)
2425	{
2426	profile_close_perf_events(obj: profile_obj);
2427	profile_read_values(obj: profile_obj);
2428	profile_print_readings();
2429	profiler_bpf__destroy(profile_obj);
2430
2431	close(profile_tgt_fd);
2432	free(profile_tgt_name);
2433	}
2434
2435	static void int_exit(int signo)
2436	{
2437	profile_print_and_cleanup();
2438	exit(0);
2439	}
2440
2441	static int do_profile(int argc, char **argv)
2442	{
2443	int num_metric, num_cpu, err = -1;
2444	struct bpf_program *prog;
2445	unsigned long duration;
2446	char *endptr;
2447
2448	/* we at least need two args for the prog and one metric */
2449	if (!REQ_ARGS(3))
2450	return -EINVAL;
2451
2452	/* parse target fd */
2453	profile_tgt_fd = prog_parse_fd(argc: &argc, argv: &argv);
2454	if (profile_tgt_fd < 0) {
2455	p_err(fmt: "failed to parse fd");
2456	return -1;
2457	}
2458
2459	/* parse profiling optional duration */
2460	if (argc > 2 && is_prefix(pfx: argv[0], str: "duration")) {
2461	NEXT_ARG();
2462	duration = strtoul(*argv, &endptr, 0);
2463	if (*endptr)
2464	usage();
2465	NEXT_ARG();
2466	} else {
2467	duration = UINT_MAX;
2468	}
2469
2470	num_metric = profile_parse_metrics(argc, argv);
2471	if (num_metric <= 0)
2472	goto out;
2473
2474	num_cpu = libbpf_num_possible_cpus();
2475	if (num_cpu <= 0) {
2476	p_err(fmt: "failed to identify number of CPUs");
2477	goto out;
2478	}
2479
2480	profile_obj = profiler_bpf__open();
2481	if (!profile_obj) {
2482	p_err(fmt: "failed to open and/or load BPF object");
2483	goto out;
2484	}
2485
2486	profile_obj->rodata->num_cpu = num_cpu;
2487	profile_obj->rodata->num_metric = num_metric;
2488
2489	/* adjust map sizes */
2490	bpf_map__set_max_entries(profile_obj->maps.events, num_metric * num_cpu);
2491	bpf_map__set_max_entries(profile_obj->maps.fentry_readings, num_metric);
2492	bpf_map__set_max_entries(profile_obj->maps.accum_readings, num_metric);
2493	bpf_map__set_max_entries(profile_obj->maps.counts, 1);
2494
2495	/* change target name */
2496	profile_tgt_name = profile_target_name(tgt_fd: profile_tgt_fd);
2497	if (!profile_tgt_name)
2498	goto out;
2499
2500	bpf_object__for_each_program(prog, profile_obj->obj) {
2501	err = bpf_program__set_attach_target(prog, profile_tgt_fd,
2502	profile_tgt_name);
2503	if (err) {
2504	p_err(fmt: "failed to set attach target\n");
2505	goto out;
2506	}
2507	}
2508
2509	set_max_rlimit();
2510	err = profiler_bpf__load(profile_obj);
2511	if (err) {
2512	p_err(fmt: "failed to load profile_obj");
2513	goto out;
2514	}
2515
2516	err = profile_open_perf_events(obj: profile_obj);
2517	if (err)
2518	goto out;
2519
2520	err = profiler_bpf__attach(profile_obj);
2521	if (err) {
2522	p_err(fmt: "failed to attach profile_obj");
2523	goto out;
2524	}
2525	signal(SIGINT, int_exit);
2526
2527	sleep(duration);
2528	profile_print_and_cleanup();
2529	return 0;
2530
2531	out:
2532	profile_close_perf_events(obj: profile_obj);
2533	if (profile_obj)
2534	profiler_bpf__destroy(profile_obj);
2535	close(profile_tgt_fd);
2536	free(profile_tgt_name);
2537	return err;
2538	}
2539
2540	#endif /* BPFTOOL_WITHOUT_SKELETONS */
2541
2542	static int do_help(int argc, char **argv)
2543	{
2544	if (json_output) {
2545	jsonw_null(self: json_wtr);
2546	return 0;
2547	}
2548
2549	fprintf(stderr,
2550	"Usage: %1$s %2$s { show | list } [PROG]\n"
2551	" %1$s %2$s dump xlated PROG [{ file FILE | [opcodes] [linum] [visual] }]\n"
2552	" %1$s %2$s dump jited PROG [{ file FILE | [opcodes] [linum] }]\n"
2553	" %1$s %2$s pin PROG FILE\n"
2554	" %1$s %2$s { load | loadall } OBJ PATH \\\n"
2555	" [type TYPE] [{ offload_dev | xdpmeta_dev } NAME] \\\n"
2556	" [map { idx IDX | name NAME } MAP]\\\n"
2557	" [pinmaps MAP_DIR]\n"
2558	" [autoattach]\n"
2559	" [kernel_btf BTF_FILE]\n"
2560	" %1$s %2$s attach PROG ATTACH_TYPE [MAP]\n"
2561	" %1$s %2$s detach PROG ATTACH_TYPE [MAP]\n"
2562	" %1$s %2$s run PROG \\\n"
2563	" data_in FILE \\\n"
2564	" [data_out FILE [data_size_out L]] \\\n"
2565	" [ctx_in FILE [ctx_out FILE [ctx_size_out M]]] \\\n"
2566	" [repeat N]\n"
2567	" %1$s %2$s profile PROG [duration DURATION] METRICs\n"
2568	" %1$s %2$s tracelog\n"
2569	" %1$s %2$s tracelog { stdout | stderr } PROG\n"
2570	" %1$s %2$s help\n"
2571	"\n"
2572	" " HELP_SPEC_MAP "\n"
2573	" " HELP_SPEC_PROGRAM "\n"
2574	" TYPE := { socket | kprobe | kretprobe | classifier | action |\n"
2575	" tracepoint | raw_tracepoint | xdp | perf_event | cgroup/skb |\n"
2576	" cgroup/sock | cgroup/dev | lwt_in | lwt_out | lwt_xmit |\n"
2577	" lwt_seg6local | sockops | sk_skb | sk_msg | lirc_mode2 |\n"
2578	" sk_reuseport | flow_dissector | cgroup/sysctl |\n"
2579	" cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n"
2580	" cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n"
2581	" cgroup/connect_unix | cgroup/getpeername4 | cgroup/getpeername6 |\n"
2582	" cgroup/getpeername_unix | cgroup/getsockname4 | cgroup/getsockname6 |\n"
2583	" cgroup/getsockname_unix | cgroup/sendmsg4 | cgroup/sendmsg6 |\n"
2584	" cgroup/sendmsg_unix | cgroup/recvmsg4 | cgroup/recvmsg6 | cgroup/recvmsg_unix |\n"
2585	" cgroup/getsockopt | cgroup/setsockopt | cgroup/sock_release |\n"
2586	" struct_ops | fentry | fexit | freplace | sk_lookup }\n"
2587	" ATTACH_TYPE := { sk_msg_verdict | sk_skb_verdict | sk_skb_stream_verdict |\n"
2588	" sk_skb_stream_parser | flow_dissector }\n"
2589	" METRIC := { cycles | instructions | l1d_loads | llc_misses | itlb_misses | dtlb_misses }\n"
2590	" " HELP_SPEC_OPTIONS " |\n"
2591	" {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} |\n"
2592	" {-L|--use-loader} | [ {-S|--sign } {-k} <private_key.pem> {-i} <certificate.x509> ] \n"
2593	"",
2594	bin_name, argv[-2]);
2595
2596	return 0;
2597	}
2598
2599	static const struct cmd cmds[] = {
2600	{ "show", do_show },
2601	{ "list", do_show },
2602	{ "help", do_help },
2603	{ "dump", do_dump },
2604	{ "pin", do_pin },
2605	{ "load", do_load },
2606	{ "loadall", do_loadall },
2607	{ "attach", do_attach },
2608	{ "detach", do_detach },
2609	{ "tracelog", do_tracelog_any },
2610	{ "run", do_run },
2611	{ "profile", do_profile },
2612	{ 0 }
2613	};
2614
2615	int do_prog(int argc, char **argv)
2616	{
2617	return cmd_select(cmds, argc, argv, help: do_help);
2618	}
2619