1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (c) 2020 Facebook */
3	#include <test_progs.h>
4	#include <unistd.h>
5	#include <sys/syscall.h>
6	#include <task_local_storage_helpers.h>
7	#include "bpf_iter_ipv6_route.skel.h"
8	#include "bpf_iter_netlink.skel.h"
9	#include "bpf_iter_bpf_map.skel.h"
10	#include "bpf_iter_tasks.skel.h"
11	#include "bpf_iter_task_stack.skel.h"
12	#include "bpf_iter_task_file.skel.h"
13	#include "bpf_iter_task_vmas.skel.h"
14	#include "bpf_iter_task_btf.skel.h"
15	#include "bpf_iter_tcp4.skel.h"
16	#include "bpf_iter_tcp6.skel.h"
17	#include "bpf_iter_udp4.skel.h"
18	#include "bpf_iter_udp6.skel.h"
19	#include "bpf_iter_unix.skel.h"
20	#include "bpf_iter_vma_offset.skel.h"
21	#include "bpf_iter_test_kern1.skel.h"
22	#include "bpf_iter_test_kern2.skel.h"
23	#include "bpf_iter_test_kern3.skel.h"
24	#include "bpf_iter_test_kern4.skel.h"
25	#include "bpf_iter_bpf_hash_map.skel.h"
26	#include "bpf_iter_bpf_percpu_hash_map.skel.h"
27	#include "bpf_iter_bpf_array_map.skel.h"
28	#include "bpf_iter_bpf_percpu_array_map.skel.h"
29	#include "bpf_iter_bpf_sk_storage_helpers.skel.h"
30	#include "bpf_iter_bpf_sk_storage_map.skel.h"
31	#include "bpf_iter_test_kern5.skel.h"
32	#include "bpf_iter_test_kern6.skel.h"
33	#include "bpf_iter_bpf_link.skel.h"
34	#include "bpf_iter_ksym.skel.h"
35	#include "bpf_iter_sockmap.skel.h"
36
37	static void test_btf_id_or_null(void)
38	{
39	struct bpf_iter_test_kern3 *skel;
40
41	skel = bpf_iter_test_kern3__open_and_load();
42	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
43	bpf_iter_test_kern3__destroy(skel);
44	return;
45	}
46	}
47
48	static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
49	{
50	struct bpf_link *link;
51	char buf[16] = {};
52	int iter_fd, len;
53
54	link = bpf_program__attach_iter(prog, opts);
55	if (!ASSERT_OK_PTR(link, "attach_iter"))
56	return;
57
58	iter_fd = bpf_iter_create(bpf_link__fd(link));
59	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
60	goto free_link;
61
62	/* not check contents, but ensure read() ends without error */
63	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
64	;
65	ASSERT_GE(len, 0, "read");
66
67	close(iter_fd);
68
69	free_link:
70	bpf_link__destroy(link);
71	}
72
73	static void do_dummy_read(struct bpf_program *prog)
74	{
75	do_dummy_read_opts(prog, opts: NULL);
76	}
77
78	static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
79	struct bpf_map *map)
80	{
81	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
82	union bpf_iter_link_info linfo;
83	struct bpf_link *link;
84	char buf[16] = {};
85	int iter_fd, len;
86
87	memset(&linfo, 0, sizeof(linfo));
88	linfo.map.map_fd = bpf_map__fd(map);
89	opts.link_info = &linfo;
90	opts.link_info_len = sizeof(linfo);
91	link = bpf_program__attach_iter(prog, &opts);
92	if (!ASSERT_OK_PTR(link, "attach_map_iter"))
93	return;
94
95	iter_fd = bpf_iter_create(bpf_link__fd(link));
96	if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
97	bpf_link__destroy(link);
98	return;
99	}
100
101	/* Close link and map fd prematurely */
102	bpf_link__destroy(link);
103	bpf_object__destroy_skeleton(*skel);
104	*skel = NULL;
105
106	/* Try to let map free work to run first if map is freed */
107	usleep(100);
108	/* Memory used by both sock map and sock local storage map are
109	* freed after two synchronize_rcu() calls, so wait for it
110	*/
111	kern_sync_rcu();
112	kern_sync_rcu();
113
114	/* Read after both map fd and link fd are closed */
115	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
116	;
117	ASSERT_GE(len, 0, "read_iterator");
118
119	close(iter_fd);
120	}
121
122	static int read_fd_into_buffer(int fd, char *buf, int size)
123	{
124	int bufleft = size;
125	int len;
126
127	do {
128	len = read(fd, buf, bufleft);
129	if (len > 0) {
130	buf += len;
131	bufleft -= len;
132	}
133	} while (len > 0);
134
135	return len < 0 ? len : size - bufleft;
136	}
137
138	static void test_ipv6_route(void)
139	{
140	struct bpf_iter_ipv6_route *skel;
141
142	skel = bpf_iter_ipv6_route__open_and_load();
143	if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
144	return;
145
146	do_dummy_read(prog: skel->progs.dump_ipv6_route);
147
148	bpf_iter_ipv6_route__destroy(skel);
149	}
150
151	static void test_netlink(void)
152	{
153	struct bpf_iter_netlink *skel;
154
155	skel = bpf_iter_netlink__open_and_load();
156	if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
157	return;
158
159	do_dummy_read(prog: skel->progs.dump_netlink);
160
161	bpf_iter_netlink__destroy(skel);
162	}
163
164	static void test_bpf_map(void)
165	{
166	struct bpf_iter_bpf_map *skel;
167
168	skel = bpf_iter_bpf_map__open_and_load();
169	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
170	return;
171
172	do_dummy_read(prog: skel->progs.dump_bpf_map);
173
174	bpf_iter_bpf_map__destroy(skel);
175	}
176
177	static void check_bpf_link_info(const struct bpf_program *prog)
178	{
179	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
180	union bpf_iter_link_info linfo;
181	struct bpf_link_info info = {};
182	struct bpf_link *link;
183	__u32 info_len;
184	int err;
185
186	memset(&linfo, 0, sizeof(linfo));
187	linfo.task.tid = getpid();
188	opts.link_info = &linfo;
189	opts.link_info_len = sizeof(linfo);
190
191	link = bpf_program__attach_iter(prog, &opts);
192	if (!ASSERT_OK_PTR(link, "attach_iter"))
193	return;
194
195	info_len = sizeof(info);
196	err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
197	ASSERT_OK(err, "bpf_link_get_info_by_fd");
198	ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
199
200	bpf_link__destroy(link);
201	}
202
203	static pthread_mutex_t do_nothing_mutex;
204
205	static void *do_nothing_wait(void *arg)
206	{
207	pthread_mutex_lock(&do_nothing_mutex);
208	pthread_mutex_unlock(&do_nothing_mutex);
209
210	pthread_exit(arg);
211	}
212
213	static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
214	int *num_unknown, int *num_known)
215	{
216	struct bpf_iter_tasks *skel;
217	pthread_t thread_id;
218	void *ret;
219
220	skel = bpf_iter_tasks__open_and_load();
221	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
222	return;
223
224	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
225
226	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
227	"pthread_create");
228
229	skel->bss->tid = sys_gettid();
230
231	do_dummy_read_opts(prog: skel->progs.dump_task, opts);
232
233	*num_unknown = skel->bss->num_unknown_tid;
234	*num_known = skel->bss->num_known_tid;
235
236	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
237	ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
238	"pthread_join");
239
240	bpf_iter_tasks__destroy(skel);
241	}
242
243	static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
244	{
245	int num_unknown_tid, num_known_tid;
246
247	test_task_common_nocheck(opts, num_unknown: &num_unknown_tid, num_known: &num_known_tid);
248	ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
249	ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
250	}
251
252	static void *run_test_task_tid(void *arg)
253	{
254	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
255	union bpf_iter_link_info linfo;
256	int num_unknown_tid, num_known_tid;
257
258	ASSERT_NEQ(getpid(), sys_gettid(), "check_new_thread_id");
259
260	memset(&linfo, 0, sizeof(linfo));
261	linfo.task.tid = sys_gettid();
262	opts.link_info = &linfo;
263	opts.link_info_len = sizeof(linfo);
264	test_task_common(&opts, 0, 1);
265
266	linfo.task.tid = 0;
267	linfo.task.pid = getpid();
268	/* This includes the parent thread, this thread, watchdog timer thread
269	* and the do_nothing_wait thread
270	*/
271	test_task_common(&opts, 3, 1);
272
273	test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
274	ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");
275	ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
276
277	return NULL;
278	}
279
280	static void test_task_tid(void)
281	{
282	pthread_t thread_id;
283
284	/* Create a new thread so pid and tid aren't the same */
285	ASSERT_OK(pthread_create(&thread_id, NULL, &run_test_task_tid, NULL),
286	"pthread_create");
287	ASSERT_FALSE(pthread_join(thread_id, NULL), "pthread_join");
288	}
289
290	static void test_task_pid(void)
291	{
292	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
293	union bpf_iter_link_info linfo;
294
295	memset(&linfo, 0, sizeof(linfo));
296	linfo.task.pid = getpid();
297	opts.link_info = &linfo;
298	opts.link_info_len = sizeof(linfo);
299
300	test_task_common(&opts, 2, 1);
301	}
302
303	static void test_task_pidfd(void)
304	{
305	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
306	union bpf_iter_link_info linfo;
307	int pidfd;
308
309	pidfd = sys_pidfd_open(getpid(), 0);
310	if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
311	return;
312
313	memset(&linfo, 0, sizeof(linfo));
314	linfo.task.pid_fd = pidfd;
315	opts.link_info = &linfo;
316	opts.link_info_len = sizeof(linfo);
317
318	test_task_common(&opts, 2, 1);
319
320	close(pidfd);
321	}
322
323	static void test_task_sleepable(void)
324	{
325	struct bpf_iter_tasks *skel;
326	int pid, status, err, data_pipe[2], finish_pipe[2], c = 0;
327	char *test_data = NULL;
328	char *test_data_long = NULL;
329	char *data[2];
330
331	if (!ASSERT_OK(pipe(data_pipe), "data_pipe") ||
332	!ASSERT_OK(pipe(finish_pipe), "finish_pipe"))
333	return;
334
335	skel = bpf_iter_tasks__open_and_load();
336	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
337	return;
338
339	pid = fork();
340	if (!ASSERT_GE(pid, 0, "fork"))
341	return;
342
343	if (pid == 0) {
344	/* child */
345	close(data_pipe[0]);
346	close(finish_pipe[1]);
347
348	test_data = malloc(sizeof(char) * 10);
349	strncpy(test_data, "test_data", 10);
350	test_data[9] = '\0';
351
352	test_data_long = malloc(sizeof(char) * 5000);
353	for (int i = 0; i < 5000; ++i) {
354	if (i % 2 == 0)
355	test_data_long[i] = 'b';
356	else
357	test_data_long[i] = 'a';
358	}
359	test_data_long[4999] = '\0';
360
361	data[0] = test_data;
362	data[1] = test_data_long;
363
364	write(data_pipe[1], &data, sizeof(data));
365
366	/* keep child alive until after the test */
367	err = read(finish_pipe[0], &c, 1);
368	if (err != 1)
369	exit(-1);
370
371	close(data_pipe[1]);
372	close(finish_pipe[0]);
373	_exit(0);
374	}
375
376	/* parent */
377	close(data_pipe[1]);
378	close(finish_pipe[0]);
379
380	err = read(data_pipe[0], &data, sizeof(data));
381	ASSERT_EQ(err, sizeof(data), "read_check");
382
383	skel->bss->user_ptr = data[0];
384	skel->bss->user_ptr_long = data[1];
385	skel->bss->pid = pid;
386
387	do_dummy_read(prog: skel->progs.dump_task_sleepable);
388
389	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
390	"num_expected_failure_copy_from_user_task");
391	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
392	"num_success_copy_from_user_task");
393	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task_str, 0,
394	"num_expected_failure_copy_from_user_task_str");
395	ASSERT_GT(skel->bss->num_success_copy_from_user_task_str, 0,
396	"num_success_copy_from_user_task_str");
397
398	bpf_iter_tasks__destroy(skel);
399
400	write(finish_pipe[1], &c, 1);
401	err = waitpid(pid, &status, 0);
402	ASSERT_EQ(err, pid, "waitpid");
403	ASSERT_EQ(status, 0, "zero_child_exit");
404
405	close(data_pipe[0]);
406	close(finish_pipe[1]);
407	}
408
409	static void test_task_stack(void)
410	{
411	struct bpf_iter_task_stack *skel;
412
413	skel = bpf_iter_task_stack__open_and_load();
414	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
415	return;
416
417	do_dummy_read(prog: skel->progs.dump_task_stack);
418	do_dummy_read(prog: skel->progs.get_task_user_stacks);
419
420	ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");
421
422	bpf_iter_task_stack__destroy(skel);
423	}
424
425	static void test_task_file(void)
426	{
427	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
428	struct bpf_iter_task_file *skel;
429	union bpf_iter_link_info linfo;
430	pthread_t thread_id;
431	void *ret;
432
433	skel = bpf_iter_task_file__open_and_load();
434	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
435	return;
436
437	skel->bss->tgid = getpid();
438
439	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
440
441	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
442	"pthread_create");
443
444	memset(&linfo, 0, sizeof(linfo));
445	linfo.task.tid = getpid();
446	opts.link_info = &linfo;
447	opts.link_info_len = sizeof(linfo);
448
449	do_dummy_read_opts(skel->progs.dump_task_file, &opts);
450
451	ASSERT_EQ(skel->bss->count, 0, "check_count");
452	ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
453
454	skel->bss->last_tgid = 0;
455	skel->bss->count = 0;
456	skel->bss->unique_tgid_count = 0;
457
458	do_dummy_read(prog: skel->progs.dump_task_file);
459
460	ASSERT_EQ(skel->bss->count, 0, "check_count");
461	ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
462
463	check_bpf_link_info(prog: skel->progs.dump_task_file);
464
465	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
466	ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
467	ASSERT_NULL(ret, "pthread_join");
468
469	bpf_iter_task_file__destroy(skel);
470	}
471
472	#define TASKBUFSZ 32768
473
474	static char taskbuf[TASKBUFSZ];
475
476	static int do_btf_read(struct bpf_iter_task_btf *skel)
477	{
478	struct bpf_program *prog = skel->progs.dump_task_struct;
479	struct bpf_iter_task_btf__bss *bss = skel->bss;
480	int iter_fd = -1, err;
481	struct bpf_link *link;
482	char *buf = taskbuf;
483	int ret = 0;
484
485	link = bpf_program__attach_iter(prog, NULL);
486	if (!ASSERT_OK_PTR(link, "attach_iter"))
487	return ret;
488
489	iter_fd = bpf_iter_create(bpf_link__fd(link));
490	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
491	goto free_link;
492
493	err = read_fd_into_buffer(fd: iter_fd, buf, TASKBUFSZ);
494	if (bss->skip) {
495	printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
496	ret = 1;
497	test__skip();
498	goto free_link;
499	}
500
501	if (!ASSERT_GE(err, 0, "read"))
502	goto free_link;
503
504	ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
505	"check for btf representation of task_struct in iter data");
506	free_link:
507	if (iter_fd > 0)
508	close(iter_fd);
509	bpf_link__destroy(link);
510	return ret;
511	}
512
513	static void test_task_btf(void)
514	{
515	struct bpf_iter_task_btf__bss *bss;
516	struct bpf_iter_task_btf *skel;
517	int ret;
518
519	skel = bpf_iter_task_btf__open_and_load();
520	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
521	return;
522
523	bss = skel->bss;
524
525	ret = do_btf_read(skel);
526	if (ret)
527	goto cleanup;
528
529	if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
530	goto cleanup;
531
532	ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");
533
534	cleanup:
535	bpf_iter_task_btf__destroy(skel);
536	}
537
538	static void test_tcp4(void)
539	{
540	struct bpf_iter_tcp4 *skel;
541
542	skel = bpf_iter_tcp4__open_and_load();
543	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
544	return;
545
546	do_dummy_read(prog: skel->progs.dump_tcp4);
547
548	bpf_iter_tcp4__destroy(skel);
549	}
550
551	static void test_tcp6(void)
552	{
553	struct bpf_iter_tcp6 *skel;
554
555	skel = bpf_iter_tcp6__open_and_load();
556	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
557	return;
558
559	do_dummy_read(prog: skel->progs.dump_tcp6);
560
561	bpf_iter_tcp6__destroy(skel);
562	}
563
564	static void test_udp4(void)
565	{
566	struct bpf_iter_udp4 *skel;
567
568	skel = bpf_iter_udp4__open_and_load();
569	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
570	return;
571
572	do_dummy_read(prog: skel->progs.dump_udp4);
573
574	bpf_iter_udp4__destroy(skel);
575	}
576
577	static void test_udp6(void)
578	{
579	struct bpf_iter_udp6 *skel;
580
581	skel = bpf_iter_udp6__open_and_load();
582	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
583	return;
584
585	do_dummy_read(prog: skel->progs.dump_udp6);
586
587	bpf_iter_udp6__destroy(skel);
588	}
589
590	static void test_unix(void)
591	{
592	struct bpf_iter_unix *skel;
593
594	skel = bpf_iter_unix__open_and_load();
595	if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
596	return;
597
598	do_dummy_read(prog: skel->progs.dump_unix);
599
600	bpf_iter_unix__destroy(skel);
601	}
602
603	/* The expected string is less than 16 bytes */
604	static int do_read_with_fd(int iter_fd, const char *expected,
605	bool read_one_char)
606	{
607	int len, read_buf_len, start;
608	char buf[16] = {};
609
610	read_buf_len = read_one_char ? 1 : 16;
611	start = 0;
612	while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
613	start += len;
614	if (!ASSERT_LT(start, 16, "read"))
615	return -1;
616	read_buf_len = read_one_char ? 1 : 16 - start;
617	}
618	if (!ASSERT_GE(len, 0, "read"))
619	return -1;
620
621	if (!ASSERT_STREQ(buf, expected, "read"))
622	return -1;
623
624	return 0;
625	}
626
627	static void test_anon_iter(bool read_one_char)
628	{
629	struct bpf_iter_test_kern1 *skel;
630	struct bpf_link *link;
631	int iter_fd, err;
632
633	skel = bpf_iter_test_kern1__open_and_load();
634	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
635	return;
636
637	err = bpf_iter_test_kern1__attach(skel);
638	if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
639	goto out;
640	}
641
642	link = skel->links.dump_task;
643	iter_fd = bpf_iter_create(bpf_link__fd(link));
644	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
645	goto out;
646
647	do_read_with_fd(iter_fd, "abcd", read_one_char);
648	close(iter_fd);
649
650	out:
651	bpf_iter_test_kern1__destroy(skel);
652	}
653
654	static int do_read(const char *path, const char *expected)
655	{
656	int err, iter_fd;
657
658	iter_fd = open(path, O_RDONLY);
659	if (!ASSERT_GE(iter_fd, 0, "open"))
660	return -1;
661
662	err = do_read_with_fd(iter_fd, expected, false);
663	close(iter_fd);
664	return err;
665	}
666
667	static void test_file_iter(void)
668	{
669	const char *path = "/sys/fs/bpf/bpf_iter_test1";
670	struct bpf_iter_test_kern1 *skel1;
671	struct bpf_iter_test_kern2 *skel2;
672	struct bpf_link *link;
673	int err;
674
675	skel1 = bpf_iter_test_kern1__open_and_load();
676	if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
677	return;
678
679	link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
680	if (!ASSERT_OK_PTR(link, "attach_iter"))
681	goto out;
682
683	/* unlink this path if it exists. */
684	unlink(path);
685
686	err = bpf_link__pin(link, path);
687	if (!ASSERT_OK(err, "pin_iter"))
688	goto free_link;
689
690	err = do_read(path, expected: "abcd");
691	if (err)
692	goto unlink_path;
693
694	/* file based iterator seems working fine. Let us a link update
695	* of the underlying link and `cat` the iterator again, its content
696	* should change.
697	*/
698	skel2 = bpf_iter_test_kern2__open_and_load();
699	if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
700	goto unlink_path;
701
702	err = bpf_link__update_program(link, skel2->progs.dump_task);
703	if (!ASSERT_OK(err, "update_prog"))
704	goto destroy_skel2;
705
706	do_read(path, expected: "ABCD");
707
708	destroy_skel2:
709	bpf_iter_test_kern2__destroy(skel2);
710	unlink_path:
711	unlink(path);
712	free_link:
713	bpf_link__destroy(link);
714	out:
715	bpf_iter_test_kern1__destroy(skel1);
716	}
717
718	static void test_overflow(bool test_e2big_overflow, bool ret1)
719	{
720	__u32 map_info_len, total_read_len, expected_read_len;
721	int err, iter_fd, map1_fd, map2_fd, len;
722	struct bpf_map_info map_info = {};
723	struct bpf_iter_test_kern4 *skel;
724	struct bpf_link *link;
725	__u32 iter_size;
726	char *buf;
727
728	skel = bpf_iter_test_kern4__open();
729	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
730	return;
731
732	/* create two maps: bpf program will only do bpf_seq_write
733	* for these two maps. The goal is one map output almost
734	* fills seq_file buffer and then the other will trigger
735	* overflow and needs restart.
736	*/
737	map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
738	if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
739	goto out;
740	map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
741	if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
742	goto free_map1;
743
744	/* bpf_seq_printf kernel buffer is 8 pages, so one map
745	* bpf_seq_write will mostly fill it, and the other map
746	* will partially fill and then trigger overflow and need
747	* bpf_seq_read restart.
748	*/
749	iter_size = sysconf(_SC_PAGE_SIZE) << 3;
750
751	if (test_e2big_overflow) {
752	skel->rodata->print_len = (iter_size + 8) / 8;
753	expected_read_len = 2 * (iter_size + 8);
754	} else if (!ret1) {
755	skel->rodata->print_len = (iter_size - 8) / 8;
756	expected_read_len = 2 * (iter_size - 8);
757	} else {
758	skel->rodata->print_len = 1;
759	expected_read_len = 2 * 8;
760	}
761	skel->rodata->ret1 = ret1;
762
763	if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
764	"bpf_iter_test_kern4__load"))
765	goto free_map2;
766
767	/* setup filtering map_id in bpf program */
768	map_info_len = sizeof(map_info);
769	err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
770	if (!ASSERT_OK(err, "get_map_info"))
771	goto free_map2;
772	skel->bss->map1_id = map_info.id;
773
774	err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
775	if (!ASSERT_OK(err, "get_map_info"))
776	goto free_map2;
777	skel->bss->map2_id = map_info.id;
778
779	link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
780	if (!ASSERT_OK_PTR(link, "attach_iter"))
781	goto free_map2;
782
783	iter_fd = bpf_iter_create(bpf_link__fd(link));
784	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
785	goto free_link;
786
787	buf = malloc(expected_read_len);
788	if (!ASSERT_OK_PTR(buf, "malloc"))
789	goto close_iter;
790
791	/* do read */
792	total_read_len = 0;
793	if (test_e2big_overflow) {
794	while ((len = read(iter_fd, buf, expected_read_len)) > 0)
795	total_read_len += len;
796
797	ASSERT_EQ(len, -1, "read");
798	ASSERT_EQ(errno, E2BIG, "read");
799	goto free_buf;
800	} else if (!ret1) {
801	while ((len = read(iter_fd, buf, expected_read_len)) > 0)
802	total_read_len += len;
803
804	if (!ASSERT_GE(len, 0, "read"))
805	goto free_buf;
806	} else {
807	do {
808	len = read(iter_fd, buf, expected_read_len);
809	if (len > 0)
810	total_read_len += len;
811	} while (len > 0 || len == -EAGAIN);
812
813	if (!ASSERT_GE(len, 0, "read"))
814	goto free_buf;
815	}
816
817	if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
818	goto free_buf;
819
820	if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
821	goto free_buf;
822
823	if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
824	goto free_buf;
825
826	ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");
827
828	free_buf:
829	free(buf);
830	close_iter:
831	close(iter_fd);
832	free_link:
833	bpf_link__destroy(link);
834	free_map2:
835	close(map2_fd);
836	free_map1:
837	close(map1_fd);
838	out:
839	bpf_iter_test_kern4__destroy(skel);
840	}
841
842	static void test_bpf_hash_map(void)
843	{
844	__u32 expected_key_a = 0, expected_key_b = 0;
845	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
846	struct bpf_iter_bpf_hash_map *skel;
847	int err, i, len, map_fd, iter_fd;
848	union bpf_iter_link_info linfo;
849	__u64 val, expected_val = 0;
850	struct bpf_link *link;
851	struct key_t {
852	int a;
853	int b;
854	int c;
855	} key;
856	char buf[64];
857
858	skel = bpf_iter_bpf_hash_map__open();
859	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
860	return;
861
862	skel->bss->in_test_mode = true;
863
864	err = bpf_iter_bpf_hash_map__load(skel);
865	if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
866	goto out;
867
868	/* iterator with hashmap2 and hashmap3 should fail */
869	memset(&linfo, 0, sizeof(linfo));
870	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
871	opts.link_info = &linfo;
872	opts.link_info_len = sizeof(linfo);
873	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
874	if (!ASSERT_ERR_PTR(link, "attach_iter"))
875	goto out;
876
877	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
878	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
879	if (!ASSERT_ERR_PTR(link, "attach_iter"))
880	goto out;
881
882	/* hashmap1 should be good, update map values here */
883	map_fd = bpf_map__fd(skel->maps.hashmap1);
884	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
885	key.a = i + 1;
886	key.b = i + 2;
887	key.c = i + 3;
888	val = i + 4;
889	expected_key_a += key.a;
890	expected_key_b += key.b;
891	expected_val += val;
892
893	err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
894	if (!ASSERT_OK(err, "map_update"))
895	goto out;
896	}
897
898	/* Sleepable program is prohibited for hash map iterator */
899	linfo.map.map_fd = map_fd;
900	link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
901	if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
902	goto out;
903
904	linfo.map.map_fd = map_fd;
905	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
906	if (!ASSERT_OK_PTR(link, "attach_iter"))
907	goto out;
908
909	iter_fd = bpf_iter_create(bpf_link__fd(link));
910	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
911	goto free_link;
912
913	/* do some tests */
914	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
915	;
916	if (!ASSERT_GE(len, 0, "read"))
917	goto close_iter;
918
919	/* test results */
920	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
921	goto close_iter;
922	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
923	goto close_iter;
924	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
925	goto close_iter;
926
927	close_iter:
928	close(iter_fd);
929	free_link:
930	bpf_link__destroy(link);
931	out:
932	bpf_iter_bpf_hash_map__destroy(skel);
933	}
934
935	static void test_bpf_percpu_hash_map(void)
936	{
937	__u32 expected_key_a = 0, expected_key_b = 0;
938	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
939	struct bpf_iter_bpf_percpu_hash_map *skel;
940	int err, i, j, len, map_fd, iter_fd;
941	union bpf_iter_link_info linfo;
942	__u32 expected_val = 0;
943	struct bpf_link *link;
944	struct key_t {
945	int a;
946	int b;
947	int c;
948	} key;
949	char buf[64];
950	void *val;
951
952	skel = bpf_iter_bpf_percpu_hash_map__open();
953	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
954	return;
955
956	skel->rodata->num_cpus = bpf_num_possible_cpus();
957	val = malloc(8 * bpf_num_possible_cpus());
958	if (!ASSERT_OK_PTR(val, "malloc"))
959	goto out;
960
961	err = bpf_iter_bpf_percpu_hash_map__load(skel);
962	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
963	goto out;
964
965	/* update map values here */
966	map_fd = bpf_map__fd(skel->maps.hashmap1);
967	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
968	key.a = i + 1;
969	key.b = i + 2;
970	key.c = i + 3;
971	expected_key_a += key.a;
972	expected_key_b += key.b;
973
974	for (j = 0; j < bpf_num_possible_cpus(); j++) {
975	*(__u32 *)(val + j * 8) = i + j;
976	expected_val += i + j;
977	}
978
979	err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
980	if (!ASSERT_OK(err, "map_update"))
981	goto out;
982	}
983
984	memset(&linfo, 0, sizeof(linfo));
985	linfo.map.map_fd = map_fd;
986	opts.link_info = &linfo;
987	opts.link_info_len = sizeof(linfo);
988	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
989	if (!ASSERT_OK_PTR(link, "attach_iter"))
990	goto out;
991
992	iter_fd = bpf_iter_create(bpf_link__fd(link));
993	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
994	goto free_link;
995
996	/* do some tests */
997	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
998	;
999	if (!ASSERT_GE(len, 0, "read"))
1000	goto close_iter;
1001
1002	/* test results */
1003	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
1004	goto close_iter;
1005	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
1006	goto close_iter;
1007	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1008	goto close_iter;
1009
1010	close_iter:
1011	close(iter_fd);
1012	free_link:
1013	bpf_link__destroy(link);
1014	out:
1015	bpf_iter_bpf_percpu_hash_map__destroy(skel);
1016	free(val);
1017	}
1018
1019	static void test_bpf_array_map(void)
1020	{
1021	__u64 val, expected_val = 0, res_first_val, first_val = 0;
1022	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1023	__u32 key, expected_key = 0, res_first_key;
1024	int err, i, map_fd, hash_fd, iter_fd;
1025	struct bpf_iter_bpf_array_map *skel;
1026	union bpf_iter_link_info linfo;
1027	struct bpf_link *link;
1028	char buf[64] = {};
1029	int len, start;
1030
1031	skel = bpf_iter_bpf_array_map__open_and_load();
1032	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1033	return;
1034
1035	map_fd = bpf_map__fd(skel->maps.arraymap1);
1036	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1037	val = i + 4;
1038	expected_key += i;
1039	expected_val += val;
1040
1041	if (i == 0)
1042	first_val = val;
1043
1044	err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
1045	if (!ASSERT_OK(err, "map_update"))
1046	goto out;
1047	}
1048
1049	memset(&linfo, 0, sizeof(linfo));
1050	linfo.map.map_fd = map_fd;
1051	opts.link_info = &linfo;
1052	opts.link_info_len = sizeof(linfo);
1053	link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
1054	if (!ASSERT_OK_PTR(link, "attach_iter"))
1055	goto out;
1056
1057	iter_fd = bpf_iter_create(bpf_link__fd(link));
1058	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1059	goto free_link;
1060
1061	/* do some tests */
1062	start = 0;
1063	while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
1064	start += len;
1065	if (!ASSERT_GE(len, 0, "read"))
1066	goto close_iter;
1067
1068	/* test results */
1069	res_first_key = *(__u32 *)buf;
1070	res_first_val = *(__u64 *)(buf + sizeof(__u32));
1071	if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
1072	!ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
1073	goto close_iter;
1074
1075	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1076	goto close_iter;
1077	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1078	goto close_iter;
1079
1080	hash_fd = bpf_map__fd(skel->maps.hashmap1);
1081	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1082	err = bpf_map_lookup_elem(map_fd, &i, &val);
1083	if (!ASSERT_OK(err, "map_lookup arraymap1"))
1084	goto close_iter;
1085	if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
1086	goto close_iter;
1087
1088	val = i + 4;
1089	err = bpf_map_lookup_elem(hash_fd, &val, &key);
1090	if (!ASSERT_OK(err, "map_lookup hashmap1"))
1091	goto close_iter;
1092	if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
1093	goto close_iter;
1094	}
1095
1096	close_iter:
1097	close(iter_fd);
1098	free_link:
1099	bpf_link__destroy(link);
1100	out:
1101	bpf_iter_bpf_array_map__destroy(skel);
1102	}
1103
1104	static void test_bpf_array_map_iter_fd(void)
1105	{
1106	struct bpf_iter_bpf_array_map *skel;
1107
1108	skel = bpf_iter_bpf_array_map__open_and_load();
1109	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1110	return;
1111
1112	do_read_map_iter_fd(skel: &skel->skeleton, prog: skel->progs.dump_bpf_array_map,
1113	map: skel->maps.arraymap1);
1114
1115	bpf_iter_bpf_array_map__destroy(skel);
1116	}
1117
1118	static void test_bpf_percpu_array_map(void)
1119	{
1120	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1121	struct bpf_iter_bpf_percpu_array_map *skel;
1122	__u32 expected_key = 0, expected_val = 0;
1123	union bpf_iter_link_info linfo;
1124	int err, i, j, map_fd, iter_fd;
1125	struct bpf_link *link;
1126	char buf[64];
1127	void *val;
1128	int len;
1129
1130	skel = bpf_iter_bpf_percpu_array_map__open();
1131	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
1132	return;
1133
1134	skel->rodata->num_cpus = bpf_num_possible_cpus();
1135	val = malloc(8 * bpf_num_possible_cpus());
1136	if (!ASSERT_OK_PTR(val, "malloc"))
1137	goto out;
1138
1139	err = bpf_iter_bpf_percpu_array_map__load(skel);
1140	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
1141	goto out;
1142
1143	/* update map values here */
1144	map_fd = bpf_map__fd(skel->maps.arraymap1);
1145	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1146	expected_key += i;
1147
1148	for (j = 0; j < bpf_num_possible_cpus(); j++) {
1149	*(__u32 *)(val + j * 8) = i + j;
1150	expected_val += i + j;
1151	}
1152
1153	err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
1154	if (!ASSERT_OK(err, "map_update"))
1155	goto out;
1156	}
1157
1158	memset(&linfo, 0, sizeof(linfo));
1159	linfo.map.map_fd = map_fd;
1160	opts.link_info = &linfo;
1161	opts.link_info_len = sizeof(linfo);
1162	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
1163	if (!ASSERT_OK_PTR(link, "attach_iter"))
1164	goto out;
1165
1166	iter_fd = bpf_iter_create(bpf_link__fd(link));
1167	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1168	goto free_link;
1169
1170	/* do some tests */
1171	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1172	;
1173	if (!ASSERT_GE(len, 0, "read"))
1174	goto close_iter;
1175
1176	/* test results */
1177	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1178	goto close_iter;
1179	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1180	goto close_iter;
1181
1182	close_iter:
1183	close(iter_fd);
1184	free_link:
1185	bpf_link__destroy(link);
1186	out:
1187	bpf_iter_bpf_percpu_array_map__destroy(skel);
1188	free(val);
1189	}
1190
1191	/* An iterator program deletes all local storage in a map. */
1192	static void test_bpf_sk_storage_delete(void)
1193	{
1194	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1195	struct bpf_iter_bpf_sk_storage_helpers *skel;
1196	union bpf_iter_link_info linfo;
1197	int err, len, map_fd, iter_fd;
1198	struct bpf_link *link;
1199	int sock_fd = -1;
1200	__u32 val = 42;
1201	char buf[64];
1202
1203	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1204	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1205	return;
1206
1207	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1208
1209	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1210	if (!ASSERT_GE(sock_fd, 0, "socket"))
1211	goto out;
1212
1213	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1214	if (!ASSERT_OK(err, "map_update"))
1215	goto out;
1216
1217	memset(&linfo, 0, sizeof(linfo));
1218	linfo.map.map_fd = map_fd;
1219	opts.link_info = &linfo;
1220	opts.link_info_len = sizeof(linfo);
1221	link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
1222	&opts);
1223	if (!ASSERT_OK_PTR(link, "attach_iter"))
1224	goto out;
1225
1226	iter_fd = bpf_iter_create(bpf_link__fd(link));
1227	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1228	goto free_link;
1229
1230	/* do some tests */
1231	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1232	;
1233	if (!ASSERT_GE(len, 0, "read"))
1234	goto close_iter;
1235
1236	/* test results */
1237	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1238
1239	/* Note: The following assertions serve to ensure
1240	* the value was deleted. It does so by asserting
1241	* that bpf_map_lookup_elem has failed. This might
1242	* seem counterintuitive at first.
1243	*/
1244	ASSERT_ERR(err, "bpf_map_lookup_elem");
1245	ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");
1246
1247	close_iter:
1248	close(iter_fd);
1249	free_link:
1250	bpf_link__destroy(link);
1251	out:
1252	if (sock_fd >= 0)
1253	close(sock_fd);
1254	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1255	}
1256
1257	/* This creates a socket and its local storage. It then runs a task_iter BPF
1258	* program that replaces the existing socket local storage with the tgid of the
1259	* only task owning a file descriptor to this socket, this process, prog_tests.
1260	* It then runs a tcp socket iterator that negates the value in the existing
1261	* socket local storage, the test verifies that the resulting value is -pid.
1262	*/
1263	static void test_bpf_sk_storage_get(void)
1264	{
1265	struct bpf_iter_bpf_sk_storage_helpers *skel;
1266	int err, map_fd, val = -1;
1267	int sock_fd = -1;
1268
1269	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1270	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1271	return;
1272
1273	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1274	if (!ASSERT_GE(sock_fd, 0, "socket"))
1275	goto out;
1276
1277	err = listen(sock_fd, 1);
1278	if (!ASSERT_OK(err, "listen"))
1279	goto close_socket;
1280
1281	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1282
1283	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1284	if (!ASSERT_OK(err, "bpf_map_update_elem"))
1285	goto close_socket;
1286
1287	do_dummy_read(prog: skel->progs.fill_socket_owner);
1288
1289	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1290	if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
1291	!ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
1292	goto close_socket;
1293
1294	do_dummy_read(prog: skel->progs.negate_socket_local_storage);
1295
1296	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1297	ASSERT_OK(err, "bpf_map_lookup_elem");
1298	ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");
1299
1300	close_socket:
1301	close(sock_fd);
1302	out:
1303	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1304	}
1305
1306	static void test_bpf_sk_storage_map_iter_fd(void)
1307	{
1308	struct bpf_iter_bpf_sk_storage_map *skel;
1309
1310	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1311	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1312	return;
1313
1314	do_read_map_iter_fd(skel: &skel->skeleton, prog: skel->progs.rw_bpf_sk_storage_map,
1315	map: skel->maps.sk_stg_map);
1316
1317	bpf_iter_bpf_sk_storage_map__destroy(skel);
1318	}
1319
1320	static void test_bpf_sk_storage_map(void)
1321	{
1322	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1323	int err, i, len, map_fd, iter_fd, num_sockets;
1324	struct bpf_iter_bpf_sk_storage_map *skel;
1325	union bpf_iter_link_info linfo;
1326	int sock_fd[3] = {-1, -1, -1};
1327	__u32 val, expected_val = 0;
1328	struct bpf_link *link;
1329	char buf[64];
1330
1331	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1332	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1333	return;
1334
1335	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1336	num_sockets = ARRAY_SIZE(sock_fd);
1337	for (i = 0; i < num_sockets; i++) {
1338	sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
1339	if (!ASSERT_GE(sock_fd[i], 0, "socket"))
1340	goto out;
1341
1342	val = i + 1;
1343	expected_val += val;
1344
1345	err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
1346	BPF_NOEXIST);
1347	if (!ASSERT_OK(err, "map_update"))
1348	goto out;
1349	}
1350
1351	memset(&linfo, 0, sizeof(linfo));
1352	linfo.map.map_fd = map_fd;
1353	opts.link_info = &linfo;
1354	opts.link_info_len = sizeof(linfo);
1355	link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
1356	err = libbpf_get_error(link);
1357	if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
1358	if (!err)
1359	bpf_link__destroy(link);
1360	goto out;
1361	}
1362
1363	link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
1364	if (!ASSERT_OK_PTR(link, "attach_iter"))
1365	goto out;
1366
1367	iter_fd = bpf_iter_create(bpf_link__fd(link));
1368	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1369	goto free_link;
1370
1371	skel->bss->to_add_val = time(NULL);
1372	/* do some tests */
1373	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1374	;
1375	if (!ASSERT_GE(len, 0, "read"))
1376	goto close_iter;
1377
1378	/* test results */
1379	if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
1380	goto close_iter;
1381
1382	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1383	goto close_iter;
1384
1385	for (i = 0; i < num_sockets; i++) {
1386	err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
1387	if (!ASSERT_OK(err, "map_lookup") ||
1388	!ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
1389	break;
1390	}
1391
1392	close_iter:
1393	close(iter_fd);
1394	free_link:
1395	bpf_link__destroy(link);
1396	out:
1397	for (i = 0; i < num_sockets; i++) {
1398	if (sock_fd[i] >= 0)
1399	close(sock_fd[i]);
1400	}
1401	bpf_iter_bpf_sk_storage_map__destroy(skel);
1402	}
1403
1404	static void test_rdonly_buf_out_of_bound(void)
1405	{
1406	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1407	struct bpf_iter_test_kern5 *skel;
1408	union bpf_iter_link_info linfo;
1409	struct bpf_link *link;
1410
1411	skel = bpf_iter_test_kern5__open_and_load();
1412	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
1413	return;
1414
1415	memset(&linfo, 0, sizeof(linfo));
1416	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
1417	opts.link_info = &linfo;
1418	opts.link_info_len = sizeof(linfo);
1419	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
1420	if (!ASSERT_ERR_PTR(link, "attach_iter"))
1421	bpf_link__destroy(link);
1422
1423	bpf_iter_test_kern5__destroy(skel);
1424	}
1425
1426	static void test_buf_neg_offset(void)
1427	{
1428	struct bpf_iter_test_kern6 *skel;
1429
1430	skel = bpf_iter_test_kern6__open_and_load();
1431	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
1432	bpf_iter_test_kern6__destroy(skel);
1433	}
1434
1435	static void test_link_iter(void)
1436	{
1437	struct bpf_iter_bpf_link *skel;
1438
1439	skel = bpf_iter_bpf_link__open_and_load();
1440	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
1441	return;
1442
1443	do_dummy_read(prog: skel->progs.dump_bpf_link);
1444
1445	bpf_iter_bpf_link__destroy(skel);
1446	}
1447
1448	static void test_ksym_iter(void)
1449	{
1450	struct bpf_iter_ksym *skel;
1451
1452	skel = bpf_iter_ksym__open_and_load();
1453	if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
1454	return;
1455
1456	do_dummy_read(prog: skel->progs.dump_ksym);
1457
1458	bpf_iter_ksym__destroy(skel);
1459	}
1460
1461	#define CMP_BUFFER_SIZE 1024
1462	static char task_vma_output[CMP_BUFFER_SIZE];
1463	static char proc_maps_output[CMP_BUFFER_SIZE];
1464
1465	/* remove \0 and \t from str, and only keep the first line */
1466	static void str_strip_first_line(char *str)
1467	{
1468	char *dst = str, *src = str;
1469
1470	do {
1471	if (*src == ' ' || *src == '\t')
1472	src++;
1473	else
1474	*(dst++) = *(src++);
1475
1476	} while (*src != '\0' && *src != '\n');
1477
1478	*dst = '\0';
1479	}
1480
1481	static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
1482	{
1483	int err, iter_fd = -1, proc_maps_fd = -1;
1484	struct bpf_iter_task_vmas *skel;
1485	int len, read_size = 4;
1486	char maps_path[64];
1487
1488	skel = bpf_iter_task_vmas__open();
1489	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1490	return;
1491
1492	skel->bss->pid = getpid();
1493	skel->bss->one_task = opts ? 1 : 0;
1494
1495	err = bpf_iter_task_vmas__load(skel);
1496	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1497	goto out;
1498
1499	skel->links.proc_maps = bpf_program__attach_iter(
1500	skel->progs.proc_maps, opts);
1501
1502	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1503	skel->links.proc_maps = NULL;
1504	goto out;
1505	}
1506
1507	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1508	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1509	goto out;
1510
1511	/* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1512	* to trigger seq_file corner cases.
1513	*/
1514	len = 0;
1515	while (len < CMP_BUFFER_SIZE) {
1516	err = read_fd_into_buffer(fd: iter_fd, buf: task_vma_output + len,
1517	size: MIN(read_size, CMP_BUFFER_SIZE - len));
1518	if (!err)
1519	break;
1520	if (!ASSERT_GE(err, 0, "read_iter_fd"))
1521	goto out;
1522	len += err;
1523	}
1524	if (opts)
1525	ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
1526
1527	/* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1528	snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1529	proc_maps_fd = open(maps_path, O_RDONLY);
1530	if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
1531	goto out;
1532	err = read_fd_into_buffer(fd: proc_maps_fd, buf: proc_maps_output, CMP_BUFFER_SIZE);
1533	if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
1534	goto out;
1535
1536	/* strip and compare the first line of the two files */
1537	str_strip_first_line(str: task_vma_output);
1538	str_strip_first_line(str: proc_maps_output);
1539
1540	ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
1541
1542	check_bpf_link_info(prog: skel->progs.proc_maps);
1543
1544	out:
1545	close(proc_maps_fd);
1546	close(iter_fd);
1547	bpf_iter_task_vmas__destroy(skel);
1548	}
1549
1550	static void test_task_vma_dead_task(void)
1551	{
1552	struct bpf_iter_task_vmas *skel;
1553	int wstatus, child_pid = -1;
1554	time_t start_tm, cur_tm;
1555	int err, iter_fd = -1;
1556	int wait_sec = 3;
1557
1558	skel = bpf_iter_task_vmas__open();
1559	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1560	return;
1561
1562	skel->bss->pid = getpid();
1563
1564	err = bpf_iter_task_vmas__load(skel);
1565	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1566	goto out;
1567
1568	skel->links.proc_maps = bpf_program__attach_iter(
1569	skel->progs.proc_maps, NULL);
1570
1571	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1572	skel->links.proc_maps = NULL;
1573	goto out;
1574	}
1575
1576	start_tm = time(NULL);
1577	cur_tm = start_tm;
1578
1579	child_pid = fork();
1580	if (child_pid == 0) {
1581	/* Fork short-lived processes in the background. */
1582	while (cur_tm < start_tm + wait_sec) {
1583	system("echo > /dev/null");
1584	cur_tm = time(NULL);
1585	}
1586	exit(0);
1587	}
1588
1589	if (!ASSERT_GE(child_pid, 0, "fork_child"))
1590	goto out;
1591
1592	while (cur_tm < start_tm + wait_sec) {
1593	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1594	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1595	goto out;
1596
1597	/* Drain all data from iter_fd. */
1598	while (cur_tm < start_tm + wait_sec) {
1599	err = read_fd_into_buffer(fd: iter_fd, buf: task_vma_output, CMP_BUFFER_SIZE);
1600	if (!ASSERT_GE(err, 0, "read_iter_fd"))
1601	goto out;
1602
1603	cur_tm = time(NULL);
1604
1605	if (err == 0)
1606	break;
1607	}
1608
1609	close(iter_fd);
1610	iter_fd = -1;
1611	}
1612
1613	check_bpf_link_info(prog: skel->progs.proc_maps);
1614
1615	out:
1616	waitpid(child_pid, &wstatus, 0);
1617	close(iter_fd);
1618	bpf_iter_task_vmas__destroy(skel);
1619	}
1620
1621	void test_bpf_sockmap_map_iter_fd(void)
1622	{
1623	struct bpf_iter_sockmap *skel;
1624
1625	skel = bpf_iter_sockmap__open_and_load();
1626	if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
1627	return;
1628
1629	do_read_map_iter_fd(skel: &skel->skeleton, prog: skel->progs.copy, map: skel->maps.sockmap);
1630
1631	bpf_iter_sockmap__destroy(skel);
1632	}
1633
1634	static void test_task_vma(void)
1635	{
1636	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1637	union bpf_iter_link_info linfo;
1638
1639	memset(&linfo, 0, sizeof(linfo));
1640	linfo.task.tid = getpid();
1641	opts.link_info = &linfo;
1642	opts.link_info_len = sizeof(linfo);
1643
1644	test_task_vma_common(&opts);
1645	test_task_vma_common(NULL);
1646	}
1647
1648	/* uprobe attach point */
1649	static noinline int trigger_func(int arg)
1650	{
1651	asm volatile ("");
1652	return arg + 1;
1653	}
1654
1655	static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
1656	{
1657	struct bpf_iter_vma_offset *skel;
1658	char buf[16] = {};
1659	int iter_fd, len;
1660	int pgsz, shift;
1661
1662	skel = bpf_iter_vma_offset__open_and_load();
1663	if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
1664	return;
1665
1666	skel->bss->pid = getpid();
1667	skel->bss->address = (uintptr_t)trigger_func;
1668	for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
1669	;
1670	skel->bss->page_shift = shift;
1671
1672	skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
1673	if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
1674	goto exit;
1675
1676	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
1677	if (!ASSERT_GT(iter_fd, 0, "create_iter"))
1678	goto exit;
1679
1680	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1681	;
1682	buf[15] = 0;
1683	ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
1684
1685	ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
1686	if (one_proc)
1687	ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1688	else
1689	ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1690
1691	close(iter_fd);
1692
1693	exit:
1694	bpf_iter_vma_offset__destroy(skel);
1695	}
1696
1697	static void test_task_vma_offset(void)
1698	{
1699	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1700	union bpf_iter_link_info linfo;
1701
1702	memset(&linfo, 0, sizeof(linfo));
1703	linfo.task.pid = getpid();
1704	opts.link_info = &linfo;
1705	opts.link_info_len = sizeof(linfo);
1706
1707	test_task_vma_offset_common(&opts, true);
1708
1709	linfo.task.pid = 0;
1710	linfo.task.tid = getpid();
1711	test_task_vma_offset_common(&opts, true);
1712
1713	test_task_vma_offset_common(NULL, false);
1714	}
1715
1716	void test_bpf_iter(void)
1717	{
1718	ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
1719
1720	if (test__start_subtest("btf_id_or_null"))
1721	test_btf_id_or_null();
1722	if (test__start_subtest("ipv6_route"))
1723	test_ipv6_route();
1724	if (test__start_subtest("netlink"))
1725	test_netlink();
1726	if (test__start_subtest("bpf_map"))
1727	test_bpf_map();
1728	if (test__start_subtest("task_tid"))
1729	test_task_tid();
1730	if (test__start_subtest("task_pid"))
1731	test_task_pid();
1732	if (test__start_subtest("task_pidfd"))
1733	test_task_pidfd();
1734	if (test__start_subtest("task_sleepable"))
1735	test_task_sleepable();
1736	if (test__start_subtest("task_stack"))
1737	test_task_stack();
1738	if (test__start_subtest("task_file"))
1739	test_task_file();
1740	if (test__start_subtest("task_vma"))
1741	test_task_vma();
1742	if (test__start_subtest("task_vma_dead_task"))
1743	test_task_vma_dead_task();
1744	if (test__start_subtest("task_btf"))
1745	test_task_btf();
1746	if (test__start_subtest("tcp4"))
1747	test_tcp4();
1748	if (test__start_subtest("tcp6"))
1749	test_tcp6();
1750	if (test__start_subtest("udp4"))
1751	test_udp4();
1752	if (test__start_subtest("udp6"))
1753	test_udp6();
1754	if (test__start_subtest("unix"))
1755	test_unix();
1756	if (test__start_subtest("anon"))
1757	test_anon_iter(false);
1758	if (test__start_subtest("anon-read-one-char"))
1759	test_anon_iter(true);
1760	if (test__start_subtest("file"))
1761	test_file_iter();
1762	if (test__start_subtest("overflow"))
1763	test_overflow(false, false);
1764	if (test__start_subtest("overflow-e2big"))
1765	test_overflow(true, false);
1766	if (test__start_subtest("prog-ret-1"))
1767	test_overflow(false, true);
1768	if (test__start_subtest("bpf_hash_map"))
1769	test_bpf_hash_map();
1770	if (test__start_subtest("bpf_percpu_hash_map"))
1771	test_bpf_percpu_hash_map();
1772	if (test__start_subtest("bpf_array_map"))
1773	test_bpf_array_map();
1774	if (test__start_subtest("bpf_array_map_iter_fd"))
1775	test_bpf_array_map_iter_fd();
1776	if (test__start_subtest("bpf_percpu_array_map"))
1777	test_bpf_percpu_array_map();
1778	if (test__start_subtest("bpf_sk_storage_map"))
1779	test_bpf_sk_storage_map();
1780	if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
1781	test_bpf_sk_storage_map_iter_fd();
1782	if (test__start_subtest("bpf_sk_storage_delete"))
1783	test_bpf_sk_storage_delete();
1784	if (test__start_subtest("bpf_sk_storage_get"))
1785	test_bpf_sk_storage_get();
1786	if (test__start_subtest("rdonly-buf-out-of-bound"))
1787	test_rdonly_buf_out_of_bound();
1788	if (test__start_subtest("buf-neg-offset"))
1789	test_buf_neg_offset();
1790	if (test__start_subtest("link-iter"))
1791	test_link_iter();
1792	if (test__start_subtest("ksym"))
1793	test_ksym_iter();
1794	if (test__start_subtest("bpf_sockmap_map_iter_fd"))
1795	test_bpf_sockmap_map_iter_fd();
1796	if (test__start_subtest("vma_offset"))
1797	test_task_vma_offset();
1798	}
1799