1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright (c) 2017 Facebook
3	*/
4	#define _GNU_SOURCE
5	#include "test_progs.h"
6	#include "testing_helpers.h"
7	#include "cgroup_helpers.h"
8	#include <argp.h>
9	#include <pthread.h>
10	#include <sched.h>
11	#include <signal.h>
12	#include <string.h>
13	#include <execinfo.h> /* backtrace */
14	#include <sys/sysinfo.h> /* get_nprocs */
15	#include <netinet/in.h>
16	#include <sys/select.h>
17	#include <sys/socket.h>
18	#include <sys/un.h>
19	#include <bpf/btf.h>
20	#include "json_writer.h"
21
22	static bool verbose(void)
23	{
24	return env.verbosity > VERBOSE_NONE;
25	}
26
27	static void stdio_hijack_init(char **log_buf, size_t *log_cnt)
28	{
29	#ifdef __GLIBC__
30	if (verbose() && env.worker_id == -1) {
31	/* nothing to do, output to stdout by default */
32	return;
33	}
34
35	fflush(stdout);
36	fflush(stderr);
37
38	stdout = open_memstream(log_buf, log_cnt);
39	if (!stdout) {
40	stdout = env.stdout;
41	perror("open_memstream");
42	return;
43	}
44
45	if (env.subtest_state)
46	env.subtest_state->stdout = stdout;
47	else
48	env.test_state->stdout = stdout;
49
50	stderr = stdout;
51	#endif
52	}
53
54	static void stdio_hijack(char **log_buf, size_t *log_cnt)
55	{
56	#ifdef __GLIBC__
57	if (verbose() && env.worker_id == -1) {
58	/* nothing to do, output to stdout by default */
59	return;
60	}
61
62	env.stdout = stdout;
63	env.stderr = stderr;
64
65	stdio_hijack_init(log_buf, log_cnt);
66	#endif
67	}
68
69	static void stdio_restore_cleanup(void)
70	{
71	#ifdef __GLIBC__
72	if (verbose() && env.worker_id == -1) {
73	/* nothing to do, output to stdout by default */
74	return;
75	}
76
77	fflush(stdout);
78
79	if (env.subtest_state) {
80	fclose(env.subtest_state->stdout);
81	env.subtest_state->stdout = NULL;
82	stdout = env.test_state->stdout;
83	stderr = env.test_state->stdout;
84	} else {
85	fclose(env.test_state->stdout);
86	env.test_state->stdout = NULL;
87	}
88	#endif
89	}
90
91	static void stdio_restore(void)
92	{
93	#ifdef __GLIBC__
94	if (verbose() && env.worker_id == -1) {
95	/* nothing to do, output to stdout by default */
96	return;
97	}
98
99	if (stdout == env.stdout)
100	return;
101
102	stdio_restore_cleanup();
103
104	stdout = env.stdout;
105	stderr = env.stderr;
106	#endif
107	}
108
109	/* Adapted from perf/util/string.c */
110	static bool glob_match(const char *str, const char *pat)
111	{
112	while (*str && *pat && *pat != '*') {
113	if (*str != *pat)
114	return false;
115	str++;
116	pat++;
117	}
118	/* Check wild card */
119	if (*pat == '*') {
120	while (*pat == '*')
121	pat++;
122	if (!*pat) /* Tail wild card matches all */
123	return true;
124	while (*str)
125	if (glob_match(str: str++, pat))
126	return true;
127	}
128	return !*str && !*pat;
129	}
130
131	#define EXIT_NO_TEST 2
132	#define EXIT_ERR_SETUP_INFRA 3
133
134	/* defined in test_progs.h */
135	struct test_env env = {};
136
137	struct prog_test_def {
138	const char *test_name;
139	int test_num;
140	void (*run_test)(void);
141	void (*run_serial_test)(void);
142	bool should_run;
143	bool need_cgroup_cleanup;
144	};
145
146	/* Override C runtime library's usleep() implementation to ensure nanosleep()
147	* is always called. Usleep is frequently used in selftests as a way to
148	* trigger kprobe and tracepoints.
149	*/
150	int usleep(useconds_t usec)
151	{
152	struct timespec ts = {
153	.tv_sec = usec / 1000000,
154	.tv_nsec = (usec % 1000000) * 1000,
155	};
156
157	return syscall(__NR_nanosleep, &ts, NULL);
158	}
159
160	static bool should_run(struct test_selector *sel, int num, const char *name)
161	{
162	int i;
163
164	for (i = 0; i < sel->blacklist.cnt; i++) {
165	if (glob_match(str: name, pat: sel->blacklist.tests[i].name) &&
166	!sel->blacklist.tests[i].subtest_cnt)
167	return false;
168	}
169
170	for (i = 0; i < sel->whitelist.cnt; i++) {
171	if (glob_match(str: name, pat: sel->whitelist.tests[i].name))
172	return true;
173	}
174
175	if (!sel->whitelist.cnt && !sel->num_set)
176	return true;
177
178	return num < sel->num_set_len && sel->num_set[num];
179	}
180
181	static bool should_run_subtest(struct test_selector *sel,
182	struct test_selector *subtest_sel,
183	int subtest_num,
184	const char *test_name,
185	const char *subtest_name)
186	{
187	int i, j;
188
189	for (i = 0; i < sel->blacklist.cnt; i++) {
190	if (glob_match(str: test_name, pat: sel->blacklist.tests[i].name)) {
191	if (!sel->blacklist.tests[i].subtest_cnt)
192	return false;
193
194	for (j = 0; j < sel->blacklist.tests[i].subtest_cnt; j++) {
195	if (glob_match(str: subtest_name,
196	pat: sel->blacklist.tests[i].subtests[j]))
197	return false;
198	}
199	}
200	}
201
202	for (i = 0; i < sel->whitelist.cnt; i++) {
203	if (glob_match(str: test_name, pat: sel->whitelist.tests[i].name)) {
204	if (!sel->whitelist.tests[i].subtest_cnt)
205	return true;
206
207	for (j = 0; j < sel->whitelist.tests[i].subtest_cnt; j++) {
208	if (glob_match(str: subtest_name,
209	pat: sel->whitelist.tests[i].subtests[j]))
210	return true;
211	}
212	}
213	}
214
215	if (!sel->whitelist.cnt && !subtest_sel->num_set)
216	return true;
217
218	return subtest_num < subtest_sel->num_set_len && subtest_sel->num_set[subtest_num];
219	}
220
221	static char *test_result(bool failed, bool skipped)
222	{
223	return failed ? "FAIL" : (skipped ? "SKIP" : "OK");
224	}
225
226	#define TEST_NUM_WIDTH 7
227
228	static void print_test_result(const struct prog_test_def *test, const struct test_state *test_state)
229	{
230	int skipped_cnt = test_state->skip_cnt;
231	int subtests_cnt = test_state->subtest_num;
232
233	fprintf(env.stdout, "#%-*d %s:", TEST_NUM_WIDTH, test->test_num, test->test_name);
234	if (test_state->error_cnt)
235	fprintf(env.stdout, "FAIL");
236	else if (!skipped_cnt)
237	fprintf(env.stdout, "OK");
238	else if (skipped_cnt == subtests_cnt || !subtests_cnt)
239	fprintf(env.stdout, "SKIP");
240	else
241	fprintf(env.stdout, "OK (SKIP: %d/%d)", skipped_cnt, subtests_cnt);
242
243	fprintf(env.stdout, "\n");
244	}
245
246	static void print_test_log(char *log_buf, size_t log_cnt)
247	{
248	log_buf[log_cnt] = '\0';
249	fprintf(env.stdout, "%s", log_buf);
250	if (log_buf[log_cnt - 1] != '\n')
251	fprintf(env.stdout, "\n");
252	}
253
254	static void print_subtest_name(int test_num, int subtest_num,
255	const char *test_name, char *subtest_name,
256	char *result)
257	{
258	char test_num_str[32];
259
260	snprintf(buf: test_num_str, size: sizeof(test_num_str), fmt: "%d/%d", test_num, subtest_num);
261
262	fprintf(env.stdout, "#%-*s %s/%s",
263	TEST_NUM_WIDTH, test_num_str,
264	test_name, subtest_name);
265
266	if (result)
267	fprintf(env.stdout, ":%s", result);
268
269	fprintf(env.stdout, "\n");
270	}
271
272	static void jsonw_write_log_message(json_writer_t *w, char *log_buf, size_t log_cnt)
273	{
274	/* open_memstream (from stdio_hijack_init) ensures that log_bug is terminated by a
275	* null byte. Yet in parallel mode, log_buf will be NULL if there is no message.
276	*/
277	if (log_cnt) {
278	jsonw_string_field(self: w, prop: "message", val: log_buf);
279	} else {
280	jsonw_string_field(self: w, prop: "message", val: "");
281	}
282	}
283
284	static void dump_test_log(const struct prog_test_def *test,
285	const struct test_state *test_state,
286	bool skip_ok_subtests,
287	bool par_exec_result,
288	json_writer_t *w)
289	{
290	bool test_failed = test_state->error_cnt > 0;
291	bool force_log = test_state->force_log;
292	bool print_test = verbose() || force_log || test_failed;
293	int i;
294	struct subtest_state *subtest_state;
295	bool subtest_failed;
296	bool subtest_filtered;
297	bool print_subtest;
298
299	/* we do not print anything in the worker thread */
300	if (env.worker_id != -1)
301	return;
302
303	/* there is nothing to print when verbose log is used and execution
304	* is not in parallel mode
305	*/
306	if (verbose() && !par_exec_result)
307	return;
308
309	if (test_state->log_cnt && print_test)
310	print_test_log(log_buf: test_state->log_buf, log_cnt: test_state->log_cnt);
311
312	if (w && print_test) {
313	jsonw_start_object(self: w);
314	jsonw_string_field(self: w, prop: "name", val: test->test_name);
315	jsonw_uint_field(self: w, prop: "number", num: test->test_num);
316	jsonw_write_log_message(w, log_buf: test_state->log_buf, log_cnt: test_state->log_cnt);
317	jsonw_bool_field(self: w, prop: "failed", value: test_failed);
318	jsonw_name(self: w, name: "subtests");
319	jsonw_start_array(self: w);
320	}
321
322	for (i = 0; i < test_state->subtest_num; i++) {
323	subtest_state = &test_state->subtest_states[i];
324	subtest_failed = subtest_state->error_cnt;
325	subtest_filtered = subtest_state->filtered;
326	print_subtest = verbose() || force_log || subtest_failed;
327
328	if ((skip_ok_subtests && !subtest_failed) || subtest_filtered)
329	continue;
330
331	if (subtest_state->log_cnt && print_subtest) {
332	print_test_log(log_buf: subtest_state->log_buf,
333	log_cnt: subtest_state->log_cnt);
334	}
335
336	print_subtest_name(test_num: test->test_num, subtest_num: i + 1,
337	test_name: test->test_name, subtest_name: subtest_state->name,
338	result: test_result(failed: subtest_state->error_cnt,
339	skipped: subtest_state->skipped));
340
341	if (w && print_subtest) {
342	jsonw_start_object(self: w);
343	jsonw_string_field(self: w, prop: "name", val: subtest_state->name);
344	jsonw_uint_field(self: w, prop: "number", num: i+1);
345	jsonw_write_log_message(w, log_buf: subtest_state->log_buf, log_cnt: subtest_state->log_cnt);
346	jsonw_bool_field(self: w, prop: "failed", value: subtest_failed);
347	jsonw_end_object(self: w);
348	}
349	}
350
351	if (w && print_test) {
352	jsonw_end_array(self: w);
353	jsonw_end_object(self: w);
354	}
355
356	print_test_result(test, test_state);
357	}
358
359	static void stdio_restore(void);
360
361	/* A bunch of tests set custom affinity per-thread and/or per-process. Reset
362	* it after each test/sub-test.
363	*/
364	static void reset_affinity(void)
365	{
366	cpu_set_t cpuset;
367	int i, err;
368
369	CPU_ZERO(&cpuset);
370	for (i = 0; i < env.nr_cpus; i++)
371	CPU_SET(i, &cpuset);
372
373	err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
374	if (err < 0) {
375	stdio_restore();
376	fprintf(stderr, "Failed to reset process affinity: %d!\n", err);
377	exit(EXIT_ERR_SETUP_INFRA);
378	}
379	err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
380	if (err < 0) {
381	stdio_restore();
382	fprintf(stderr, "Failed to reset thread affinity: %d!\n", err);
383	exit(EXIT_ERR_SETUP_INFRA);
384	}
385	}
386
387	static void save_netns(void)
388	{
389	env.saved_netns_fd = open("/proc/self/ns/net", O_RDONLY);
390	if (env.saved_netns_fd == -1) {
391	perror("open(/proc/self/ns/net)");
392	exit(EXIT_ERR_SETUP_INFRA);
393	}
394	}
395
396	static void restore_netns(void)
397	{
398	if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) {
399	stdio_restore();
400	perror("setns(CLONE_NEWNS)");
401	exit(EXIT_ERR_SETUP_INFRA);
402	}
403	}
404
405	void test__end_subtest(void)
406	{
407	struct prog_test_def *test = env.test;
408	struct test_state *test_state = env.test_state;
409	struct subtest_state *subtest_state = env.subtest_state;
410
411	if (subtest_state->error_cnt) {
412	test_state->error_cnt++;
413	} else {
414	if (!subtest_state->skipped)
415	test_state->sub_succ_cnt++;
416	else
417	test_state->skip_cnt++;
418	}
419
420	if (verbose() && !env.workers)
421	print_subtest_name(test_num: test->test_num, subtest_num: test_state->subtest_num,
422	test_name: test->test_name, subtest_name: subtest_state->name,
423	result: test_result(failed: subtest_state->error_cnt,
424	skipped: subtest_state->skipped));
425
426	stdio_restore_cleanup();
427	env.subtest_state = NULL;
428	}
429
430	bool test__start_subtest(const char *subtest_name)
431	{
432	struct prog_test_def *test = env.test;
433	struct test_state *state = env.test_state;
434	struct subtest_state *subtest_state;
435	size_t sub_state_size = sizeof(*subtest_state);
436
437	if (env.subtest_state)
438	test__end_subtest();
439
440	state->subtest_num++;
441	state->subtest_states =
442	realloc(state->subtest_states,
443	state->subtest_num * sub_state_size);
444	if (!state->subtest_states) {
445	fprintf(stderr, "Not enough memory to allocate subtest result\n");
446	return false;
447	}
448
449	subtest_state = &state->subtest_states[state->subtest_num - 1];
450
451	memset(subtest_state, 0, sub_state_size);
452
453	if (!subtest_name || !subtest_name[0]) {
454	fprintf(env.stderr,
455	"Subtest #%d didn't provide sub-test name!\n",
456	state->subtest_num);
457	return false;
458	}
459
460	subtest_state->name = strdup(subtest_name);
461	if (!subtest_state->name) {
462	fprintf(env.stderr,
463	"Subtest #%d: failed to copy subtest name!\n",
464	state->subtest_num);
465	return false;
466	}
467
468	if (!should_run_subtest(sel: &env.test_selector,
469	subtest_sel: &env.subtest_selector,
470	subtest_num: state->subtest_num,
471	test_name: test->test_name,
472	subtest_name)) {
473	subtest_state->filtered = true;
474	return false;
475	}
476
477	env.subtest_state = subtest_state;
478	stdio_hijack_init(log_buf: &subtest_state->log_buf, log_cnt: &subtest_state->log_cnt);
479
480	return true;
481	}
482
483	void test__force_log(void)
484	{
485	env.test_state->force_log = true;
486	}
487
488	void test__skip(void)
489	{
490	if (env.subtest_state)
491	env.subtest_state->skipped = true;
492	else
493	env.test_state->skip_cnt++;
494	}
495
496	void test__fail(void)
497	{
498	if (env.subtest_state)
499	env.subtest_state->error_cnt++;
500	else
501	env.test_state->error_cnt++;
502	}
503
504	int test__join_cgroup(const char *path)
505	{
506	int fd;
507
508	if (!env.test->need_cgroup_cleanup) {
509	if (setup_cgroup_environment()) {
510	fprintf(stderr,
511	"#%d %s: Failed to setup cgroup environment\n",
512	env.test->test_num, env.test->test_name);
513	return -1;
514	}
515
516	env.test->need_cgroup_cleanup = true;
517	}
518
519	fd = create_and_get_cgroup(relative_path: path);
520	if (fd < 0) {
521	fprintf(stderr,
522	"#%d %s: Failed to create cgroup '%s' (errno=%d)\n",
523	env.test->test_num, env.test->test_name, path, errno);
524	return fd;
525	}
526
527	if (join_cgroup(relative_path: path)) {
528	fprintf(stderr,
529	"#%d %s: Failed to join cgroup '%s' (errno=%d)\n",
530	env.test->test_num, env.test->test_name, path, errno);
531	return -1;
532	}
533
534	return fd;
535	}
536
537	int bpf_find_map(const char *test, struct bpf_object *obj, const char *name)
538	{
539	struct bpf_map *map;
540
541	map = bpf_object__find_map_by_name(obj, name);
542	if (!map) {
543	fprintf(stdout, "%s:FAIL:map '%s' not found\n", test, name);
544	test__fail();
545	return -1;
546	}
547	return bpf_map__fd(map);
548	}
549
550	int compare_map_keys(int map1_fd, int map2_fd)
551	{
552	__u32 key, next_key;
553	char val_buf[PERF_MAX_STACK_DEPTH *
554	sizeof(struct bpf_stack_build_id)];
555	int err;
556
557	err = bpf_map_get_next_key(map1_fd, NULL, &key);
558	if (err)
559	return err;
560	err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
561	if (err)
562	return err;
563
564	while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
565	err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
566	if (err)
567	return err;
568
569	key = next_key;
570	}
571	if (errno != ENOENT)
572	return -1;
573
574	return 0;
575	}
576
577	int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
578	{
579	__u32 key, next_key, *cur_key_p, *next_key_p;
580	char *val_buf1, *val_buf2;
581	int i, err = 0;
582
583	val_buf1 = malloc(stack_trace_len);
584	val_buf2 = malloc(stack_trace_len);
585	cur_key_p = NULL;
586	next_key_p = &key;
587	while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
588	err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
589	if (err)
590	goto out;
591	err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
592	if (err)
593	goto out;
594	for (i = 0; i < stack_trace_len; i++) {
595	if (val_buf1[i] != val_buf2[i]) {
596	err = -1;
597	goto out;
598	}
599	}
600	key = *next_key_p;
601	cur_key_p = &key;
602	next_key_p = &next_key;
603	}
604	if (errno != ENOENT)
605	err = -1;
606
607	out:
608	free(val_buf1);
609	free(val_buf2);
610	return err;
611	}
612
613	/* extern declarations for test funcs */
614	#define DEFINE_TEST(name) \
615	extern void test_##name(void) __weak; \
616	extern void serial_test_##name(void) __weak;
617	#include <prog_tests/tests.h>
618	#undef DEFINE_TEST
619
620	static struct prog_test_def prog_test_defs[] = {
621	#define DEFINE_TEST(name) { \
622	.test_name = #name, \
623	.run_test = &test_##name, \
624	.run_serial_test = &serial_test_##name, \
625	},
626	#include <prog_tests/tests.h>
627	#undef DEFINE_TEST
628	};
629
630	static const int prog_test_cnt = ARRAY_SIZE(prog_test_defs);
631
632	static struct test_state test_states[ARRAY_SIZE(prog_test_defs)];
633
634	const char *argp_program_version = "test_progs 0.1";
635	const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
636	static const char argp_program_doc[] =
637	"BPF selftests test runner\v"
638	"Options accepting the NAMES parameter take either a comma-separated list\n"
639	"of test names, or a filename prefixed with @. The file contains one name\n"
640	"(or wildcard pattern) per line, and comments beginning with # are ignored.\n"
641	"\n"
642	"These options can be passed repeatedly to read multiple files.\n";
643
644	enum ARG_KEYS {
645	ARG_TEST_NUM = 'n',
646	ARG_TEST_NAME = 't',
647	ARG_TEST_NAME_BLACKLIST = 'b',
648	ARG_VERIFIER_STATS = 's',
649	ARG_VERBOSE = 'v',
650	ARG_GET_TEST_CNT = 'c',
651	ARG_LIST_TEST_NAMES = 'l',
652	ARG_TEST_NAME_GLOB_ALLOWLIST = 'a',
653	ARG_TEST_NAME_GLOB_DENYLIST = 'd',
654	ARG_NUM_WORKERS = 'j',
655	ARG_DEBUG = -1,
656	ARG_JSON_SUMMARY = 'J'
657	};
658
659	static const struct argp_option opts[] = {
660	{ "num", ARG_TEST_NUM, "NUM", 0,
661	"Run test number NUM only " },
662	{ "name", ARG_TEST_NAME, "NAMES", 0,
663	"Run tests with names containing any string from NAMES list" },
664	{ "name-blacklist", ARG_TEST_NAME_BLACKLIST, "NAMES", 0,
665	"Don't run tests with names containing any string from NAMES list" },
666	{ "verifier-stats", ARG_VERIFIER_STATS, NULL, 0,
667	"Output verifier statistics", },
668	{ "verbose", ARG_VERBOSE, "LEVEL", OPTION_ARG_OPTIONAL,
669	"Verbose output (use -vv or -vvv for progressively verbose output)" },
670	{ "count", ARG_GET_TEST_CNT, NULL, 0,
671	"Get number of selected top-level tests " },
672	{ "list", ARG_LIST_TEST_NAMES, NULL, 0,
673	"List test names that would run (without running them) " },
674	{ "allow", ARG_TEST_NAME_GLOB_ALLOWLIST, "NAMES", 0,
675	"Run tests with name matching the pattern (supports '*' wildcard)." },
676	{ "deny", ARG_TEST_NAME_GLOB_DENYLIST, "NAMES", 0,
677	"Don't run tests with name matching the pattern (supports '*' wildcard)." },
678	{ "workers", ARG_NUM_WORKERS, "WORKERS", OPTION_ARG_OPTIONAL,
679	"Number of workers to run in parallel, default to number of cpus." },
680	{ "debug", ARG_DEBUG, NULL, 0,
681	"print extra debug information for test_progs." },
682	{ "json-summary", ARG_JSON_SUMMARY, "FILE", 0, "Write report in json format to this file."},
683	{},
684	};
685
686	static FILE *libbpf_capture_stream;
687
688	static struct {
689	char *buf;
690	size_t buf_sz;
691	} libbpf_output_capture;
692
693	/* Creates a global memstream capturing INFO and WARN level output
694	* passed to libbpf_print_fn.
695	* Returns 0 on success, negative value on failure.
696	* On failure the description is printed using PRINT_FAIL and
697	* current test case is marked as fail.
698	*/
699	int start_libbpf_log_capture(void)
700	{
701	if (libbpf_capture_stream) {
702	PRINT_FAIL("%s: libbpf_capture_stream != NULL\n", __func__);
703	return -EINVAL;
704	}
705
706	libbpf_capture_stream = open_memstream(&libbpf_output_capture.buf,
707	&libbpf_output_capture.buf_sz);
708	if (!libbpf_capture_stream) {
709	PRINT_FAIL("%s: open_memstream failed errno=%d\n", __func__, errno);
710	return -EINVAL;
711	}
712
713	return 0;
714	}
715
716	/* Destroys global memstream created by start_libbpf_log_capture().
717	* Returns a pointer to captured data which has to be freed.
718	* Returned buffer is null terminated.
719	*/
720	char *stop_libbpf_log_capture(void)
721	{
722	char *buf;
723
724	if (!libbpf_capture_stream)
725	return NULL;
726
727	fputc(0, libbpf_capture_stream);
728	fclose(libbpf_capture_stream);
729	libbpf_capture_stream = NULL;
730	/* get 'buf' after fclose(), see open_memstream() documentation */
731	buf = libbpf_output_capture.buf;
732	memset(&libbpf_output_capture, 0, sizeof(libbpf_output_capture));
733	return buf;
734	}
735
736	static int libbpf_print_fn(enum libbpf_print_level level,
737	const char *format, va_list args)
738	{
739	if (libbpf_capture_stream && level != LIBBPF_DEBUG) {
740	va_list args2;
741
742	va_copy(args2, args);
743	vfprintf(libbpf_capture_stream, format, args2);
744	}
745
746	if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)
747	return 0;
748
749	vfprintf(stdout, format, args);
750	return 0;
751	}
752
753	static void free_test_filter_set(const struct test_filter_set *set)
754	{
755	int i, j;
756
757	if (!set)
758	return;
759
760	for (i = 0; i < set->cnt; i++) {
761	free((void *)set->tests[i].name);
762	for (j = 0; j < set->tests[i].subtest_cnt; j++)
763	free((void *)set->tests[i].subtests[j]);
764
765	free((void *)set->tests[i].subtests);
766	}
767
768	free((void *)set->tests);
769	}
770
771	static void free_test_selector(struct test_selector *test_selector)
772	{
773	free_test_filter_set(set: &test_selector->blacklist);
774	free_test_filter_set(set: &test_selector->whitelist);
775	free(test_selector->num_set);
776	}
777
778	extern int extra_prog_load_log_flags;
779
780	static error_t parse_arg(int key, char *arg, struct argp_state *state)
781	{
782	struct test_env *env = state->input;
783	int err = 0;
784
785	switch (key) {
786	case ARG_TEST_NUM: {
787	char *subtest_str = strchr(arg, '/');
788
789	if (subtest_str) {
790	*subtest_str = '\0';
791	if (parse_num_list(s: subtest_str + 1,
792	set: &env->subtest_selector.num_set,
793	set_len: &env->subtest_selector.num_set_len)) {
794	fprintf(stderr,
795	"Failed to parse subtest numbers.\n");
796	return -EINVAL;
797	}
798	}
799	if (parse_num_list(s: arg, set: &env->test_selector.num_set,
800	set_len: &env->test_selector.num_set_len)) {
801	fprintf(stderr, "Failed to parse test numbers.\n");
802	return -EINVAL;
803	}
804	break;
805	}
806	case ARG_TEST_NAME_GLOB_ALLOWLIST:
807	case ARG_TEST_NAME: {
808	if (arg[0] == '@')
809	err = parse_test_list_file(path: arg + 1,
810	test_set: &env->test_selector.whitelist,
811	is_glob_pattern: key == ARG_TEST_NAME_GLOB_ALLOWLIST);
812	else
813	err = parse_test_list(s: arg,
814	test_set: &env->test_selector.whitelist,
815	is_glob_pattern: key == ARG_TEST_NAME_GLOB_ALLOWLIST);
816
817	break;
818	}
819	case ARG_TEST_NAME_GLOB_DENYLIST:
820	case ARG_TEST_NAME_BLACKLIST: {
821	if (arg[0] == '@')
822	err = parse_test_list_file(path: arg + 1,
823	test_set: &env->test_selector.blacklist,
824	is_glob_pattern: key == ARG_TEST_NAME_GLOB_DENYLIST);
825	else
826	err = parse_test_list(s: arg,
827	test_set: &env->test_selector.blacklist,
828	is_glob_pattern: key == ARG_TEST_NAME_GLOB_DENYLIST);
829
830	break;
831	}
832	case ARG_VERIFIER_STATS:
833	env->verifier_stats = true;
834	break;
835	case ARG_VERBOSE:
836	env->verbosity = VERBOSE_NORMAL;
837	if (arg) {
838	if (strcmp(arg, "v") == 0) {
839	env->verbosity = VERBOSE_VERY;
840	extra_prog_load_log_flags = 1;
841	} else if (strcmp(arg, "vv") == 0) {
842	env->verbosity = VERBOSE_SUPER;
843	extra_prog_load_log_flags = 2;
844	} else {
845	fprintf(stderr,
846	"Unrecognized verbosity setting ('%s'), only -v and -vv are supported\n",
847	arg);
848	return -EINVAL;
849	}
850	}
851
852	if (verbose()) {
853	if (setenv("SELFTESTS_VERBOSE", "1", 1) == -1) {
854	fprintf(stderr,
855	"Unable to setenv SELFTESTS_VERBOSE=1 (errno=%d)",
856	errno);
857	return -EINVAL;
858	}
859	}
860
861	break;
862	case ARG_GET_TEST_CNT:
863	env->get_test_cnt = true;
864	break;
865	case ARG_LIST_TEST_NAMES:
866	env->list_test_names = true;
867	break;
868	case ARG_NUM_WORKERS:
869	if (arg) {
870	env->workers = atoi(arg);
871	if (!env->workers) {
872	fprintf(stderr, "Invalid number of worker: %s.", arg);
873	return -EINVAL;
874	}
875	} else {
876	env->workers = get_nprocs();
877	}
878	break;
879	case ARG_DEBUG:
880	env->debug = true;
881	break;
882	case ARG_JSON_SUMMARY:
883	env->json = fopen(arg, "w");
884	if (env->json == NULL) {
885	perror("Failed to open json summary file");
886	return -errno;
887	}
888	break;
889	case ARGP_KEY_ARG:
890	argp_usage(state);
891	break;
892	case ARGP_KEY_END:
893	break;
894	default:
895	return ARGP_ERR_UNKNOWN;
896	}
897	return err;
898	}
899
900	/*
901	* Determine if test_progs is running as a "flavored" test runner and switch
902	* into corresponding sub-directory to load correct BPF objects.
903	*
904	* This is done by looking at executable name. If it contains "-flavor"
905	* suffix, then we are running as a flavored test runner.
906	*/
907	int cd_flavor_subdir(const char *exec_name)
908	{
909	/* General form of argv[0] passed here is:
910	* some/path/to/test_progs[-flavor], where -flavor part is optional.
911	* First cut out "test_progs[-flavor]" part, then extract "flavor"
912	* part, if it's there.
913	*/
914	const char *flavor = strrchr(exec_name, '/');
915
916	if (!flavor)
917	flavor = exec_name;
918	else
919	flavor++;
920
921	flavor = strrchr(flavor, '-');
922	if (!flavor)
923	return 0;
924	flavor++;
925	if (verbose())
926	fprintf(stdout, "Switching to flavor '%s' subdirectory...\n", flavor);
927
928	return chdir(flavor);
929	}
930
931	int trigger_module_test_read(int read_sz)
932	{
933	int fd, err;
934
935	fd = open(BPF_TESTMOD_TEST_FILE, O_RDONLY);
936	err = -errno;
937	if (!ASSERT_GE(fd, 0, "testmod_file_open"))
938	return err;
939
940	read(fd, NULL, read_sz);
941	close(fd);
942
943	return 0;
944	}
945
946	int trigger_module_test_write(int write_sz)
947	{
948	int fd, err;
949	char *buf = malloc(write_sz);
950
951	if (!buf)
952	return -ENOMEM;
953
954	memset(buf, 'a', write_sz);
955	buf[write_sz-1] = '\0';
956
957	fd = open(BPF_TESTMOD_TEST_FILE, O_WRONLY);
958	err = -errno;
959	if (!ASSERT_GE(fd, 0, "testmod_file_open")) {
960	free(buf);
961	return err;
962	}
963
964	write(fd, buf, write_sz);
965	close(fd);
966	free(buf);
967	return 0;
968	}
969
970	int write_sysctl(const char *sysctl, const char *value)
971	{
972	int fd, err, len;
973
974	fd = open(sysctl, O_WRONLY);
975	if (!ASSERT_NEQ(fd, -1, "open sysctl"))
976	return -1;
977
978	len = strlen(value);
979	err = write(fd, value, len);
980	close(fd);
981	if (!ASSERT_EQ(err, len, "write sysctl"))
982	return -1;
983
984	return 0;
985	}
986
987	int get_bpf_max_tramp_links_from(struct btf *btf)
988	{
989	const struct btf_enum *e;
990	const struct btf_type *t;
991	__u32 i, type_cnt;
992	const char *name;
993	__u16 j, vlen;
994
995	for (i = 1, type_cnt = btf__type_cnt(btf); i < type_cnt; i++) {
996	t = btf__type_by_id(btf, i);
997	if (!t || !btf_is_enum(t) || t->name_off)
998	continue;
999	e = btf_enum(t);
1000	for (j = 0, vlen = btf_vlen(t); j < vlen; j++, e++) {
1001	name = btf__str_by_offset(btf, e->name_off);
1002	if (name && !strcmp(name, "BPF_MAX_TRAMP_LINKS"))
1003	return e->val;
1004	}
1005	}
1006
1007	return -1;
1008	}
1009
1010	int get_bpf_max_tramp_links(void)
1011	{
1012	struct btf *vmlinux_btf;
1013	int ret;
1014
1015	vmlinux_btf = btf__load_vmlinux_btf();
1016	if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux btf"))
1017	return -1;
1018	ret = get_bpf_max_tramp_links_from(btf: vmlinux_btf);
1019	btf__free(vmlinux_btf);
1020
1021	return ret;
1022	}
1023
1024	#define MAX_BACKTRACE_SZ 128
1025	void crash_handler(int signum)
1026	{
1027	void *bt[MAX_BACKTRACE_SZ];
1028	size_t sz;
1029
1030	sz = backtrace(bt, ARRAY_SIZE(bt));
1031
1032	if (env.stdout)
1033	stdio_restore();
1034	if (env.test) {
1035	env.test_state->error_cnt++;
1036	dump_test_log(test: env.test, test_state: env.test_state, skip_ok_subtests: true, par_exec_result: false, NULL);
1037	}
1038	if (env.worker_id != -1)
1039	fprintf(stderr, "[%d]: ", env.worker_id);
1040	fprintf(stderr, "Caught signal #%d!\nStack trace:\n", signum);
1041	backtrace_symbols_fd(bt, sz, STDERR_FILENO);
1042	}
1043
1044	static void sigint_handler(int signum)
1045	{
1046	int i;
1047
1048	for (i = 0; i < env.workers; i++)
1049	if (env.worker_socks[i] > 0)
1050	close(env.worker_socks[i]);
1051	}
1052
1053	static int current_test_idx;
1054	static pthread_mutex_t current_test_lock;
1055	static pthread_mutex_t stdout_output_lock;
1056
1057	static inline const char *str_msg(const struct msg *msg, char *buf)
1058	{
1059	switch (msg->type) {
1060	case MSG_DO_TEST:
1061	sprintf(buf, fmt: "MSG_DO_TEST %d", msg->do_test.num);
1062	break;
1063	case MSG_TEST_DONE:
1064	sprintf(buf, fmt: "MSG_TEST_DONE %d (log: %d)",
1065	msg->test_done.num,
1066	msg->test_done.have_log);
1067	break;
1068	case MSG_SUBTEST_DONE:
1069	sprintf(buf, fmt: "MSG_SUBTEST_DONE %d (log: %d)",
1070	msg->subtest_done.num,
1071	msg->subtest_done.have_log);
1072	break;
1073	case MSG_TEST_LOG:
1074	sprintf(buf, fmt: "MSG_TEST_LOG (cnt: %zu, last: %d)",
1075	strlen(msg->test_log.log_buf),
1076	msg->test_log.is_last);
1077	break;
1078	case MSG_EXIT:
1079	sprintf(buf, fmt: "MSG_EXIT");
1080	break;
1081	default:
1082	sprintf(buf, fmt: "UNKNOWN");
1083	break;
1084	}
1085
1086	return buf;
1087	}
1088
1089	static int send_message(int sock, const struct msg *msg)
1090	{
1091	char buf[256];
1092
1093	if (env.debug)
1094	fprintf(stderr, "Sending msg: %s\n", str_msg(msg, buf));
1095	return send(sock, msg, sizeof(*msg), 0);
1096	}
1097
1098	static int recv_message(int sock, struct msg *msg)
1099	{
1100	int ret;
1101	char buf[256];
1102
1103	memset(msg, 0, sizeof(*msg));
1104	ret = recv(sock, msg, sizeof(*msg), 0);
1105	if (ret >= 0) {
1106	if (env.debug)
1107	fprintf(stderr, "Received msg: %s\n", str_msg(msg, buf));
1108	}
1109	return ret;
1110	}
1111
1112	static void run_one_test(int test_num)
1113	{
1114	struct prog_test_def *test = &prog_test_defs[test_num];
1115	struct test_state *state = &test_states[test_num];
1116
1117	env.test = test;
1118	env.test_state = state;
1119
1120	stdio_hijack(log_buf: &state->log_buf, log_cnt: &state->log_cnt);
1121
1122	if (test->run_test)
1123	test->run_test();
1124	else if (test->run_serial_test)
1125	test->run_serial_test();
1126
1127	/* ensure last sub-test is finalized properly */
1128	if (env.subtest_state)
1129	test__end_subtest();
1130
1131	state->tested = true;
1132
1133	if (verbose() && env.worker_id == -1)
1134	print_test_result(test, test_state: state);
1135
1136	reset_affinity();
1137	restore_netns();
1138	if (test->need_cgroup_cleanup)
1139	cleanup_cgroup_environment();
1140
1141	stdio_restore();
1142	free(stop_libbpf_log_capture());
1143
1144	dump_test_log(test, test_state: state, skip_ok_subtests: false, par_exec_result: false, NULL);
1145	}
1146
1147	struct dispatch_data {
1148	int worker_id;
1149	int sock_fd;
1150	};
1151
1152	static int read_prog_test_msg(int sock_fd, struct msg *msg, enum msg_type type)
1153	{
1154	if (recv_message(sock: sock_fd, msg) < 0)
1155	return 1;
1156
1157	if (msg->type != type) {
1158	printf("%s: unexpected message type %d. expected %d\n", __func__, msg->type, type);
1159	return 1;
1160	}
1161
1162	return 0;
1163	}
1164
1165	static int dispatch_thread_read_log(int sock_fd, char **log_buf, size_t *log_cnt)
1166	{
1167	FILE *log_fp = NULL;
1168	int result = 0;
1169
1170	log_fp = open_memstream(log_buf, log_cnt);
1171	if (!log_fp)
1172	return 1;
1173
1174	while (true) {
1175	struct msg msg;
1176
1177	if (read_prog_test_msg(sock_fd, msg: &msg, type: MSG_TEST_LOG)) {
1178	result = 1;
1179	goto out;
1180	}
1181
1182	fprintf(log_fp, "%s", msg.test_log.log_buf);
1183	if (msg.test_log.is_last)
1184	break;
1185	}
1186
1187	out:
1188	fclose(log_fp);
1189	log_fp = NULL;
1190	return result;
1191	}
1192
1193	static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state)
1194	{
1195	struct msg msg;
1196	struct subtest_state *subtest_state;
1197	int subtest_num = state->subtest_num;
1198
1199	state->subtest_states = malloc(subtest_num * sizeof(*subtest_state));
1200
1201	for (int i = 0; i < subtest_num; i++) {
1202	subtest_state = &state->subtest_states[i];
1203
1204	memset(subtest_state, 0, sizeof(*subtest_state));
1205
1206	if (read_prog_test_msg(sock_fd, msg: &msg, type: MSG_SUBTEST_DONE))
1207	return 1;
1208
1209	subtest_state->name = strdup(msg.subtest_done.name);
1210	subtest_state->error_cnt = msg.subtest_done.error_cnt;
1211	subtest_state->skipped = msg.subtest_done.skipped;
1212	subtest_state->filtered = msg.subtest_done.filtered;
1213
1214	/* collect all logs */
1215	if (msg.subtest_done.have_log)
1216	if (dispatch_thread_read_log(sock_fd,
1217	log_buf: &subtest_state->log_buf,
1218	log_cnt: &subtest_state->log_cnt))
1219	return 1;
1220	}
1221
1222	return 0;
1223	}
1224
1225	static void *dispatch_thread(void *ctx)
1226	{
1227	struct dispatch_data *data = ctx;
1228	int sock_fd;
1229
1230	sock_fd = data->sock_fd;
1231
1232	while (true) {
1233	int test_to_run = -1;
1234	struct prog_test_def *test;
1235	struct test_state *state;
1236
1237	/* grab a test */
1238	{
1239	pthread_mutex_lock(&current_test_lock);
1240
1241	if (current_test_idx >= prog_test_cnt) {
1242	pthread_mutex_unlock(&current_test_lock);
1243	goto done;
1244	}
1245
1246	test = &prog_test_defs[current_test_idx];
1247	test_to_run = current_test_idx;
1248	current_test_idx++;
1249
1250	pthread_mutex_unlock(&current_test_lock);
1251	}
1252
1253	if (!test->should_run || test->run_serial_test)
1254	continue;
1255
1256	/* run test through worker */
1257	{
1258	struct msg msg_do_test;
1259
1260	memset(&msg_do_test, 0, sizeof(msg_do_test));
1261	msg_do_test.type = MSG_DO_TEST;
1262	msg_do_test.do_test.num = test_to_run;
1263	if (send_message(sock: sock_fd, msg: &msg_do_test) < 0) {
1264	perror("Fail to send command");
1265	goto done;
1266	}
1267	env.worker_current_test[data->worker_id] = test_to_run;
1268	}
1269
1270	/* wait for test done */
1271	do {
1272	struct msg msg;
1273
1274	if (read_prog_test_msg(sock_fd, msg: &msg, type: MSG_TEST_DONE))
1275	goto error;
1276	if (test_to_run != msg.test_done.num)
1277	goto error;
1278
1279	state = &test_states[test_to_run];
1280	state->tested = true;
1281	state->error_cnt = msg.test_done.error_cnt;
1282	state->skip_cnt = msg.test_done.skip_cnt;
1283	state->sub_succ_cnt = msg.test_done.sub_succ_cnt;
1284	state->subtest_num = msg.test_done.subtest_num;
1285
1286	/* collect all logs */
1287	if (msg.test_done.have_log) {
1288	if (dispatch_thread_read_log(sock_fd,
1289	log_buf: &state->log_buf,
1290	log_cnt: &state->log_cnt))
1291	goto error;
1292	}
1293
1294	/* collect all subtests and subtest logs */
1295	if (!state->subtest_num)
1296	break;
1297
1298	if (dispatch_thread_send_subtests(sock_fd, state))
1299	goto error;
1300	} while (false);
1301
1302	pthread_mutex_lock(&stdout_output_lock);
1303	dump_test_log(test, test_state: state, skip_ok_subtests: false, par_exec_result: true, NULL);
1304	pthread_mutex_unlock(&stdout_output_lock);
1305	} /* while (true) */
1306	error:
1307	if (env.debug)
1308	fprintf(stderr, "[%d]: Protocol/IO error: %s.\n", data->worker_id, strerror(errno));
1309
1310	done:
1311	{
1312	struct msg msg_exit;
1313
1314	msg_exit.type = MSG_EXIT;
1315	if (send_message(sock: sock_fd, msg: &msg_exit) < 0) {
1316	if (env.debug)
1317	fprintf(stderr, "[%d]: send_message msg_exit: %s.\n",
1318	data->worker_id, strerror(errno));
1319	}
1320	}
1321	return NULL;
1322	}
1323
1324	static void calculate_summary_and_print_errors(struct test_env *env)
1325	{
1326	int i;
1327	int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, skip_cnt = 0;
1328	json_writer_t *w = NULL;
1329
1330	for (i = 0; i < prog_test_cnt; i++) {
1331	struct test_state *state = &test_states[i];
1332
1333	if (!state->tested)
1334	continue;
1335
1336	sub_succ_cnt += state->sub_succ_cnt;
1337	skip_cnt += state->skip_cnt;
1338
1339	if (state->error_cnt)
1340	fail_cnt++;
1341	else
1342	succ_cnt++;
1343	}
1344
1345	if (env->json) {
1346	w = jsonw_new(env->json);
1347	if (!w)
1348	fprintf(env->stderr, "Failed to create new JSON stream.");
1349	}
1350
1351	if (w) {
1352	jsonw_start_object(self: w);
1353	jsonw_uint_field(self: w, prop: "success", num: succ_cnt);
1354	jsonw_uint_field(self: w, prop: "success_subtest", num: sub_succ_cnt);
1355	jsonw_uint_field(self: w, prop: "skipped", num: skip_cnt);
1356	jsonw_uint_field(self: w, prop: "failed", num: fail_cnt);
1357	jsonw_name(self: w, name: "results");
1358	jsonw_start_array(self: w);
1359	}
1360
1361	/*
1362	* We only print error logs summary when there are failed tests and
1363	* verbose mode is not enabled. Otherwise, results may be incosistent.
1364	*
1365	*/
1366	if (!verbose() && fail_cnt) {
1367	printf("\nAll error logs:\n");
1368
1369	/* print error logs again */
1370	for (i = 0; i < prog_test_cnt; i++) {
1371	struct prog_test_def *test = &prog_test_defs[i];
1372	struct test_state *state = &test_states[i];
1373
1374	if (!state->tested || !state->error_cnt)
1375	continue;
1376
1377	dump_test_log(test, test_state: state, skip_ok_subtests: true, par_exec_result: true, w);
1378	}
1379	}
1380
1381	if (w) {
1382	jsonw_end_array(self: w);
1383	jsonw_end_object(self: w);
1384	jsonw_destroy(self_p: &w);
1385	}
1386
1387	if (env->json)
1388	fclose(env->json);
1389
1390	printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n",
1391	succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt);
1392
1393	env->succ_cnt = succ_cnt;
1394	env->sub_succ_cnt = sub_succ_cnt;
1395	env->fail_cnt = fail_cnt;
1396	env->skip_cnt = skip_cnt;
1397	}
1398
1399	static void server_main(void)
1400	{
1401	pthread_t *dispatcher_threads;
1402	struct dispatch_data *data;
1403	struct sigaction sigact_int = {
1404	.sa_handler = sigint_handler,
1405	.sa_flags = SA_RESETHAND,
1406	};
1407	int i;
1408
1409	sigaction(SIGINT, &sigact_int, NULL);
1410
1411	dispatcher_threads = calloc(sizeof(pthread_t), env.workers);
1412	data = calloc(sizeof(struct dispatch_data), env.workers);
1413
1414	env.worker_current_test = calloc(sizeof(int), env.workers);
1415	for (i = 0; i < env.workers; i++) {
1416	int rc;
1417
1418	data[i].worker_id = i;
1419	data[i].sock_fd = env.worker_socks[i];
1420	rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]);
1421	if (rc < 0) {
1422	perror("Failed to launch dispatcher thread");
1423	exit(EXIT_ERR_SETUP_INFRA);
1424	}
1425	}
1426
1427	/* wait for all dispatcher to finish */
1428	for (i = 0; i < env.workers; i++) {
1429	while (true) {
1430	int ret = pthread_tryjoin_np(dispatcher_threads[i], NULL);
1431
1432	if (!ret) {
1433	break;
1434	} else if (ret == EBUSY) {
1435	if (env.debug)
1436	fprintf(stderr, "Still waiting for thread %d (test %d).\n",
1437	i, env.worker_current_test[i] + 1);
1438	usleep(usec: 1000 * 1000);
1439	continue;
1440	} else {
1441	fprintf(stderr, "Unexpected error joining dispatcher thread: %d", ret);
1442	break;
1443	}
1444	}
1445	}
1446	free(dispatcher_threads);
1447	free(env.worker_current_test);
1448	free(data);
1449
1450	/* run serial tests */
1451	save_netns();
1452
1453	for (int i = 0; i < prog_test_cnt; i++) {
1454	struct prog_test_def *test = &prog_test_defs[i];
1455
1456	if (!test->should_run || !test->run_serial_test)
1457	continue;
1458
1459	run_one_test(test_num: i);
1460	}
1461
1462	/* generate summary */
1463	fflush(stderr);
1464	fflush(stdout);
1465
1466	calculate_summary_and_print_errors(env: &env);
1467
1468	/* reap all workers */
1469	for (i = 0; i < env.workers; i++) {
1470	int wstatus, pid;
1471
1472	pid = waitpid(env.worker_pids[i], &wstatus, 0);
1473	if (pid != env.worker_pids[i])
1474	perror("Unable to reap worker");
1475	}
1476	}
1477
1478	static void worker_main_send_log(int sock, char *log_buf, size_t log_cnt)
1479	{
1480	char *src;
1481	size_t slen;
1482
1483	src = log_buf;
1484	slen = log_cnt;
1485	while (slen) {
1486	struct msg msg_log;
1487	char *dest;
1488	size_t len;
1489
1490	memset(&msg_log, 0, sizeof(msg_log));
1491	msg_log.type = MSG_TEST_LOG;
1492	dest = msg_log.test_log.log_buf;
1493	len = slen >= MAX_LOG_TRUNK_SIZE ? MAX_LOG_TRUNK_SIZE : slen;
1494	memcpy(dest, src, len);
1495
1496	src += len;
1497	slen -= len;
1498	if (!slen)
1499	msg_log.test_log.is_last = true;
1500
1501	assert(send_message(sock, msg: &msg_log) >= 0);
1502	}
1503	}
1504
1505	static void free_subtest_state(struct subtest_state *state)
1506	{
1507	if (state->log_buf) {
1508	free(state->log_buf);
1509	state->log_buf = NULL;
1510	state->log_cnt = 0;
1511	}
1512	free(state->name);
1513	state->name = NULL;
1514	}
1515
1516	static int worker_main_send_subtests(int sock, struct test_state *state)
1517	{
1518	int i, result = 0;
1519	struct msg msg;
1520	struct subtest_state *subtest_state;
1521
1522	memset(&msg, 0, sizeof(msg));
1523	msg.type = MSG_SUBTEST_DONE;
1524
1525	for (i = 0; i < state->subtest_num; i++) {
1526	subtest_state = &state->subtest_states[i];
1527
1528	msg.subtest_done.num = i;
1529
1530	strncpy(p: msg.subtest_done.name, q: subtest_state->name, MAX_SUBTEST_NAME);
1531
1532	msg.subtest_done.error_cnt = subtest_state->error_cnt;
1533	msg.subtest_done.skipped = subtest_state->skipped;
1534	msg.subtest_done.filtered = subtest_state->filtered;
1535	msg.subtest_done.have_log = false;
1536
1537	if (verbose() || state->force_log || subtest_state->error_cnt) {
1538	if (subtest_state->log_cnt)
1539	msg.subtest_done.have_log = true;
1540	}
1541
1542	if (send_message(sock, msg: &msg) < 0) {
1543	perror("Fail to send message done");
1544	result = 1;
1545	goto out;
1546	}
1547
1548	/* send logs */
1549	if (msg.subtest_done.have_log)
1550	worker_main_send_log(sock, log_buf: subtest_state->log_buf, log_cnt: subtest_state->log_cnt);
1551
1552	free_subtest_state(state: subtest_state);
1553	free(subtest_state->name);
1554	}
1555
1556	out:
1557	for (; i < state->subtest_num; i++)
1558	free_subtest_state(state: &state->subtest_states[i]);
1559	free(state->subtest_states);
1560	return result;
1561	}
1562
1563	static int worker_main(int sock)
1564	{
1565	save_netns();
1566
1567	while (true) {
1568	/* receive command */
1569	struct msg msg;
1570
1571	if (recv_message(sock, msg: &msg) < 0)
1572	goto out;
1573
1574	switch (msg.type) {
1575	case MSG_EXIT:
1576	if (env.debug)
1577	fprintf(stderr, "[%d]: worker exit.\n",
1578	env.worker_id);
1579	goto out;
1580	case MSG_DO_TEST: {
1581	int test_to_run = msg.do_test.num;
1582	struct prog_test_def *test = &prog_test_defs[test_to_run];
1583	struct test_state *state = &test_states[test_to_run];
1584	struct msg msg;
1585
1586	if (env.debug)
1587	fprintf(stderr, "[%d]: #%d:%s running.\n",
1588	env.worker_id,
1589	test_to_run + 1,
1590	test->test_name);
1591
1592	run_one_test(test_num: test_to_run);
1593
1594	memset(&msg, 0, sizeof(msg));
1595	msg.type = MSG_TEST_DONE;
1596	msg.test_done.num = test_to_run;
1597	msg.test_done.error_cnt = state->error_cnt;
1598	msg.test_done.skip_cnt = state->skip_cnt;
1599	msg.test_done.sub_succ_cnt = state->sub_succ_cnt;
1600	msg.test_done.subtest_num = state->subtest_num;
1601	msg.test_done.have_log = false;
1602
1603	if (verbose() || state->force_log || state->error_cnt) {
1604	if (state->log_cnt)
1605	msg.test_done.have_log = true;
1606	}
1607	if (send_message(sock, msg: &msg) < 0) {
1608	perror("Fail to send message done");
1609	goto out;
1610	}
1611
1612	/* send logs */
1613	if (msg.test_done.have_log)
1614	worker_main_send_log(sock, log_buf: state->log_buf, log_cnt: state->log_cnt);
1615
1616	if (state->log_buf) {
1617	free(state->log_buf);
1618	state->log_buf = NULL;
1619	state->log_cnt = 0;
1620	}
1621
1622	if (state->subtest_num)
1623	if (worker_main_send_subtests(sock, state))
1624	goto out;
1625
1626	if (env.debug)
1627	fprintf(stderr, "[%d]: #%d:%s done.\n",
1628	env.worker_id,
1629	test_to_run + 1,
1630	test->test_name);
1631	break;
1632	} /* case MSG_DO_TEST */
1633	default:
1634	if (env.debug)
1635	fprintf(stderr, "[%d]: unknown message.\n", env.worker_id);
1636	return -1;
1637	}
1638	}
1639	out:
1640	return 0;
1641	}
1642
1643	static void free_test_states(void)
1644	{
1645	int i, j;
1646
1647	for (i = 0; i < ARRAY_SIZE(prog_test_defs); i++) {
1648	struct test_state *test_state = &test_states[i];
1649
1650	for (j = 0; j < test_state->subtest_num; j++)
1651	free_subtest_state(state: &test_state->subtest_states[j]);
1652
1653	free(test_state->subtest_states);
1654	free(test_state->log_buf);
1655	test_state->subtest_states = NULL;
1656	test_state->log_buf = NULL;
1657	}
1658	}
1659
1660	int main(int argc, char **argv)
1661	{
1662	static const struct argp argp = {
1663	.options = opts,
1664	.parser = parse_arg,
1665	.doc = argp_program_doc,
1666	};
1667	struct sigaction sigact = {
1668	.sa_handler = crash_handler,
1669	.sa_flags = SA_RESETHAND,
1670	};
1671	int err, i;
1672
1673	sigaction(SIGSEGV, &sigact, NULL);
1674
1675	err = argp_parse(&argp, argc, argv, 0, NULL, &env);
1676	if (err)
1677	return err;
1678
1679	err = cd_flavor_subdir(exec_name: argv[0]);
1680	if (err)
1681	return err;
1682
1683	/* Use libbpf 1.0 API mode */
1684	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1685	libbpf_set_print(libbpf_print_fn);
1686
1687	srand(time(NULL));
1688
1689	env.jit_enabled = is_jit_enabled();
1690	env.nr_cpus = libbpf_num_possible_cpus();
1691	if (env.nr_cpus < 0) {
1692	fprintf(stderr, "Failed to get number of CPUs: %d!\n",
1693	env.nr_cpus);
1694	return -1;
1695	}
1696
1697	env.stdout = stdout;
1698	env.stderr = stderr;
1699
1700	env.has_testmod = true;
1701	if (!env.list_test_names) {
1702	/* ensure previous instance of the module is unloaded */
1703	unload_bpf_testmod(verbose: verbose());
1704
1705	if (load_bpf_testmod(verbose: verbose())) {
1706	fprintf(env.stderr, "WARNING! Selftests relying on bpf_testmod.ko will be skipped.\n");
1707	env.has_testmod = false;
1708	}
1709	}
1710
1711	/* initializing tests */
1712	for (i = 0; i < prog_test_cnt; i++) {
1713	struct prog_test_def *test = &prog_test_defs[i];
1714
1715	test->test_num = i + 1;
1716	test->should_run = should_run(sel: &env.test_selector,
1717	num: test->test_num, name: test->test_name);
1718
1719	if ((test->run_test == NULL && test->run_serial_test == NULL) ||
1720	(test->run_test != NULL && test->run_serial_test != NULL)) {
1721	fprintf(stderr, "Test %d:%s must have either test_%s() or serial_test_%sl() defined.\n",
1722	test->test_num, test->test_name, test->test_name, test->test_name);
1723	exit(EXIT_ERR_SETUP_INFRA);
1724	}
1725	}
1726
1727	/* ignore workers if we are just listing */
1728	if (env.get_test_cnt || env.list_test_names)
1729	env.workers = 0;
1730
1731	/* launch workers if requested */
1732	env.worker_id = -1; /* main process */
1733	if (env.workers) {
1734	env.worker_pids = calloc(sizeof(__pid_t), env.workers);
1735	env.worker_socks = calloc(sizeof(int), env.workers);
1736	if (env.debug)
1737	fprintf(stdout, "Launching %d workers.\n", env.workers);
1738	for (i = 0; i < env.workers; i++) {
1739	int sv[2];
1740	pid_t pid;
1741
1742	if (socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, sv) < 0) {
1743	perror("Fail to create worker socket");
1744	return -1;
1745	}
1746	pid = fork();
1747	if (pid < 0) {
1748	perror("Failed to fork worker");
1749	return -1;
1750	} else if (pid != 0) { /* main process */
1751	close(sv[1]);
1752	env.worker_pids[i] = pid;
1753	env.worker_socks[i] = sv[0];
1754	} else { /* inside each worker process */
1755	close(sv[0]);
1756	env.worker_id = i;
1757	return worker_main(sock: sv[1]);
1758	}
1759	}
1760
1761	if (env.worker_id == -1) {
1762	server_main();
1763	goto out;
1764	}
1765	}
1766
1767	/* The rest of the main process */
1768
1769	/* on single mode */
1770	save_netns();
1771
1772	for (i = 0; i < prog_test_cnt; i++) {
1773	struct prog_test_def *test = &prog_test_defs[i];
1774
1775	if (!test->should_run)
1776	continue;
1777
1778	if (env.get_test_cnt) {
1779	env.succ_cnt++;
1780	continue;
1781	}
1782
1783	if (env.list_test_names) {
1784	fprintf(env.stdout, "%s\n", test->test_name);
1785	env.succ_cnt++;
1786	continue;
1787	}
1788
1789	run_one_test(test_num: i);
1790	}
1791
1792	if (env.get_test_cnt) {
1793	printf("%d\n", env.succ_cnt);
1794	goto out;
1795	}
1796
1797	if (env.list_test_names)
1798	goto out;
1799
1800	calculate_summary_and_print_errors(env: &env);
1801
1802	close(env.saved_netns_fd);
1803	out:
1804	if (!env.list_test_names && env.has_testmod)
1805	unload_bpf_testmod(verbose: verbose());
1806
1807	free_test_selector(test_selector: &env.test_selector);
1808	free_test_selector(test_selector: &env.subtest_selector);
1809	free_test_states();
1810
1811	if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0)
1812	return EXIT_NO_TEST;
1813
1814	return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
1815	}
1816