1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Testsuite for eBPF verifier
4	*
5	* Copyright (c) 2014 PLUMgrid, http://plumgrid.com
6	* Copyright (c) 2017 Facebook
7	* Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
8	*/
9
10	#include <endian.h>
11	#include <asm/types.h>
12	#include <linux/types.h>
13	#include <stdint.h>
14	#include <stdio.h>
15	#include <stdlib.h>
16	#include <unistd.h>
17	#include <errno.h>
18	#include <string.h>
19	#include <stddef.h>
20	#include <stdbool.h>
21	#include <sched.h>
22	#include <limits.h>
23	#include <assert.h>
24
25	#include <linux/unistd.h>
26	#include <linux/filter.h>
27	#include <linux/bpf_perf_event.h>
28	#include <linux/bpf.h>
29	#include <linux/if_ether.h>
30	#include <linux/btf.h>
31
32	#include <bpf/btf.h>
33	#include <bpf/bpf.h>
34	#include <bpf/libbpf.h>
35
36	#include "autoconf_helper.h"
37	#include "unpriv_helpers.h"
38	#include "cap_helpers.h"
39	#include "bpf_rand.h"
40	#include "bpf_util.h"
41	#include "test_btf.h"
42	#include "../../../include/linux/filter.h"
43	#include "testing_helpers.h"
44
45	#ifndef ENOTSUPP
46	#define ENOTSUPP 524
47	#endif
48
49	#define MAX_INSNS BPF_MAXINSNS
50	#define MAX_EXPECTED_INSNS 32
51	#define MAX_UNEXPECTED_INSNS 32
52	#define MAX_TEST_INSNS 1000000
53	#define MAX_FIXUPS 8
54	#define MAX_NR_MAPS 23
55	#define MAX_TEST_RUNS 8
56	#define POINTER_VALUE 0xcafe4all
57	#define TEST_DATA_LEN 64
58	#define MAX_FUNC_INFOS 8
59	#define MAX_BTF_STRINGS 256
60	#define MAX_BTF_TYPES 256
61
62	#define INSN_OFF_MASK ((__s16)0xFFFF)
63	#define INSN_IMM_MASK ((__s32)0xFFFFFFFF)
64	#define SKIP_INSNS() BPF_RAW_INSN(0xde, 0xa, 0xd, 0xbeef, 0xdeadbeef)
65
66	#define DEFAULT_LIBBPF_LOG_LEVEL 4
67
68	#define F_NEEDS_EFFICIENT_UNALIGNED_ACCESS (1 << 0)
69	#define F_LOAD_WITH_STRICT_ALIGNMENT (1 << 1)
70	#define F_NEEDS_JIT_ENABLED (1 << 2)
71
72	/* need CAP_BPF, CAP_NET_ADMIN, CAP_PERFMON to load progs */
73	#define ADMIN_CAPS (1ULL << CAP_NET_ADMIN | \
74	1ULL << CAP_PERFMON | \
75	1ULL << CAP_BPF)
76	#define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled"
77	static bool unpriv_disabled = false;
78	static bool jit_disabled;
79	static int skips;
80	static bool verbose = false;
81	static int verif_log_level = 0;
82
83	struct kfunc_btf_id_pair {
84	const char *kfunc;
85	int insn_idx;
86	};
87
88	struct bpf_test {
89	const char *descr;
90	struct bpf_insn insns[MAX_INSNS];
91	struct bpf_insn *fill_insns;
92	/* If specified, test engine looks for this sequence of
93	* instructions in the BPF program after loading. Allows to
94	* test rewrites applied by verifier. Use values
95	* INSN_OFF_MASK and INSN_IMM_MASK to mask `off` and `imm`
96	* fields if content does not matter. The test case fails if
97	* specified instructions are not found.
98	*
99	* The sequence could be split into sub-sequences by adding
100	* SKIP_INSNS instruction at the end of each sub-sequence. In
101	* such case sub-sequences are searched for one after another.
102	*/
103	struct bpf_insn expected_insns[MAX_EXPECTED_INSNS];
104	/* If specified, test engine applies same pattern matching
105	* logic as for `expected_insns`. If the specified pattern is
106	* matched test case is marked as failed.
107	*/
108	struct bpf_insn unexpected_insns[MAX_UNEXPECTED_INSNS];
109	int fixup_map_hash_8b[MAX_FIXUPS];
110	int fixup_map_hash_48b[MAX_FIXUPS];
111	int fixup_map_hash_16b[MAX_FIXUPS];
112	int fixup_map_array_48b[MAX_FIXUPS];
113	int fixup_map_sockmap[MAX_FIXUPS];
114	int fixup_map_sockhash[MAX_FIXUPS];
115	int fixup_map_xskmap[MAX_FIXUPS];
116	int fixup_map_stacktrace[MAX_FIXUPS];
117	int fixup_prog1[MAX_FIXUPS];
118	int fixup_prog2[MAX_FIXUPS];
119	int fixup_map_in_map[MAX_FIXUPS];
120	int fixup_cgroup_storage[MAX_FIXUPS];
121	int fixup_percpu_cgroup_storage[MAX_FIXUPS];
122	int fixup_map_spin_lock[MAX_FIXUPS];
123	int fixup_map_array_ro[MAX_FIXUPS];
124	int fixup_map_array_wo[MAX_FIXUPS];
125	int fixup_map_array_small[MAX_FIXUPS];
126	int fixup_sk_storage_map[MAX_FIXUPS];
127	int fixup_map_event_output[MAX_FIXUPS];
128	int fixup_map_reuseport_array[MAX_FIXUPS];
129	int fixup_map_ringbuf[MAX_FIXUPS];
130	int fixup_map_timer[MAX_FIXUPS];
131	int fixup_map_kptr[MAX_FIXUPS];
132	struct kfunc_btf_id_pair fixup_kfunc_btf_id[MAX_FIXUPS];
133	/* Expected verifier log output for result REJECT or VERBOSE_ACCEPT.
134	* Can be a tab-separated sequence of expected strings. An empty string
135	* means no log verification.
136	*/
137	const char *errstr;
138	const char *errstr_unpriv;
139	uint32_t insn_processed;
140	int prog_len;
141	enum {
142	UNDEF,
143	ACCEPT,
144	REJECT,
145	VERBOSE_ACCEPT,
146	} result, result_unpriv;
147	enum bpf_prog_type prog_type;
148	uint8_t flags;
149	void (*fill_helper)(struct bpf_test *self);
150	int runs;
151	#define bpf_testdata_struct_t \
152	struct { \
153	uint32_t retval, retval_unpriv; \
154	union { \
155	__u8 data[TEST_DATA_LEN]; \
156	__u64 data64[TEST_DATA_LEN / 8]; \
157	}; \
158	}
159	union {
160	bpf_testdata_struct_t;
161	bpf_testdata_struct_t retvals[MAX_TEST_RUNS];
162	};
163	enum bpf_attach_type expected_attach_type;
164	const char *kfunc;
165	struct bpf_func_info func_info[MAX_FUNC_INFOS];
166	int func_info_cnt;
167	char btf_strings[MAX_BTF_STRINGS];
168	/* A set of BTF types to load when specified,
169	* use macro definitions from test_btf.h,
170	* must end with BTF_END_RAW
171	*/
172	__u32 btf_types[MAX_BTF_TYPES];
173	};
174
175	/* Note we want this to be 64 bit aligned so that the end of our array is
176	* actually the end of the structure.
177	*/
178	#define MAX_ENTRIES 11
179
180	struct test_val {
181	unsigned int index;
182	int foo[MAX_ENTRIES];
183	};
184
185	struct other_val {
186	long long foo;
187	long long bar;
188	};
189
190	static void bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self)
191	{
192	/* test: {skb->data[0], vlan_push} x 51 + {skb->data[0], vlan_pop} x 51 */
193	#define PUSH_CNT 51
194	/* jump range is limited to 16 bit. PUSH_CNT of ld_abs needs room */
195	unsigned int len = (1 << 15) - PUSH_CNT * 2 * 5 * 6;
196	struct bpf_insn *insn = self->fill_insns;
197	int i = 0, j, k = 0;
198
199	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
200	loop:
201	for (j = 0; j < PUSH_CNT; j++) {
202	insn[i++] = BPF_LD_ABS(BPF_B, 0);
203	/* jump to error label */
204	insn[i] = BPF_JMP32_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 3);
205	i++;
206	insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
207	insn[i++] = BPF_MOV64_IMM(BPF_REG_2, 1);
208	insn[i++] = BPF_MOV64_IMM(BPF_REG_3, 2);
209	insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
210	BPF_FUNC_skb_vlan_push);
211	insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 3);
212	i++;
213	}
214
215	for (j = 0; j < PUSH_CNT; j++) {
216	insn[i++] = BPF_LD_ABS(BPF_B, 0);
217	insn[i] = BPF_JMP32_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 3);
218	i++;
219	insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
220	insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
221	BPF_FUNC_skb_vlan_pop);
222	insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 3);
223	i++;
224	}
225	if (++k < 5)
226	goto loop;
227
228	for (; i < len - 3; i++)
229	insn[i] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0xbef);
230	insn[len - 3] = BPF_JMP_A(1);
231	/* error label */
232	insn[len - 2] = BPF_MOV32_IMM(BPF_REG_0, 0);
233	insn[len - 1] = BPF_EXIT_INSN();
234	self->prog_len = len;
235	}
236
237	static void bpf_fill_jump_around_ld_abs(struct bpf_test *self)
238	{
239	struct bpf_insn *insn = self->fill_insns;
240	/* jump range is limited to 16 bit. every ld_abs is replaced by 6 insns,
241	* but on arches like arm, ppc etc, there will be one BPF_ZEXT inserted
242	* to extend the error value of the inlined ld_abs sequence which then
243	* contains 7 insns. so, set the dividend to 7 so the testcase could
244	* work on all arches.
245	*/
246	unsigned int len = (1 << 15) / 7;
247	int i = 0;
248
249	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
250	insn[i++] = BPF_LD_ABS(BPF_B, 0);
251	insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 10, len - i - 2);
252	i++;
253	while (i < len - 1)
254	insn[i++] = BPF_LD_ABS(BPF_B, 1);
255	insn[i] = BPF_EXIT_INSN();
256	self->prog_len = i + 1;
257	}
258
259	static void bpf_fill_rand_ld_dw(struct bpf_test *self)
260	{
261	struct bpf_insn *insn = self->fill_insns;
262	uint64_t res = 0;
263	int i = 0;
264
265	insn[i++] = BPF_MOV32_IMM(BPF_REG_0, 0);
266	while (i < self->retval) {
267	uint64_t val = bpf_semi_rand_get();
268	struct bpf_insn tmp[2] = { BPF_LD_IMM64(BPF_REG_1, val) };
269
270	res ^= val;
271	insn[i++] = tmp[0];
272	insn[i++] = tmp[1];
273	insn[i++] = BPF_ALU64_REG(BPF_XOR, BPF_REG_0, BPF_REG_1);
274	}
275	insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
276	insn[i++] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_1, 32);
277	insn[i++] = BPF_ALU64_REG(BPF_XOR, BPF_REG_0, BPF_REG_1);
278	insn[i] = BPF_EXIT_INSN();
279	self->prog_len = i + 1;
280	res ^= (res >> 32);
281	self->retval = (uint32_t)res;
282	}
283
284	#define MAX_JMP_SEQ 8192
285
286	/* test the sequence of 8k jumps */
287	static void bpf_fill_scale1(struct bpf_test *self)
288	{
289	struct bpf_insn *insn = self->fill_insns;
290	int i = 0, k = 0;
291
292	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
293	/* test to check that the long sequence of jumps is acceptable */
294	while (k++ < MAX_JMP_SEQ) {
295	insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
296	BPF_FUNC_get_prandom_u32);
297	insn[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, bpf_semi_rand_get(), 2);
298	insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_10);
299	insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6,
300	-8 * (k % 64 + 1));
301	}
302	/* is_state_visited() doesn't allocate state for pruning for every jump.
303	* Hence multiply jmps by 4 to accommodate that heuristic
304	*/
305	while (i < MAX_TEST_INSNS - MAX_JMP_SEQ * 4)
306	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 42);
307	insn[i] = BPF_EXIT_INSN();
308	self->prog_len = i + 1;
309	self->retval = 42;
310	}
311
312	/* test the sequence of 8k jumps in inner most function (function depth 8)*/
313	static void bpf_fill_scale2(struct bpf_test *self)
314	{
315	struct bpf_insn *insn = self->fill_insns;
316	int i = 0, k = 0;
317
318	#define FUNC_NEST 7
319	for (k = 0; k < FUNC_NEST; k++) {
320	insn[i++] = BPF_CALL_REL(1);
321	insn[i++] = BPF_EXIT_INSN();
322	}
323	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
324	/* test to check that the long sequence of jumps is acceptable */
325	k = 0;
326	while (k++ < MAX_JMP_SEQ) {
327	insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
328	BPF_FUNC_get_prandom_u32);
329	insn[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, bpf_semi_rand_get(), 2);
330	insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_10);
331	insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6,
332	-8 * (k % (64 - 4 * FUNC_NEST) + 1));
333	}
334	while (i < MAX_TEST_INSNS - MAX_JMP_SEQ * 4)
335	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 42);
336	insn[i] = BPF_EXIT_INSN();
337	self->prog_len = i + 1;
338	self->retval = 42;
339	}
340
341	static void bpf_fill_scale(struct bpf_test *self)
342	{
343	switch (self->retval) {
344	case 1:
345	return bpf_fill_scale1(self);
346	case 2:
347	return bpf_fill_scale2(self);
348	default:
349	self->prog_len = 0;
350	break;
351	}
352	}
353
354	static int bpf_fill_torturous_jumps_insn_1(struct bpf_insn *insn)
355	{
356	unsigned int len = 259, hlen = 128;
357	int i;
358
359	insn[0] = BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32);
360	for (i = 1; i <= hlen; i++) {
361	insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, i, hlen);
362	insn[i + hlen] = BPF_JMP_A(hlen - i);
363	}
364	insn[len - 2] = BPF_MOV64_IMM(BPF_REG_0, 1);
365	insn[len - 1] = BPF_EXIT_INSN();
366
367	return len;
368	}
369
370	static int bpf_fill_torturous_jumps_insn_2(struct bpf_insn *insn)
371	{
372	unsigned int len = 4100, jmp_off = 2048;
373	int i, j;
374
375	insn[0] = BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32);
376	for (i = 1; i <= jmp_off; i++) {
377	insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, i, jmp_off);
378	}
379	insn[i++] = BPF_JMP_A(jmp_off);
380	for (; i <= jmp_off * 2 + 1; i+=16) {
381	for (j = 0; j < 16; j++) {
382	insn[i + j] = BPF_JMP_A(16 - j - 1);
383	}
384	}
385
386	insn[len - 2] = BPF_MOV64_IMM(BPF_REG_0, 2);
387	insn[len - 1] = BPF_EXIT_INSN();
388
389	return len;
390	}
391
392	static void bpf_fill_torturous_jumps(struct bpf_test *self)
393	{
394	struct bpf_insn *insn = self->fill_insns;
395	int i = 0;
396
397	switch (self->retval) {
398	case 1:
399	self->prog_len = bpf_fill_torturous_jumps_insn_1(insn);
400	return;
401	case 2:
402	self->prog_len = bpf_fill_torturous_jumps_insn_2(insn);
403	return;
404	case 3:
405	/* main */
406	insn[i++] = BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4);
407	insn[i++] = BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 262);
408	insn[i++] = BPF_ST_MEM(BPF_B, BPF_REG_10, -32, 0);
409	insn[i++] = BPF_MOV64_IMM(BPF_REG_0, 3);
410	insn[i++] = BPF_EXIT_INSN();
411
412	/* subprog 1 */
413	i += bpf_fill_torturous_jumps_insn_1(insn: insn + i);
414
415	/* subprog 2 */
416	i += bpf_fill_torturous_jumps_insn_2(insn: insn + i);
417
418	self->prog_len = i;
419	return;
420	default:
421	self->prog_len = 0;
422	break;
423	}
424	}
425
426	static void bpf_fill_big_prog_with_loop_1(struct bpf_test *self)
427	{
428	struct bpf_insn *insn = self->fill_insns;
429	/* This test was added to catch a specific use after free
430	* error, which happened upon BPF program reallocation.
431	* Reallocation is handled by core.c:bpf_prog_realloc, which
432	* reuses old memory if page boundary is not crossed. The
433	* value of `len` is chosen to cross this boundary on bpf_loop
434	* patching.
435	*/
436	const int len = getpagesize() - 25;
437	int callback_load_idx;
438	int callback_idx;
439	int i = 0;
440
441	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1);
442	callback_load_idx = i;
443	insn[i++] = BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW,
444	BPF_REG_2, BPF_PSEUDO_FUNC, 0,
445	777 /* filled below */);
446	insn[i++] = BPF_RAW_INSN(0, 0, 0, 0, 0);
447	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0);
448	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0);
449	insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop);
450
451	while (i < len - 3)
452	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0);
453	insn[i++] = BPF_EXIT_INSN();
454
455	callback_idx = i;
456	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0);
457	insn[i++] = BPF_EXIT_INSN();
458
459	insn[callback_load_idx].imm = callback_idx - callback_load_idx - 1;
460	self->func_info[1].insn_off = callback_idx;
461	self->prog_len = i;
462	assert(i == len);
463	}
464
465	/* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */
466	#define BPF_SK_LOOKUP(func) \
467	/* struct bpf_sock_tuple tuple = {} */ \
468	BPF_MOV64_IMM(BPF_REG_2, 0), \
469	BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -8), \
470	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -16), \
471	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -24), \
472	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -32), \
473	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -40), \
474	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -48), \
475	/* sk = func(ctx, &tuple, sizeof tuple, 0, 0) */ \
476	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), \
477	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -48), \
478	BPF_MOV64_IMM(BPF_REG_3, sizeof(struct bpf_sock_tuple)), \
479	BPF_MOV64_IMM(BPF_REG_4, 0), \
480	BPF_MOV64_IMM(BPF_REG_5, 0), \
481	BPF_EMIT_CALL(BPF_FUNC_ ## func)
482
483	/* BPF_DIRECT_PKT_R2 contains 7 instructions, it initializes default return
484	* value into 0 and does necessary preparation for direct packet access
485	* through r2. The allowed access range is 8 bytes.
486	*/
487	#define BPF_DIRECT_PKT_R2 \
488	BPF_MOV64_IMM(BPF_REG_0, 0), \
489	BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, \
490	offsetof(struct __sk_buff, data)), \
491	BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, \
492	offsetof(struct __sk_buff, data_end)), \
493	BPF_MOV64_REG(BPF_REG_4, BPF_REG_2), \
494	BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 8), \
495	BPF_JMP_REG(BPF_JLE, BPF_REG_4, BPF_REG_3, 1), \
496	BPF_EXIT_INSN()
497
498	/* BPF_RAND_UEXT_R7 contains 4 instructions, it initializes R7 into a random
499	* positive u32, and zero-extend it into 64-bit.
500	*/
501	#define BPF_RAND_UEXT_R7 \
502	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, \
503	BPF_FUNC_get_prandom_u32), \
504	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0), \
505	BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 33), \
506	BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 33)
507
508	/* BPF_RAND_SEXT_R7 contains 5 instructions, it initializes R7 into a random
509	* negative u32, and sign-extend it into 64-bit.
510	*/
511	#define BPF_RAND_SEXT_R7 \
512	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, \
513	BPF_FUNC_get_prandom_u32), \
514	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0), \
515	BPF_ALU64_IMM(BPF_OR, BPF_REG_7, 0x80000000), \
516	BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 32), \
517	BPF_ALU64_IMM(BPF_ARSH, BPF_REG_7, 32)
518
519	static struct bpf_test tests[] = {
520	#define FILL_ARRAY
521	#include <verifier/tests.h>
522	#undef FILL_ARRAY
523	};
524
525	static int probe_filter_length(const struct bpf_insn *fp)
526	{
527	int len;
528
529	for (len = MAX_INSNS - 1; len > 0; --len)
530	if (fp[len].code != 0 || fp[len].imm != 0)
531	break;
532	return len + 1;
533	}
534
535	static bool skip_unsupported_map(enum bpf_map_type map_type)
536	{
537	if (!libbpf_probe_bpf_map_type(map_type, NULL)) {
538	printf("SKIP (unsupported map type %d)\n", map_type);
539	skips++;
540	return true;
541	}
542	return false;
543	}
544
545	static int __create_map(uint32_t type, uint32_t size_key,
546	uint32_t size_value, uint32_t max_elem,
547	uint32_t extra_flags)
548	{
549	LIBBPF_OPTS(bpf_map_create_opts, opts);
550	int fd;
551
552	opts.map_flags = (type == BPF_MAP_TYPE_HASH ? BPF_F_NO_PREALLOC : 0) | extra_flags;
553	fd = bpf_map_create(type, NULL, size_key, size_value, max_elem, &opts);
554	if (fd < 0) {
555	if (skip_unsupported_map(map_type: type))
556	return -1;
557	printf("Failed to create hash map '%s'!\n", strerror(errno));
558	}
559
560	return fd;
561	}
562
563	static int create_map(uint32_t type, uint32_t size_key,
564	uint32_t size_value, uint32_t max_elem)
565	{
566	return __create_map(type, size_key, size_value, max_elem, extra_flags: 0);
567	}
568
569	static void update_map(int fd, int index)
570	{
571	struct test_val value = {
572	.index = (6 + 1) * sizeof(int),
573	.foo[6] = 0xabcdef12,
574	};
575
576	assert(!bpf_map_update_elem(fd, &index, &value, 0));
577	}
578
579	static int create_prog_dummy_simple(enum bpf_prog_type prog_type, int ret)
580	{
581	struct bpf_insn prog[] = {
582	BPF_MOV64_IMM(BPF_REG_0, ret),
583	BPF_EXIT_INSN(),
584	};
585
586	return bpf_prog_load(prog_type, NULL, "GPL", prog, ARRAY_SIZE(prog), NULL);
587	}
588
589	static int create_prog_dummy_loop(enum bpf_prog_type prog_type, int mfd,
590	int idx, int ret)
591	{
592	struct bpf_insn prog[] = {
593	BPF_MOV64_IMM(BPF_REG_3, idx),
594	BPF_LD_MAP_FD(BPF_REG_2, mfd),
595	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
596	BPF_FUNC_tail_call),
597	BPF_MOV64_IMM(BPF_REG_0, ret),
598	BPF_EXIT_INSN(),
599	};
600
601	return bpf_prog_load(prog_type, NULL, "GPL", prog, ARRAY_SIZE(prog), NULL);
602	}
603
604	static int create_prog_array(enum bpf_prog_type prog_type, uint32_t max_elem,
605	int p1key, int p2key, int p3key)
606	{
607	int mfd, p1fd, p2fd, p3fd;
608
609	mfd = bpf_map_create(BPF_MAP_TYPE_PROG_ARRAY, NULL, sizeof(int),
610	sizeof(int), max_elem, NULL);
611	if (mfd < 0) {
612	if (skip_unsupported_map(map_type: BPF_MAP_TYPE_PROG_ARRAY))
613	return -1;
614	printf("Failed to create prog array '%s'!\n", strerror(errno));
615	return -1;
616	}
617
618	p1fd = create_prog_dummy_simple(prog_type, ret: 42);
619	p2fd = create_prog_dummy_loop(prog_type, mfd, idx: p2key, ret: 41);
620	p3fd = create_prog_dummy_simple(prog_type, ret: 24);
621	if (p1fd < 0 || p2fd < 0 || p3fd < 0)
622	goto err;
623	if (bpf_map_update_elem(mfd, &p1key, &p1fd, BPF_ANY) < 0)
624	goto err;
625	if (bpf_map_update_elem(mfd, &p2key, &p2fd, BPF_ANY) < 0)
626	goto err;
627	if (bpf_map_update_elem(mfd, &p3key, &p3fd, BPF_ANY) < 0) {
628	err:
629	close(mfd);
630	mfd = -1;
631	}
632	close(p3fd);
633	close(p2fd);
634	close(p1fd);
635	return mfd;
636	}
637
638	static int create_map_in_map(void)
639	{
640	LIBBPF_OPTS(bpf_map_create_opts, opts);
641	int inner_map_fd, outer_map_fd;
642
643	inner_map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(int),
644	sizeof(int), 1, NULL);
645	if (inner_map_fd < 0) {
646	if (skip_unsupported_map(map_type: BPF_MAP_TYPE_ARRAY))
647	return -1;
648	printf("Failed to create array '%s'!\n", strerror(errno));
649	return inner_map_fd;
650	}
651
652	opts.inner_map_fd = inner_map_fd;
653	outer_map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, NULL,
654	sizeof(int), sizeof(int), 1, &opts);
655	if (outer_map_fd < 0) {
656	if (skip_unsupported_map(map_type: BPF_MAP_TYPE_ARRAY_OF_MAPS))
657	return -1;
658	printf("Failed to create array of maps '%s'!\n",
659	strerror(errno));
660	}
661
662	close(inner_map_fd);
663
664	return outer_map_fd;
665	}
666
667	static int create_cgroup_storage(bool percpu)
668	{
669	enum bpf_map_type type = percpu ? BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE :
670	BPF_MAP_TYPE_CGROUP_STORAGE;
671	int fd;
672
673	fd = bpf_map_create(type, NULL, sizeof(struct bpf_cgroup_storage_key),
674	TEST_DATA_LEN, 0, NULL);
675	if (fd < 0) {
676	if (skip_unsupported_map(map_type: type))
677	return -1;
678	printf("Failed to create cgroup storage '%s'!\n",
679	strerror(errno));
680	}
681
682	return fd;
683	}
684
685	/* struct bpf_spin_lock {
686	* int val;
687	* };
688	* struct val {
689	* int cnt;
690	* struct bpf_spin_lock l;
691	* };
692	* struct bpf_timer {
693	* __u64 :64;
694	* __u64 :64;
695	* } __attribute__((aligned(8)));
696	* struct timer {
697	* struct bpf_timer t;
698	* };
699	* struct btf_ptr {
700	* struct prog_test_ref_kfunc __kptr_untrusted *ptr;
701	* struct prog_test_ref_kfunc __kptr *ptr;
702	* struct prog_test_member __kptr *ptr;
703	* }
704	*/
705	static const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l\0bpf_timer\0timer\0t"
706	"\0btf_ptr\0prog_test_ref_kfunc\0ptr\0kptr\0kptr_untrusted"
707	"\0prog_test_member";
708	static __u32 btf_raw_types[] = {
709	/* int */
710	BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
711	/* struct bpf_spin_lock */ /* [2] */
712	BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4),
713	BTF_MEMBER_ENC(15, 1, 0), /* int val; */
714	/* struct val */ /* [3] */
715	BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
716	BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */
717	BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */
718	/* struct bpf_timer */ /* [4] */
719	BTF_TYPE_ENC(25, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0), 16),
720	/* struct timer */ /* [5] */
721	BTF_TYPE_ENC(35, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 16),
722	BTF_MEMBER_ENC(41, 4, 0), /* struct bpf_timer t; */
723	/* struct prog_test_ref_kfunc */ /* [6] */
724	BTF_STRUCT_ENC(51, 0, 0),
725	BTF_STRUCT_ENC(95, 0, 0), /* [7] */
726	/* type tag "kptr_untrusted" */
727	BTF_TYPE_TAG_ENC(80, 6), /* [8] */
728	/* type tag "kptr" */
729	BTF_TYPE_TAG_ENC(75, 6), /* [9] */
730	BTF_TYPE_TAG_ENC(75, 7), /* [10] */
731	BTF_PTR_ENC(8), /* [11] */
732	BTF_PTR_ENC(9), /* [12] */
733	BTF_PTR_ENC(10), /* [13] */
734	/* struct btf_ptr */ /* [14] */
735	BTF_STRUCT_ENC(43, 3, 24),
736	BTF_MEMBER_ENC(71, 11, 0), /* struct prog_test_ref_kfunc __kptr_untrusted *ptr; */
737	BTF_MEMBER_ENC(71, 12, 64), /* struct prog_test_ref_kfunc __kptr *ptr; */
738	BTF_MEMBER_ENC(71, 13, 128), /* struct prog_test_member __kptr *ptr; */
739	};
740
741	static char bpf_vlog[UINT_MAX >> 8];
742
743	static int load_btf_spec(__u32 *types, int types_len,
744	const char *strings, int strings_len)
745	{
746	struct btf_header hdr = {
747	.magic = BTF_MAGIC,
748	.version = BTF_VERSION,
749	.hdr_len = sizeof(struct btf_header),
750	.type_len = types_len,
751	.str_off = types_len,
752	.str_len = strings_len,
753	};
754	void *ptr, *raw_btf;
755	int btf_fd;
756	LIBBPF_OPTS(bpf_btf_load_opts, opts,
757	.log_buf = bpf_vlog,
758	.log_size = sizeof(bpf_vlog),
759	.log_level = (verbose
760	? verif_log_level
761	: DEFAULT_LIBBPF_LOG_LEVEL),
762	);
763
764	raw_btf = malloc(sizeof(hdr) + types_len + strings_len);
765
766	ptr = raw_btf;
767	memcpy(ptr, &hdr, sizeof(hdr));
768	ptr += sizeof(hdr);
769	memcpy(ptr, types, hdr.type_len);
770	ptr += hdr.type_len;
771	memcpy(ptr, strings, hdr.str_len);
772	ptr += hdr.str_len;
773
774	btf_fd = bpf_btf_load(raw_btf, ptr - raw_btf, &opts);
775	if (btf_fd < 0)
776	printf("Failed to load BTF spec: '%s'\n", strerror(errno));
777
778	free(raw_btf);
779
780	return btf_fd < 0 ? -1 : btf_fd;
781	}
782
783	static int load_btf(void)
784	{
785	return load_btf_spec(types: btf_raw_types, types_len: sizeof(btf_raw_types),
786	strings: btf_str_sec, strings_len: sizeof(btf_str_sec));
787	}
788
789	static int load_btf_for_test(struct bpf_test *test)
790	{
791	int types_num = 0;
792
793	while (types_num < MAX_BTF_TYPES &&
794	test->btf_types[types_num] != BTF_END_RAW)
795	++types_num;
796
797	int types_len = types_num * sizeof(test->btf_types[0]);
798
799	return load_btf_spec(types: test->btf_types, types_len,
800	strings: test->btf_strings, strings_len: sizeof(test->btf_strings));
801	}
802
803	static int create_map_spin_lock(void)
804	{
805	LIBBPF_OPTS(bpf_map_create_opts, opts,
806	.btf_key_type_id = 1,
807	.btf_value_type_id = 3,
808	);
809	int fd, btf_fd;
810
811	btf_fd = load_btf();
812	if (btf_fd < 0)
813	return -1;
814	opts.btf_fd = btf_fd;
815	fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "test_map", 4, 8, 1, &opts);
816	if (fd < 0)
817	printf("Failed to create map with spin_lock\n");
818	return fd;
819	}
820
821	static int create_sk_storage_map(void)
822	{
823	LIBBPF_OPTS(bpf_map_create_opts, opts,
824	.map_flags = BPF_F_NO_PREALLOC,
825	.btf_key_type_id = 1,
826	.btf_value_type_id = 3,
827	);
828	int fd, btf_fd;
829
830	btf_fd = load_btf();
831	if (btf_fd < 0)
832	return -1;
833	opts.btf_fd = btf_fd;
834	fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "test_map", 4, 8, 0, &opts);
835	close(opts.btf_fd);
836	if (fd < 0)
837	printf("Failed to create sk_storage_map\n");
838	return fd;
839	}
840
841	static int create_map_timer(void)
842	{
843	LIBBPF_OPTS(bpf_map_create_opts, opts,
844	.btf_key_type_id = 1,
845	.btf_value_type_id = 5,
846	);
847	int fd, btf_fd;
848
849	btf_fd = load_btf();
850	if (btf_fd < 0)
851	return -1;
852
853	opts.btf_fd = btf_fd;
854	fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "test_map", 4, 16, 1, &opts);
855	if (fd < 0)
856	printf("Failed to create map with timer\n");
857	return fd;
858	}
859
860	static int create_map_kptr(void)
861	{
862	LIBBPF_OPTS(bpf_map_create_opts, opts,
863	.btf_key_type_id = 1,
864	.btf_value_type_id = 14,
865	);
866	int fd, btf_fd;
867
868	btf_fd = load_btf();
869	if (btf_fd < 0)
870	return -1;
871
872	opts.btf_fd = btf_fd;
873	fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "test_map", 4, 24, 1, &opts);
874	if (fd < 0)
875	printf("Failed to create map with btf_id pointer\n");
876	return fd;
877	}
878
879	static void set_root(bool set)
880	{
881	__u64 caps;
882
883	if (set) {
884	if (cap_enable_effective(caps: 1ULL << CAP_SYS_ADMIN, old_caps: &caps))
885	perror("cap_disable_effective(CAP_SYS_ADMIN)");
886	} else {
887	if (cap_disable_effective(caps: 1ULL << CAP_SYS_ADMIN, old_caps: &caps))
888	perror("cap_disable_effective(CAP_SYS_ADMIN)");
889	}
890	}
891
892	static __u64 ptr_to_u64(const void *ptr)
893	{
894	return (uintptr_t) ptr;
895	}
896
897	static struct btf *btf__load_testmod_btf(struct btf *vmlinux)
898	{
899	struct bpf_btf_info info;
900	__u32 len = sizeof(info);
901	struct btf *btf = NULL;
902	char name[64];
903	__u32 id = 0;
904	int err, fd;
905
906	/* Iterate all loaded BTF objects and find bpf_testmod,
907	* we need SYS_ADMIN cap for that.
908	*/
909	set_root(true);
910
911	while (true) {
912	err = bpf_btf_get_next_id(id, &id);
913	if (err) {
914	if (errno == ENOENT)
915	break;
916	perror("bpf_btf_get_next_id failed");
917	break;
918	}
919
920	fd = bpf_btf_get_fd_by_id(id);
921	if (fd < 0) {
922	if (errno == ENOENT)
923	continue;
924	perror("bpf_btf_get_fd_by_id failed");
925	break;
926	}
927
928	memset(&info, 0, sizeof(info));
929	info.name_len = sizeof(name);
930	info.name = ptr_to_u64(ptr: name);
931	len = sizeof(info);
932
933	err = bpf_obj_get_info_by_fd(fd, &info, &len);
934	if (err) {
935	close(fd);
936	perror("bpf_obj_get_info_by_fd failed");
937	break;
938	}
939
940	if (strcmp("bpf_testmod", name)) {
941	close(fd);
942	continue;
943	}
944
945	btf = btf__load_from_kernel_by_id_split(id, vmlinux);
946	if (!btf) {
947	close(fd);
948	break;
949	}
950
951	/* We need the fd to stay open so it can be used in fd_array.
952	* The final cleanup call to btf__free will free btf object
953	* and close the file descriptor.
954	*/
955	btf__set_fd(btf, fd);
956	break;
957	}
958
959	set_root(false);
960	return btf;
961	}
962
963	static struct btf *testmod_btf;
964	static struct btf *vmlinux_btf;
965
966	static void kfuncs_cleanup(void)
967	{
968	btf__free(testmod_btf);
969	btf__free(vmlinux_btf);
970	}
971
972	static void fixup_prog_kfuncs(struct bpf_insn *prog, int *fd_array,
973	struct kfunc_btf_id_pair *fixup_kfunc_btf_id)
974	{
975	/* Patch in kfunc BTF IDs */
976	while (fixup_kfunc_btf_id->kfunc) {
977	int btf_id = 0;
978
979	/* try to find kfunc in kernel BTF */
980	vmlinux_btf = vmlinux_btf ?: btf__load_vmlinux_btf();
981	if (vmlinux_btf) {
982	btf_id = btf__find_by_name_kind(vmlinux_btf,
983	fixup_kfunc_btf_id->kfunc,
984	BTF_KIND_FUNC);
985	btf_id = btf_id < 0 ? 0 : btf_id;
986	}
987
988	/* kfunc not found in kernel BTF, try bpf_testmod BTF */
989	if (!btf_id) {
990	testmod_btf = testmod_btf ?: btf__load_testmod_btf(vmlinux: vmlinux_btf);
991	if (testmod_btf) {
992	btf_id = btf__find_by_name_kind(testmod_btf,
993	fixup_kfunc_btf_id->kfunc,
994	BTF_KIND_FUNC);
995	btf_id = btf_id < 0 ? 0 : btf_id;
996	if (btf_id) {
997	/* We put bpf_testmod module fd into fd_array
998	* and its index 1 into instruction 'off'.
999	*/
1000	*fd_array = btf__fd(testmod_btf);
1001	prog[fixup_kfunc_btf_id->insn_idx].off = 1;
1002	}
1003	}
1004	}
1005
1006	prog[fixup_kfunc_btf_id->insn_idx].imm = btf_id;
1007	fixup_kfunc_btf_id++;
1008	}
1009	}
1010
1011	static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type,
1012	struct bpf_insn *prog, int *map_fds, int *fd_array)
1013	{
1014	int *fixup_map_hash_8b = test->fixup_map_hash_8b;
1015	int *fixup_map_hash_48b = test->fixup_map_hash_48b;
1016	int *fixup_map_hash_16b = test->fixup_map_hash_16b;
1017	int *fixup_map_array_48b = test->fixup_map_array_48b;
1018	int *fixup_map_sockmap = test->fixup_map_sockmap;
1019	int *fixup_map_sockhash = test->fixup_map_sockhash;
1020	int *fixup_map_xskmap = test->fixup_map_xskmap;
1021	int *fixup_map_stacktrace = test->fixup_map_stacktrace;
1022	int *fixup_prog1 = test->fixup_prog1;
1023	int *fixup_prog2 = test->fixup_prog2;
1024	int *fixup_map_in_map = test->fixup_map_in_map;
1025	int *fixup_cgroup_storage = test->fixup_cgroup_storage;
1026	int *fixup_percpu_cgroup_storage = test->fixup_percpu_cgroup_storage;
1027	int *fixup_map_spin_lock = test->fixup_map_spin_lock;
1028	int *fixup_map_array_ro = test->fixup_map_array_ro;
1029	int *fixup_map_array_wo = test->fixup_map_array_wo;
1030	int *fixup_map_array_small = test->fixup_map_array_small;
1031	int *fixup_sk_storage_map = test->fixup_sk_storage_map;
1032	int *fixup_map_event_output = test->fixup_map_event_output;
1033	int *fixup_map_reuseport_array = test->fixup_map_reuseport_array;
1034	int *fixup_map_ringbuf = test->fixup_map_ringbuf;
1035	int *fixup_map_timer = test->fixup_map_timer;
1036	int *fixup_map_kptr = test->fixup_map_kptr;
1037
1038	if (test->fill_helper) {
1039	test->fill_insns = calloc(MAX_TEST_INSNS, sizeof(struct bpf_insn));
1040	test->fill_helper(test);
1041	}
1042
1043	/* Allocating HTs with 1 elem is fine here, since we only test
1044	* for verifier and not do a runtime lookup, so the only thing
1045	* that really matters is value size in this case.
1046	*/
1047	if (*fixup_map_hash_8b) {
1048	map_fds[0] = create_map(type: BPF_MAP_TYPE_HASH, size_key: sizeof(long long),
1049	size_value: sizeof(long long), max_elem: 1);
1050	do {
1051	prog[*fixup_map_hash_8b].imm = map_fds[0];
1052	fixup_map_hash_8b++;
1053	} while (*fixup_map_hash_8b);
1054	}
1055
1056	if (*fixup_map_hash_48b) {
1057	map_fds[1] = create_map(type: BPF_MAP_TYPE_HASH, size_key: sizeof(long long),
1058	size_value: sizeof(struct test_val), max_elem: 1);
1059	do {
1060	prog[*fixup_map_hash_48b].imm = map_fds[1];
1061	fixup_map_hash_48b++;
1062	} while (*fixup_map_hash_48b);
1063	}
1064
1065	if (*fixup_map_hash_16b) {
1066	map_fds[2] = create_map(type: BPF_MAP_TYPE_HASH, size_key: sizeof(long long),
1067	size_value: sizeof(struct other_val), max_elem: 1);
1068	do {
1069	prog[*fixup_map_hash_16b].imm = map_fds[2];
1070	fixup_map_hash_16b++;
1071	} while (*fixup_map_hash_16b);
1072	}
1073
1074	if (*fixup_map_array_48b) {
1075	map_fds[3] = create_map(type: BPF_MAP_TYPE_ARRAY, size_key: sizeof(int),
1076	size_value: sizeof(struct test_val), max_elem: 1);
1077	update_map(fd: map_fds[3], index: 0);
1078	do {
1079	prog[*fixup_map_array_48b].imm = map_fds[3];
1080	fixup_map_array_48b++;
1081	} while (*fixup_map_array_48b);
1082	}
1083
1084	if (*fixup_prog1) {
1085	map_fds[4] = create_prog_array(prog_type, max_elem: 4, p1key: 0, p2key: 1, p3key: 2);
1086	do {
1087	prog[*fixup_prog1].imm = map_fds[4];
1088	fixup_prog1++;
1089	} while (*fixup_prog1);
1090	}
1091
1092	if (*fixup_prog2) {
1093	map_fds[5] = create_prog_array(prog_type, max_elem: 8, p1key: 7, p2key: 1, p3key: 2);
1094	do {
1095	prog[*fixup_prog2].imm = map_fds[5];
1096	fixup_prog2++;
1097	} while (*fixup_prog2);
1098	}
1099
1100	if (*fixup_map_in_map) {
1101	map_fds[6] = create_map_in_map();
1102	do {
1103	prog[*fixup_map_in_map].imm = map_fds[6];
1104	fixup_map_in_map++;
1105	} while (*fixup_map_in_map);
1106	}
1107
1108	if (*fixup_cgroup_storage) {
1109	map_fds[7] = create_cgroup_storage(percpu: false);
1110	do {
1111	prog[*fixup_cgroup_storage].imm = map_fds[7];
1112	fixup_cgroup_storage++;
1113	} while (*fixup_cgroup_storage);
1114	}
1115
1116	if (*fixup_percpu_cgroup_storage) {
1117	map_fds[8] = create_cgroup_storage(percpu: true);
1118	do {
1119	prog[*fixup_percpu_cgroup_storage].imm = map_fds[8];
1120	fixup_percpu_cgroup_storage++;
1121	} while (*fixup_percpu_cgroup_storage);
1122	}
1123	if (*fixup_map_sockmap) {
1124	map_fds[9] = create_map(type: BPF_MAP_TYPE_SOCKMAP, size_key: sizeof(int),
1125	size_value: sizeof(int), max_elem: 1);
1126	do {
1127	prog[*fixup_map_sockmap].imm = map_fds[9];
1128	fixup_map_sockmap++;
1129	} while (*fixup_map_sockmap);
1130	}
1131	if (*fixup_map_sockhash) {
1132	map_fds[10] = create_map(type: BPF_MAP_TYPE_SOCKHASH, size_key: sizeof(int),
1133	size_value: sizeof(int), max_elem: 1);
1134	do {
1135	prog[*fixup_map_sockhash].imm = map_fds[10];
1136	fixup_map_sockhash++;
1137	} while (*fixup_map_sockhash);
1138	}
1139	if (*fixup_map_xskmap) {
1140	map_fds[11] = create_map(type: BPF_MAP_TYPE_XSKMAP, size_key: sizeof(int),
1141	size_value: sizeof(int), max_elem: 1);
1142	do {
1143	prog[*fixup_map_xskmap].imm = map_fds[11];
1144	fixup_map_xskmap++;
1145	} while (*fixup_map_xskmap);
1146	}
1147	if (*fixup_map_stacktrace) {
1148	map_fds[12] = create_map(type: BPF_MAP_TYPE_STACK_TRACE, size_key: sizeof(u32),
1149	size_value: sizeof(u64), max_elem: 1);
1150	do {
1151	prog[*fixup_map_stacktrace].imm = map_fds[12];
1152	fixup_map_stacktrace++;
1153	} while (*fixup_map_stacktrace);
1154	}
1155	if (*fixup_map_spin_lock) {
1156	map_fds[13] = create_map_spin_lock();
1157	do {
1158	prog[*fixup_map_spin_lock].imm = map_fds[13];
1159	fixup_map_spin_lock++;
1160	} while (*fixup_map_spin_lock);
1161	}
1162	if (*fixup_map_array_ro) {
1163	map_fds[14] = __create_map(type: BPF_MAP_TYPE_ARRAY, size_key: sizeof(int),
1164	size_value: sizeof(struct test_val), max_elem: 1,
1165	extra_flags: BPF_F_RDONLY_PROG);
1166	update_map(fd: map_fds[14], index: 0);
1167	do {
1168	prog[*fixup_map_array_ro].imm = map_fds[14];
1169	fixup_map_array_ro++;
1170	} while (*fixup_map_array_ro);
1171	}
1172	if (*fixup_map_array_wo) {
1173	map_fds[15] = __create_map(type: BPF_MAP_TYPE_ARRAY, size_key: sizeof(int),
1174	size_value: sizeof(struct test_val), max_elem: 1,
1175	extra_flags: BPF_F_WRONLY_PROG);
1176	update_map(fd: map_fds[15], index: 0);
1177	do {
1178	prog[*fixup_map_array_wo].imm = map_fds[15];
1179	fixup_map_array_wo++;
1180	} while (*fixup_map_array_wo);
1181	}
1182	if (*fixup_map_array_small) {
1183	map_fds[16] = __create_map(type: BPF_MAP_TYPE_ARRAY, size_key: sizeof(int),
1184	size_value: 1, max_elem: 1, extra_flags: 0);
1185	update_map(fd: map_fds[16], index: 0);
1186	do {
1187	prog[*fixup_map_array_small].imm = map_fds[16];
1188	fixup_map_array_small++;
1189	} while (*fixup_map_array_small);
1190	}
1191	if (*fixup_sk_storage_map) {
1192	map_fds[17] = create_sk_storage_map();
1193	do {
1194	prog[*fixup_sk_storage_map].imm = map_fds[17];
1195	fixup_sk_storage_map++;
1196	} while (*fixup_sk_storage_map);
1197	}
1198	if (*fixup_map_event_output) {
1199	map_fds[18] = __create_map(type: BPF_MAP_TYPE_PERF_EVENT_ARRAY,
1200	size_key: sizeof(int), size_value: sizeof(int), max_elem: 1, extra_flags: 0);
1201	do {
1202	prog[*fixup_map_event_output].imm = map_fds[18];
1203	fixup_map_event_output++;
1204	} while (*fixup_map_event_output);
1205	}
1206	if (*fixup_map_reuseport_array) {
1207	map_fds[19] = __create_map(type: BPF_MAP_TYPE_REUSEPORT_SOCKARRAY,
1208	size_key: sizeof(u32), size_value: sizeof(u64), max_elem: 1, extra_flags: 0);
1209	do {
1210	prog[*fixup_map_reuseport_array].imm = map_fds[19];
1211	fixup_map_reuseport_array++;
1212	} while (*fixup_map_reuseport_array);
1213	}
1214	if (*fixup_map_ringbuf) {
1215	map_fds[20] = create_map(type: BPF_MAP_TYPE_RINGBUF, size_key: 0,
1216	size_value: 0, max_elem: getpagesize());
1217	do {
1218	prog[*fixup_map_ringbuf].imm = map_fds[20];
1219	fixup_map_ringbuf++;
1220	} while (*fixup_map_ringbuf);
1221	}
1222	if (*fixup_map_timer) {
1223	map_fds[21] = create_map_timer();
1224	do {
1225	prog[*fixup_map_timer].imm = map_fds[21];
1226	fixup_map_timer++;
1227	} while (*fixup_map_timer);
1228	}
1229	if (*fixup_map_kptr) {
1230	map_fds[22] = create_map_kptr();
1231	do {
1232	prog[*fixup_map_kptr].imm = map_fds[22];
1233	fixup_map_kptr++;
1234	} while (*fixup_map_kptr);
1235	}
1236
1237	fixup_prog_kfuncs(prog, fd_array, fixup_kfunc_btf_id: test->fixup_kfunc_btf_id);
1238	}
1239
1240	struct libcap {
1241	struct __user_cap_header_struct hdr;
1242	struct __user_cap_data_struct data[2];
1243	};
1244
1245	static int set_admin(bool admin)
1246	{
1247	int err;
1248
1249	if (admin) {
1250	err = cap_enable_effective(ADMIN_CAPS, NULL);
1251	if (err)
1252	perror("cap_enable_effective(ADMIN_CAPS)");
1253	} else {
1254	err = cap_disable_effective(ADMIN_CAPS, NULL);
1255	if (err)
1256	perror("cap_disable_effective(ADMIN_CAPS)");
1257	}
1258
1259	return err;
1260	}
1261
1262	static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val,
1263	void *data, size_t size_data)
1264	{
1265	__u8 tmp[TEST_DATA_LEN << 2];
1266	__u32 size_tmp = sizeof(tmp);
1267	int err, saved_errno;
1268	LIBBPF_OPTS(bpf_test_run_opts, topts,
1269	.data_in = data,
1270	.data_size_in = size_data,
1271	.data_out = tmp,
1272	.data_size_out = size_tmp,
1273	.repeat = 1,
1274	);
1275
1276	if (unpriv)
1277	set_admin(true);
1278	err = bpf_prog_test_run_opts(fd_prog, &topts);
1279	saved_errno = errno;
1280
1281	if (unpriv)
1282	set_admin(false);
1283
1284	if (err) {
1285	switch (saved_errno) {
1286	case ENOTSUPP:
1287	printf("Did not run the program (not supported) ");
1288	return 0;
1289	case EPERM:
1290	if (unpriv) {
1291	printf("Did not run the program (no permission) ");
1292	return 0;
1293	}
1294	/* fallthrough; */
1295	default:
1296	printf("FAIL: Unexpected bpf_prog_test_run error (%s) ",
1297	strerror(saved_errno));
1298	return err;
1299	}
1300	}
1301
1302	if (topts.retval != expected_val && expected_val != POINTER_VALUE) {
1303	printf("FAIL retval %d != %d ", topts.retval, expected_val);
1304	return 1;
1305	}
1306
1307	return 0;
1308	}
1309
1310	/* Returns true if every part of exp (tab-separated) appears in log, in order.
1311	*
1312	* If exp is an empty string, returns true.
1313	*/
1314	static bool cmp_str_seq(const char *log, const char *exp)
1315	{
1316	char needle[200];
1317	const char *p, *q;
1318	int len;
1319
1320	do {
1321	if (!strlen(exp))
1322	break;
1323	p = strchr(exp, '\t');
1324	if (!p)
1325	p = exp + strlen(exp);
1326
1327	len = p - exp;
1328	if (len >= sizeof(needle) || !len) {
1329	printf("FAIL\nTestcase bug\n");
1330	return false;
1331	}
1332	strncpy(p: needle, q: exp, size: len);
1333	needle[len] = 0;
1334	q = strstr(log, needle);
1335	if (!q) {
1336	printf("FAIL\nUnexpected verifier log!\n"
1337	"EXP: %s\nRES:\n", needle);
1338	return false;
1339	}
1340	log = q + len;
1341	exp = p + 1;
1342	} while (*p);
1343	return true;
1344	}
1345
1346	static bool is_null_insn(struct bpf_insn *insn)
1347	{
1348	struct bpf_insn null_insn = {};
1349
1350	return memcmp(p: insn, q: &null_insn, size: sizeof(null_insn)) == 0;
1351	}
1352
1353	static bool is_skip_insn(struct bpf_insn *insn)
1354	{
1355	struct bpf_insn skip_insn = SKIP_INSNS();
1356
1357	return memcmp(p: insn, q: &skip_insn, size: sizeof(skip_insn)) == 0;
1358	}
1359
1360	static int null_terminated_insn_len(struct bpf_insn *seq, int max_len)
1361	{
1362	int i;
1363
1364	for (i = 0; i < max_len; ++i) {
1365	if (is_null_insn(insn: &seq[i]))
1366	return i;
1367	}
1368	return max_len;
1369	}
1370
1371	static bool compare_masked_insn(struct bpf_insn *orig, struct bpf_insn *masked)
1372	{
1373	struct bpf_insn orig_masked;
1374
1375	memcpy(&orig_masked, orig, sizeof(orig_masked));
1376	if (masked->imm == INSN_IMM_MASK)
1377	orig_masked.imm = INSN_IMM_MASK;
1378	if (masked->off == INSN_OFF_MASK)
1379	orig_masked.off = INSN_OFF_MASK;
1380
1381	return memcmp(p: &orig_masked, q: masked, size: sizeof(orig_masked)) == 0;
1382	}
1383
1384	static int find_insn_subseq(struct bpf_insn *seq, struct bpf_insn *subseq,
1385	int seq_len, int subseq_len)
1386	{
1387	int i, j;
1388
1389	if (subseq_len > seq_len)
1390	return -1;
1391
1392	for (i = 0; i < seq_len - subseq_len + 1; ++i) {
1393	bool found = true;
1394
1395	for (j = 0; j < subseq_len; ++j) {
1396	if (!compare_masked_insn(orig: &seq[i + j], masked: &subseq[j])) {
1397	found = false;
1398	break;
1399	}
1400	}
1401	if (found)
1402	return i;
1403	}
1404
1405	return -1;
1406	}
1407
1408	static int find_skip_insn_marker(struct bpf_insn *seq, int len)
1409	{
1410	int i;
1411
1412	for (i = 0; i < len; ++i)
1413	if (is_skip_insn(insn: &seq[i]))
1414	return i;
1415
1416	return -1;
1417	}
1418
1419	/* Return true if all sub-sequences in `subseqs` could be found in
1420	* `seq` one after another. Sub-sequences are separated by a single
1421	* nil instruction.
1422	*/
1423	static bool find_all_insn_subseqs(struct bpf_insn *seq, struct bpf_insn *subseqs,
1424	int seq_len, int max_subseqs_len)
1425	{
1426	int subseqs_len = null_terminated_insn_len(seq: subseqs, max_len: max_subseqs_len);
1427
1428	while (subseqs_len > 0) {
1429	int skip_idx = find_skip_insn_marker(seq: subseqs, len: subseqs_len);
1430	int cur_subseq_len = skip_idx < 0 ? subseqs_len : skip_idx;
1431	int subseq_idx = find_insn_subseq(seq, subseq: subseqs,
1432	seq_len, subseq_len: cur_subseq_len);
1433
1434	if (subseq_idx < 0)
1435	return false;
1436	seq += subseq_idx + cur_subseq_len;
1437	seq_len -= subseq_idx + cur_subseq_len;
1438	subseqs += cur_subseq_len + 1;
1439	subseqs_len -= cur_subseq_len + 1;
1440	}
1441
1442	return true;
1443	}
1444
1445	static void print_insn(struct bpf_insn *buf, int cnt)
1446	{
1447	int i;
1448
1449	printf(" addr op d s off imm\n");
1450	for (i = 0; i < cnt; ++i) {
1451	struct bpf_insn *insn = &buf[i];
1452
1453	if (is_null_insn(insn))
1454	break;
1455
1456	if (is_skip_insn(insn))
1457	printf(" ...\n");
1458	else
1459	printf(" %04x: %02x %1x %x %04hx %08x\n",
1460	i, insn->code, insn->dst_reg,
1461	insn->src_reg, insn->off, insn->imm);
1462	}
1463	}
1464
1465	static bool check_xlated_program(struct bpf_test *test, int fd_prog)
1466	{
1467	struct bpf_insn *buf;
1468	unsigned int cnt;
1469	bool result = true;
1470	bool check_expected = !is_null_insn(insn: test->expected_insns);
1471	bool check_unexpected = !is_null_insn(insn: test->unexpected_insns);
1472
1473	if (!check_expected && !check_unexpected)
1474	goto out;
1475
1476	if (get_xlated_program(fd_prog, buf: &buf, cnt: &cnt)) {
1477	printf("FAIL: can't get xlated program\n");
1478	result = false;
1479	goto out;
1480	}
1481
1482	if (check_expected &&
1483	!find_all_insn_subseqs(seq: buf, subseqs: test->expected_insns,
1484	seq_len: cnt, MAX_EXPECTED_INSNS)) {
1485	printf("FAIL: can't find expected subsequence of instructions\n");
1486	result = false;
1487	if (verbose) {
1488	printf("Program:\n");
1489	print_insn(buf, cnt);
1490	printf("Expected subsequence:\n");
1491	print_insn(buf: test->expected_insns, MAX_EXPECTED_INSNS);
1492	}
1493	}
1494
1495	if (check_unexpected &&
1496	find_all_insn_subseqs(seq: buf, subseqs: test->unexpected_insns,
1497	seq_len: cnt, MAX_UNEXPECTED_INSNS)) {
1498	printf("FAIL: found unexpected subsequence of instructions\n");
1499	result = false;
1500	if (verbose) {
1501	printf("Program:\n");
1502	print_insn(buf, cnt);
1503	printf("Un-expected subsequence:\n");
1504	print_insn(buf: test->unexpected_insns, MAX_UNEXPECTED_INSNS);
1505	}
1506	}
1507
1508	free(buf);
1509	out:
1510	return result;
1511	}
1512
1513	static void do_test_single(struct bpf_test *test, bool unpriv,
1514	int *passes, int *errors)
1515	{
1516	int fd_prog, btf_fd, expected_ret, alignment_prevented_execution;
1517	int prog_len, prog_type = test->prog_type;
1518	struct bpf_insn *prog = test->insns;
1519	LIBBPF_OPTS(bpf_prog_load_opts, opts);
1520	int run_errs, run_successes;
1521	int map_fds[MAX_NR_MAPS];
1522	const char *expected_err;
1523	int fd_array[2] = { -1, -1 };
1524	int saved_errno;
1525	int fixup_skips;
1526	__u32 pflags;
1527	int i, err;
1528
1529	if ((test->flags & F_NEEDS_JIT_ENABLED) && jit_disabled) {
1530	printf("SKIP (requires BPF JIT)\n");
1531	skips++;
1532	sched_yield();
1533	return;
1534	}
1535
1536	fd_prog = -1;
1537	for (i = 0; i < MAX_NR_MAPS; i++)
1538	map_fds[i] = -1;
1539	btf_fd = -1;
1540
1541	if (!prog_type)
1542	prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
1543	fixup_skips = skips;
1544	do_test_fixup(test, prog_type, prog, map_fds, fd_array: &fd_array[1]);
1545	if (test->fill_insns) {
1546	prog = test->fill_insns;
1547	prog_len = test->prog_len;
1548	} else {
1549	prog_len = probe_filter_length(fp: prog);
1550	}
1551	/* If there were some map skips during fixup due to missing bpf
1552	* features, skip this test.
1553	*/
1554	if (fixup_skips != skips)
1555	return;
1556
1557	pflags = testing_prog_flags();
1558	if (test->flags & F_LOAD_WITH_STRICT_ALIGNMENT)
1559	pflags |= BPF_F_STRICT_ALIGNMENT;
1560	if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)
1561	pflags |= BPF_F_ANY_ALIGNMENT;
1562	if (test->flags & ~3)
1563	pflags |= test->flags;
1564
1565	expected_ret = unpriv && test->result_unpriv != UNDEF ?
1566	test->result_unpriv : test->result;
1567	expected_err = unpriv && test->errstr_unpriv ?
1568	test->errstr_unpriv : test->errstr;
1569
1570	opts.expected_attach_type = test->expected_attach_type;
1571	if (verbose)
1572	opts.log_level = verif_log_level | 4; /* force stats */
1573	else if (expected_ret == VERBOSE_ACCEPT)
1574	opts.log_level = 2;
1575	else
1576	opts.log_level = DEFAULT_LIBBPF_LOG_LEVEL;
1577	opts.prog_flags = pflags;
1578	if (fd_array[1] != -1)
1579	opts.fd_array = &fd_array[0];
1580
1581	if ((prog_type == BPF_PROG_TYPE_TRACING ||
1582	prog_type == BPF_PROG_TYPE_LSM) && test->kfunc) {
1583	int attach_btf_id;
1584
1585	attach_btf_id = libbpf_find_vmlinux_btf_id(test->kfunc,
1586	opts.expected_attach_type);
1587	if (attach_btf_id < 0) {
1588	printf("FAIL\nFailed to find BTF ID for '%s'!\n",
1589	test->kfunc);
1590	(*errors)++;
1591	return;
1592	}
1593
1594	opts.attach_btf_id = attach_btf_id;
1595	}
1596
1597	if (test->btf_types[0] != 0) {
1598	btf_fd = load_btf_for_test(test);
1599	if (btf_fd < 0)
1600	goto fail_log;
1601	opts.prog_btf_fd = btf_fd;
1602	}
1603
1604	if (test->func_info_cnt != 0) {
1605	opts.func_info = test->func_info;
1606	opts.func_info_cnt = test->func_info_cnt;
1607	opts.func_info_rec_size = sizeof(test->func_info[0]);
1608	}
1609
1610	opts.log_buf = bpf_vlog;
1611	opts.log_size = sizeof(bpf_vlog);
1612	fd_prog = bpf_prog_load(prog_type, NULL, "GPL", prog, prog_len, &opts);
1613	saved_errno = errno;
1614
1615	/* BPF_PROG_TYPE_TRACING requires more setup and
1616	* bpf_probe_prog_type won't give correct answer
1617	*/
1618	if (fd_prog < 0 && prog_type != BPF_PROG_TYPE_TRACING &&
1619	!libbpf_probe_bpf_prog_type(prog_type, NULL)) {
1620	printf("SKIP (unsupported program type %d)\n", prog_type);
1621	skips++;
1622	goto close_fds;
1623	}
1624
1625	if (fd_prog < 0 && saved_errno == ENOTSUPP) {
1626	printf("SKIP (program uses an unsupported feature)\n");
1627	skips++;
1628	goto close_fds;
1629	}
1630
1631	alignment_prevented_execution = 0;
1632
1633	if (expected_ret == ACCEPT || expected_ret == VERBOSE_ACCEPT) {
1634	if (fd_prog < 0) {
1635	printf("FAIL\nFailed to load prog '%s'!\n",
1636	strerror(saved_errno));
1637	goto fail_log;
1638	}
1639	#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
1640	if (fd_prog >= 0 &&
1641	(test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS))
1642	alignment_prevented_execution = 1;
1643	#endif
1644	if (expected_ret == VERBOSE_ACCEPT && !cmp_str_seq(log: bpf_vlog, exp: expected_err)) {
1645	goto fail_log;
1646	}
1647	} else {
1648	if (fd_prog >= 0) {
1649	printf("FAIL\nUnexpected success to load!\n");
1650	goto fail_log;
1651	}
1652	if (!expected_err || !cmp_str_seq(log: bpf_vlog, exp: expected_err)) {
1653	printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n",
1654	expected_err, bpf_vlog);
1655	goto fail_log;
1656	}
1657	}
1658
1659	if (!unpriv && test->insn_processed) {
1660	uint32_t insn_processed;
1661	char *proc;
1662
1663	proc = strstr(bpf_vlog, "processed ");
1664	insn_processed = atoi(proc + 10);
1665	if (test->insn_processed != insn_processed) {
1666	printf("FAIL\nUnexpected insn_processed %u vs %u\n",
1667	insn_processed, test->insn_processed);
1668	goto fail_log;
1669	}
1670	}
1671
1672	if (verbose)
1673	printf(", verifier log:\n%s", bpf_vlog);
1674
1675	if (!check_xlated_program(test, fd_prog))
1676	goto fail_log;
1677
1678	run_errs = 0;
1679	run_successes = 0;
1680	if (!alignment_prevented_execution && fd_prog >= 0 && test->runs >= 0) {
1681	uint32_t expected_val;
1682	int i;
1683
1684	if (!test->runs)
1685	test->runs = 1;
1686
1687	for (i = 0; i < test->runs; i++) {
1688	if (unpriv && test->retvals[i].retval_unpriv)
1689	expected_val = test->retvals[i].retval_unpriv;
1690	else
1691	expected_val = test->retvals[i].retval;
1692
1693	err = do_prog_test_run(fd_prog, unpriv, expected_val,
1694	data: test->retvals[i].data,
1695	size_data: sizeof(test->retvals[i].data));
1696	if (err) {
1697	printf("(run %d/%d) ", i + 1, test->runs);
1698	run_errs++;
1699	} else {
1700	run_successes++;
1701	}
1702	}
1703	}
1704
1705	if (!run_errs) {
1706	(*passes)++;
1707	if (run_successes > 1)
1708	printf("%d cases ", run_successes);
1709	printf("OK");
1710	if (alignment_prevented_execution)
1711	printf(" (NOTE: not executed due to unknown alignment)");
1712	printf("\n");
1713	} else {
1714	printf("\n");
1715	goto fail_log;
1716	}
1717	close_fds:
1718	if (test->fill_insns)
1719	free(test->fill_insns);
1720	close(fd_prog);
1721	close(btf_fd);
1722	for (i = 0; i < MAX_NR_MAPS; i++)
1723	close(map_fds[i]);
1724	sched_yield();
1725	return;
1726	fail_log:
1727	(*errors)++;
1728	printf("%s", bpf_vlog);
1729	goto close_fds;
1730	}
1731
1732	static bool is_admin(void)
1733	{
1734	__u64 caps;
1735
1736	/* The test checks for finer cap as CAP_NET_ADMIN,
1737	* CAP_PERFMON, and CAP_BPF instead of CAP_SYS_ADMIN.
1738	* Thus, disable CAP_SYS_ADMIN at the beginning.
1739	*/
1740	if (cap_disable_effective(caps: 1ULL << CAP_SYS_ADMIN, old_caps: &caps)) {
1741	perror("cap_disable_effective(CAP_SYS_ADMIN)");
1742	return false;
1743	}
1744
1745	return (caps & ADMIN_CAPS) == ADMIN_CAPS;
1746	}
1747
1748	static bool test_as_unpriv(struct bpf_test *test)
1749	{
1750	#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
1751	/* Some architectures have strict alignment requirements. In
1752	* that case, the BPF verifier detects if a program has
1753	* unaligned accesses and rejects them. A user can pass
1754	* BPF_F_ANY_ALIGNMENT to a program to override this
1755	* check. That, however, will only work when a privileged user
1756	* loads a program. An unprivileged user loading a program
1757	* with this flag will be rejected prior entering the
1758	* verifier.
1759	*/
1760	if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)
1761	return false;
1762	#endif
1763	return !test->prog_type ||
1764	test->prog_type == BPF_PROG_TYPE_SOCKET_FILTER ||
1765	test->prog_type == BPF_PROG_TYPE_CGROUP_SKB;
1766	}
1767
1768	static int do_test(bool unpriv, unsigned int from, unsigned int to)
1769	{
1770	int i, passes = 0, errors = 0;
1771
1772	/* ensure previous instance of the module is unloaded */
1773	unload_bpf_testmod(verbose);
1774
1775	if (load_bpf_testmod(verbose))
1776	return EXIT_FAILURE;
1777
1778	for (i = from; i < to; i++) {
1779	struct bpf_test *test = &tests[i];
1780
1781	/* Program types that are not supported by non-root we
1782	* skip right away.
1783	*/
1784	if (test_as_unpriv(test) && unpriv_disabled) {
1785	printf("#%d/u %s SKIP\n", i, test->descr);
1786	skips++;
1787	} else if (test_as_unpriv(test)) {
1788	if (!unpriv)
1789	set_admin(false);
1790	printf("#%d/u %s ", i, test->descr);
1791	do_test_single(test, unpriv: true, passes: &passes, errors: &errors);
1792	if (!unpriv)
1793	set_admin(true);
1794	}
1795
1796	if (unpriv) {
1797	printf("#%d/p %s SKIP\n", i, test->descr);
1798	skips++;
1799	} else {
1800	printf("#%d/p %s ", i, test->descr);
1801	do_test_single(test, unpriv: false, passes: &passes, errors: &errors);
1802	}
1803	}
1804
1805	unload_bpf_testmod(verbose);
1806	kfuncs_cleanup();
1807
1808	printf("Summary: %d PASSED, %d SKIPPED, %d FAILED\n", passes,
1809	skips, errors);
1810	return errors ? EXIT_FAILURE : EXIT_SUCCESS;
1811	}
1812
1813	int main(int argc, char **argv)
1814	{
1815	unsigned int from = 0, to = ARRAY_SIZE(tests);
1816	bool unpriv = !is_admin();
1817	int arg = 1;
1818
1819	if (argc > 1 && strcmp(argv[1], "-v") == 0) {
1820	arg++;
1821	verbose = true;
1822	verif_log_level = 1;
1823	argc--;
1824	}
1825	if (argc > 1 && strcmp(argv[1], "-vv") == 0) {
1826	arg++;
1827	verbose = true;
1828	verif_log_level = 2;
1829	argc--;
1830	}
1831
1832	if (argc == 3) {
1833	unsigned int l = atoi(argv[arg]);
1834	unsigned int u = atoi(argv[arg + 1]);
1835
1836	if (l < to && u < to) {
1837	from = l;
1838	to = u + 1;
1839	}
1840	} else if (argc == 2) {
1841	unsigned int t = atoi(argv[arg]);
1842
1843	if (t < to) {
1844	from = t;
1845	to = t + 1;
1846	}
1847	}
1848
1849	unpriv_disabled = get_unpriv_disabled();
1850	if (unpriv && unpriv_disabled) {
1851	printf("Cannot run as unprivileged user with sysctl %s.\n",
1852	UNPRIV_SYSCTL);
1853	return EXIT_FAILURE;
1854	}
1855
1856	jit_disabled = !is_jit_enabled();
1857
1858	/* Use libbpf 1.0 API mode */
1859	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1860
1861	bpf_semi_rand_init();
1862	return do_test(unpriv, from, to);
1863	}
1864