1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2020 Intel Corporation. */ |
3 | |
4 | /* |
5 | * Some functions in this program are taken from |
6 | * Linux kernel samples/bpf/xdpsock* and modified |
7 | * for use. |
8 | * |
9 | * See test_xsk.sh for detailed information on test topology |
10 | * and prerequisite network setup. |
11 | * |
12 | * This test program contains two threads, each thread is single socket with |
13 | * a unique UMEM. It validates in-order packet delivery and packet content |
14 | * by sending packets to each other. |
15 | * |
16 | * Tests Information: |
17 | * ------------------ |
18 | * These selftests test AF_XDP SKB and Native/DRV modes using veth |
19 | * Virtual Ethernet interfaces. |
20 | * |
21 | * For each mode, the following tests are run: |
22 | * a. nopoll - soft-irq processing in run-to-completion mode |
23 | * b. poll - using poll() syscall |
24 | * c. Socket Teardown |
25 | * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy |
26 | * both sockets, then repeat multiple times. Only nopoll mode is used |
27 | * d. Bi-directional sockets |
28 | * Configure sockets as bi-directional tx/rx sockets, sets up fill and |
29 | * completion rings on each socket, tx/rx in both directions. Only nopoll |
30 | * mode is used |
31 | * e. Statistics |
32 | * Trigger some error conditions and ensure that the appropriate statistics |
33 | * are incremented. Within this test, the following statistics are tested: |
34 | * i. rx dropped |
35 | * Increase the UMEM frame headroom to a value which results in |
36 | * insufficient space in the rx buffer for both the packet and the headroom. |
37 | * ii. tx invalid |
38 | * Set the 'len' field of tx descriptors to an invalid value (umem frame |
39 | * size + 1). |
40 | * iii. rx ring full |
41 | * Reduce the size of the RX ring to a fraction of the fill ring size. |
42 | * iv. fill queue empty |
43 | * Do not populate the fill queue and then try to receive pkts. |
44 | * f. bpf_link resource persistence |
45 | * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, |
46 | * then remove xsk sockets from queue 0 on both veth interfaces and |
47 | * finally run a traffic on queues ids 1 |
48 | * g. unaligned mode |
49 | * h. tests for invalid and corner case Tx descriptors so that the correct ones |
50 | * are discarded and let through, respectively. |
51 | * i. 2K frame size tests |
52 | * j. If multi-buffer is supported, send 9k packets divided into 3 frames |
53 | * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame |
54 | * using unaligned mode |
55 | * l. If multi-buffer is supported, try various nasty combinations of descriptors to |
56 | * check if they pass the validation or not |
57 | * |
58 | * Flow: |
59 | * ----- |
60 | * - Single process spawns two threads: Tx and Rx |
61 | * - Each of these two threads attach to a veth interface |
62 | * - Each thread creates one AF_XDP socket connected to a unique umem for each |
63 | * veth interface |
64 | * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy> |
65 | * - Rx thread verifies if all packets were received and delivered in-order, |
66 | * and have the right content |
67 | * |
68 | * Enable/disable packet dump mode: |
69 | * -------------------------- |
70 | * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add |
71 | * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") |
72 | */ |
73 | |
74 | #define _GNU_SOURCE |
75 | #include <assert.h> |
76 | #include <fcntl.h> |
77 | #include <errno.h> |
78 | #include <getopt.h> |
79 | #include <linux/if_link.h> |
80 | #include <linux/if_ether.h> |
81 | #include <linux/mman.h> |
82 | #include <linux/netdev.h> |
83 | #include <linux/bitmap.h> |
84 | #include <arpa/inet.h> |
85 | #include <net/if.h> |
86 | #include <locale.h> |
87 | #include <poll.h> |
88 | #include <pthread.h> |
89 | #include <signal.h> |
90 | #include <stdio.h> |
91 | #include <stdlib.h> |
92 | #include <string.h> |
93 | #include <stddef.h> |
94 | #include <sys/mman.h> |
95 | #include <sys/socket.h> |
96 | #include <sys/time.h> |
97 | #include <sys/types.h> |
98 | #include <unistd.h> |
99 | |
100 | #include "xsk_xdp_progs.skel.h" |
101 | #include "xsk.h" |
102 | #include "xskxceiver.h" |
103 | #include <bpf/bpf.h> |
104 | #include <linux/filter.h> |
105 | #include "../kselftest.h" |
106 | #include "xsk_xdp_common.h" |
107 | |
108 | static bool opt_verbose; |
109 | static bool opt_print_tests; |
110 | static enum test_mode opt_mode = TEST_MODE_ALL; |
111 | static u32 opt_run_test = RUN_ALL_TESTS; |
112 | |
113 | static void __exit_with_error(int error, const char *file, const char *func, int line) |
114 | { |
115 | ksft_test_result_fail(msg: "[%s:%s:%i]: ERROR: %d/\"%s\"\n" , file, func, line, error, |
116 | strerror(error)); |
117 | ksft_exit_xfail(); |
118 | } |
119 | |
120 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) |
121 | #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" |
122 | static char *mode_string(struct test_spec *test) |
123 | { |
124 | switch (test->mode) { |
125 | case TEST_MODE_SKB: |
126 | return "SKB" ; |
127 | case TEST_MODE_DRV: |
128 | return "DRV" ; |
129 | case TEST_MODE_ZC: |
130 | return "ZC" ; |
131 | default: |
132 | return "BOGUS" ; |
133 | } |
134 | } |
135 | |
136 | static void report_failure(struct test_spec *test) |
137 | { |
138 | if (test->fail) |
139 | return; |
140 | |
141 | ksft_test_result_fail(msg: "FAIL: %s %s%s\n" , mode_string(test), busy_poll_string(test), |
142 | test->name); |
143 | test->fail = true; |
144 | } |
145 | |
146 | /* The payload is a word consisting of a packet sequence number in the upper |
147 | * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's |
148 | * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. |
149 | */ |
150 | static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) |
151 | { |
152 | u32 *ptr = (u32 *)dest, i; |
153 | |
154 | start /= sizeof(*ptr); |
155 | size /= sizeof(*ptr); |
156 | for (i = 0; i < size; i++) |
157 | ptr[i] = htonl(pkt_nb << 16 | (i + start)); |
158 | } |
159 | |
160 | static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) |
161 | { |
162 | memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); |
163 | memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); |
164 | eth_hdr->h_proto = htons(ETH_P_LOOPBACK); |
165 | } |
166 | |
167 | static bool is_umem_valid(struct ifobject *ifobj) |
168 | { |
169 | return !!ifobj->umem->umem; |
170 | } |
171 | |
172 | static u32 mode_to_xdp_flags(enum test_mode mode) |
173 | { |
174 | return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; |
175 | } |
176 | |
177 | static u64 umem_size(struct xsk_umem_info *umem) |
178 | { |
179 | return umem->num_frames * umem->frame_size; |
180 | } |
181 | |
182 | static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, |
183 | u64 size) |
184 | { |
185 | struct xsk_umem_config cfg = { |
186 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, |
187 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, |
188 | .frame_size = umem->frame_size, |
189 | .frame_headroom = umem->frame_headroom, |
190 | .flags = XSK_UMEM__DEFAULT_FLAGS |
191 | }; |
192 | int ret; |
193 | |
194 | if (umem->unaligned_mode) |
195 | cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; |
196 | |
197 | ret = xsk_umem__create(umem: &umem->umem, umem_area: buffer, size, |
198 | fill: &umem->fq, comp: &umem->cq, config: &cfg); |
199 | if (ret) |
200 | return ret; |
201 | |
202 | umem->buffer = buffer; |
203 | if (ifobj->shared_umem && ifobj->rx_on) { |
204 | umem->base_addr = umem_size(umem); |
205 | umem->next_buffer = umem_size(umem); |
206 | } |
207 | |
208 | return 0; |
209 | } |
210 | |
211 | static u64 umem_alloc_buffer(struct xsk_umem_info *umem) |
212 | { |
213 | u64 addr; |
214 | |
215 | addr = umem->next_buffer; |
216 | umem->next_buffer += umem->frame_size; |
217 | if (umem->next_buffer >= umem->base_addr + umem_size(umem)) |
218 | umem->next_buffer = umem->base_addr; |
219 | |
220 | return addr; |
221 | } |
222 | |
223 | static void umem_reset_alloc(struct xsk_umem_info *umem) |
224 | { |
225 | umem->next_buffer = 0; |
226 | } |
227 | |
228 | static void enable_busy_poll(struct xsk_socket_info *xsk) |
229 | { |
230 | int sock_opt; |
231 | |
232 | sock_opt = 1; |
233 | if (setsockopt(xsk_socket__fd(xsk: xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, |
234 | (void *)&sock_opt, sizeof(sock_opt)) < 0) |
235 | exit_with_error(errno); |
236 | |
237 | sock_opt = 20; |
238 | if (setsockopt(xsk_socket__fd(xsk: xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, |
239 | (void *)&sock_opt, sizeof(sock_opt)) < 0) |
240 | exit_with_error(errno); |
241 | |
242 | sock_opt = BATCH_SIZE; |
243 | if (setsockopt(xsk_socket__fd(xsk: xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, |
244 | (void *)&sock_opt, sizeof(sock_opt)) < 0) |
245 | exit_with_error(errno); |
246 | } |
247 | |
248 | static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, |
249 | struct ifobject *ifobject, bool shared) |
250 | { |
251 | struct xsk_socket_config cfg = {}; |
252 | struct xsk_ring_cons *rxr; |
253 | struct xsk_ring_prod *txr; |
254 | |
255 | xsk->umem = umem; |
256 | cfg.rx_size = xsk->rxqsize; |
257 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; |
258 | cfg.bind_flags = ifobject->bind_flags; |
259 | if (shared) |
260 | cfg.bind_flags |= XDP_SHARED_UMEM; |
261 | if (ifobject->mtu > MAX_ETH_PKT_SIZE) |
262 | cfg.bind_flags |= XDP_USE_SG; |
263 | |
264 | txr = ifobject->tx_on ? &xsk->tx : NULL; |
265 | rxr = ifobject->rx_on ? &xsk->rx : NULL; |
266 | return xsk_socket__create(xsk: &xsk->xsk, ifindex: ifobject->ifindex, queue_id: 0, umem: umem->umem, rx: rxr, tx: txr, config: &cfg); |
267 | } |
268 | |
269 | static bool ifobj_zc_avail(struct ifobject *ifobject) |
270 | { |
271 | size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; |
272 | int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; |
273 | struct xsk_socket_info *xsk; |
274 | struct xsk_umem_info *umem; |
275 | bool zc_avail = false; |
276 | void *bufs; |
277 | int ret; |
278 | |
279 | bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); |
280 | if (bufs == MAP_FAILED) |
281 | exit_with_error(errno); |
282 | |
283 | umem = calloc(1, sizeof(struct xsk_umem_info)); |
284 | if (!umem) { |
285 | munmap(bufs, umem_sz); |
286 | exit_with_error(ENOMEM); |
287 | } |
288 | umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
289 | ret = xsk_configure_umem(ifobj: ifobject, umem, buffer: bufs, size: umem_sz); |
290 | if (ret) |
291 | exit_with_error(-ret); |
292 | |
293 | xsk = calloc(1, sizeof(struct xsk_socket_info)); |
294 | if (!xsk) |
295 | goto out; |
296 | ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; |
297 | ifobject->rx_on = true; |
298 | xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; |
299 | ret = __xsk_configure_socket(xsk, umem, ifobject, shared: false); |
300 | if (!ret) |
301 | zc_avail = true; |
302 | |
303 | xsk_socket__delete(xsk: xsk->xsk); |
304 | free(xsk); |
305 | out: |
306 | munmap(umem->buffer, umem_sz); |
307 | xsk_umem__delete(umem: umem->umem); |
308 | free(umem); |
309 | return zc_avail; |
310 | } |
311 | |
312 | static struct option long_options[] = { |
313 | {"interface" , required_argument, 0, 'i'}, |
314 | {"busy-poll" , no_argument, 0, 'b'}, |
315 | {"verbose" , no_argument, 0, 'v'}, |
316 | {"mode" , required_argument, 0, 'm'}, |
317 | {"list" , no_argument, 0, 'l'}, |
318 | {"test" , required_argument, 0, 't'}, |
319 | {"help" , no_argument, 0, 'h'}, |
320 | {0, 0, 0, 0} |
321 | }; |
322 | |
323 | static void print_usage(char **argv) |
324 | { |
325 | const char *str = |
326 | " Usage: xskxceiver [OPTIONS]\n" |
327 | " Options:\n" |
328 | " -i, --interface Use interface\n" |
329 | " -v, --verbose Verbose output\n" |
330 | " -b, --busy-poll Enable busy poll\n" |
331 | " -m, --mode Run only mode skb, drv, or zc\n" |
332 | " -l, --list List all available tests\n" |
333 | " -t, --test Run a specific test. Enter number from -l option.\n" |
334 | " -h, --help Display this help and exit\n" ; |
335 | |
336 | ksft_print_msg(msg: str, basename(argv[0])); |
337 | ksft_exit_xfail(); |
338 | } |
339 | |
340 | static bool validate_interface(struct ifobject *ifobj) |
341 | { |
342 | if (!strcmp(ifobj->ifname, "" )) |
343 | return false; |
344 | return true; |
345 | } |
346 | |
347 | static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, |
348 | char **argv) |
349 | { |
350 | struct ifobject *ifobj; |
351 | u32 interface_nb = 0; |
352 | int option_index, c; |
353 | |
354 | opterr = 0; |
355 | |
356 | for (;;) { |
357 | c = getopt_long(argc, argv, "i:vbm:lt:" , long_options, &option_index); |
358 | if (c == -1) |
359 | break; |
360 | |
361 | switch (c) { |
362 | case 'i': |
363 | if (interface_nb == 0) |
364 | ifobj = ifobj_tx; |
365 | else if (interface_nb == 1) |
366 | ifobj = ifobj_rx; |
367 | else |
368 | break; |
369 | |
370 | memcpy(ifobj->ifname, optarg, |
371 | min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg))); |
372 | |
373 | ifobj->ifindex = if_nametoindex(ifobj->ifname); |
374 | if (!ifobj->ifindex) |
375 | exit_with_error(errno); |
376 | |
377 | interface_nb++; |
378 | break; |
379 | case 'v': |
380 | opt_verbose = true; |
381 | break; |
382 | case 'b': |
383 | ifobj_tx->busy_poll = true; |
384 | ifobj_rx->busy_poll = true; |
385 | break; |
386 | case 'm': |
387 | if (!strncmp("skb" , optarg, strlen(optarg))) |
388 | opt_mode = TEST_MODE_SKB; |
389 | else if (!strncmp("drv" , optarg, strlen(optarg))) |
390 | opt_mode = TEST_MODE_DRV; |
391 | else if (!strncmp("zc" , optarg, strlen(optarg))) |
392 | opt_mode = TEST_MODE_ZC; |
393 | else |
394 | print_usage(argv); |
395 | break; |
396 | case 'l': |
397 | opt_print_tests = true; |
398 | break; |
399 | case 't': |
400 | errno = 0; |
401 | opt_run_test = strtol(optarg, NULL, 0); |
402 | if (errno) |
403 | print_usage(argv); |
404 | break; |
405 | case 'h': |
406 | default: |
407 | print_usage(argv); |
408 | } |
409 | } |
410 | } |
411 | |
412 | static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, |
413 | struct ifobject *ifobj_rx) |
414 | { |
415 | u32 i, j; |
416 | |
417 | for (i = 0; i < MAX_INTERFACES; i++) { |
418 | struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; |
419 | |
420 | ifobj->xsk = &ifobj->xsk_arr[0]; |
421 | ifobj->use_poll = false; |
422 | ifobj->use_fill_ring = true; |
423 | ifobj->release_rx = true; |
424 | ifobj->validation_func = NULL; |
425 | ifobj->use_metadata = false; |
426 | |
427 | if (i == 0) { |
428 | ifobj->rx_on = false; |
429 | ifobj->tx_on = true; |
430 | } else { |
431 | ifobj->rx_on = true; |
432 | ifobj->tx_on = false; |
433 | } |
434 | |
435 | memset(ifobj->umem, 0, sizeof(*ifobj->umem)); |
436 | ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; |
437 | ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
438 | |
439 | for (j = 0; j < MAX_SOCKETS; j++) { |
440 | memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); |
441 | ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; |
442 | if (i == 0) |
443 | ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; |
444 | else |
445 | ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; |
446 | |
447 | memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); |
448 | memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); |
449 | ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); |
450 | ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); |
451 | } |
452 | } |
453 | |
454 | test->ifobj_tx = ifobj_tx; |
455 | test->ifobj_rx = ifobj_rx; |
456 | test->current_step = 0; |
457 | test->total_steps = 1; |
458 | test->nb_sockets = 1; |
459 | test->fail = false; |
460 | test->mtu = MAX_ETH_PKT_SIZE; |
461 | test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; |
462 | test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; |
463 | test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; |
464 | test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; |
465 | } |
466 | |
467 | static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, |
468 | struct ifobject *ifobj_rx, enum test_mode mode, |
469 | const struct test_spec *test_to_run) |
470 | { |
471 | struct pkt_stream *tx_pkt_stream; |
472 | struct pkt_stream *rx_pkt_stream; |
473 | u32 i; |
474 | |
475 | tx_pkt_stream = test->tx_pkt_stream_default; |
476 | rx_pkt_stream = test->rx_pkt_stream_default; |
477 | memset(test, 0, sizeof(*test)); |
478 | test->tx_pkt_stream_default = tx_pkt_stream; |
479 | test->rx_pkt_stream_default = rx_pkt_stream; |
480 | |
481 | for (i = 0; i < MAX_INTERFACES; i++) { |
482 | struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; |
483 | |
484 | ifobj->bind_flags = XDP_USE_NEED_WAKEUP; |
485 | if (mode == TEST_MODE_ZC) |
486 | ifobj->bind_flags |= XDP_ZEROCOPY; |
487 | else |
488 | ifobj->bind_flags |= XDP_COPY; |
489 | } |
490 | |
491 | strncpy(p: test->name, q: test_to_run->name, MAX_TEST_NAME_SIZE); |
492 | test->test_func = test_to_run->test_func; |
493 | test->mode = mode; |
494 | __test_spec_init(test, ifobj_tx, ifobj_rx); |
495 | } |
496 | |
497 | static void test_spec_reset(struct test_spec *test) |
498 | { |
499 | __test_spec_init(test, ifobj_tx: test->ifobj_tx, ifobj_rx: test->ifobj_rx); |
500 | } |
501 | |
502 | static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, |
503 | struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, |
504 | struct bpf_map *xskmap_tx) |
505 | { |
506 | test->xdp_prog_rx = xdp_prog_rx; |
507 | test->xdp_prog_tx = xdp_prog_tx; |
508 | test->xskmap_rx = xskmap_rx; |
509 | test->xskmap_tx = xskmap_tx; |
510 | } |
511 | |
512 | static int test_spec_set_mtu(struct test_spec *test, int mtu) |
513 | { |
514 | int err; |
515 | |
516 | if (test->ifobj_rx->mtu != mtu) { |
517 | err = xsk_set_mtu(ifindex: test->ifobj_rx->ifindex, mtu); |
518 | if (err) |
519 | return err; |
520 | test->ifobj_rx->mtu = mtu; |
521 | } |
522 | if (test->ifobj_tx->mtu != mtu) { |
523 | err = xsk_set_mtu(ifindex: test->ifobj_tx->ifindex, mtu); |
524 | if (err) |
525 | return err; |
526 | test->ifobj_tx->mtu = mtu; |
527 | } |
528 | |
529 | return 0; |
530 | } |
531 | |
532 | static void pkt_stream_reset(struct pkt_stream *pkt_stream) |
533 | { |
534 | if (pkt_stream) { |
535 | pkt_stream->current_pkt_nb = 0; |
536 | pkt_stream->nb_rx_pkts = 0; |
537 | } |
538 | } |
539 | |
540 | static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) |
541 | { |
542 | if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) |
543 | return NULL; |
544 | |
545 | return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; |
546 | } |
547 | |
548 | static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) |
549 | { |
550 | while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { |
551 | (*pkts_sent)++; |
552 | if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) |
553 | return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; |
554 | pkt_stream->current_pkt_nb++; |
555 | } |
556 | return NULL; |
557 | } |
558 | |
559 | static void pkt_stream_delete(struct pkt_stream *pkt_stream) |
560 | { |
561 | free(pkt_stream->pkts); |
562 | free(pkt_stream); |
563 | } |
564 | |
565 | static void pkt_stream_restore_default(struct test_spec *test) |
566 | { |
567 | struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; |
568 | struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; |
569 | |
570 | if (tx_pkt_stream != test->tx_pkt_stream_default) { |
571 | pkt_stream_delete(pkt_stream: test->ifobj_tx->xsk->pkt_stream); |
572 | test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; |
573 | } |
574 | |
575 | if (rx_pkt_stream != test->rx_pkt_stream_default) { |
576 | pkt_stream_delete(pkt_stream: test->ifobj_rx->xsk->pkt_stream); |
577 | test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; |
578 | } |
579 | } |
580 | |
581 | static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) |
582 | { |
583 | struct pkt_stream *pkt_stream; |
584 | |
585 | pkt_stream = calloc(1, sizeof(*pkt_stream)); |
586 | if (!pkt_stream) |
587 | return NULL; |
588 | |
589 | pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); |
590 | if (!pkt_stream->pkts) { |
591 | free(pkt_stream); |
592 | return NULL; |
593 | } |
594 | |
595 | pkt_stream->nb_pkts = nb_pkts; |
596 | return pkt_stream; |
597 | } |
598 | |
599 | static bool pkt_continues(u32 options) |
600 | { |
601 | return options & XDP_PKT_CONTD; |
602 | } |
603 | |
604 | static u32 ceil_u32(u32 a, u32 b) |
605 | { |
606 | return (a + b - 1) / b; |
607 | } |
608 | |
609 | static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) |
610 | { |
611 | u32 nb_frags = 1, next_frag; |
612 | |
613 | if (!pkt) |
614 | return 1; |
615 | |
616 | if (!pkt_stream->verbatim) { |
617 | if (!pkt->valid || !pkt->len) |
618 | return 1; |
619 | return ceil_u32(a: pkt->len, b: frame_size); |
620 | } |
621 | |
622 | /* Search for the end of the packet in verbatim mode */ |
623 | if (!pkt_continues(options: pkt->options)) |
624 | return nb_frags; |
625 | |
626 | next_frag = pkt_stream->current_pkt_nb; |
627 | pkt++; |
628 | while (next_frag++ < pkt_stream->nb_pkts) { |
629 | nb_frags++; |
630 | if (!pkt_continues(options: pkt->options) || !pkt->valid) |
631 | break; |
632 | pkt++; |
633 | } |
634 | return nb_frags; |
635 | } |
636 | |
637 | static bool set_pkt_valid(int offset, u32 len) |
638 | { |
639 | return len <= MAX_ETH_JUMBO_SIZE; |
640 | } |
641 | |
642 | static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) |
643 | { |
644 | pkt->offset = offset; |
645 | pkt->len = len; |
646 | pkt->valid = set_pkt_valid(offset, len); |
647 | } |
648 | |
649 | static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) |
650 | { |
651 | bool prev_pkt_valid = pkt->valid; |
652 | |
653 | pkt_set(pkt_stream, pkt, offset, len); |
654 | pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; |
655 | } |
656 | |
657 | static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) |
658 | { |
659 | return ceil_u32(a: len, b: umem->frame_size) * umem->frame_size; |
660 | } |
661 | |
662 | static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) |
663 | { |
664 | struct pkt_stream *pkt_stream; |
665 | u32 i; |
666 | |
667 | pkt_stream = __pkt_stream_alloc(nb_pkts); |
668 | if (!pkt_stream) |
669 | exit_with_error(ENOMEM); |
670 | |
671 | pkt_stream->nb_pkts = nb_pkts; |
672 | pkt_stream->max_pkt_len = pkt_len; |
673 | for (i = 0; i < nb_pkts; i++) { |
674 | struct pkt *pkt = &pkt_stream->pkts[i]; |
675 | |
676 | pkt_stream_pkt_set(pkt_stream, pkt, offset: 0, len: pkt_len); |
677 | pkt->pkt_nb = nb_start + i * nb_off; |
678 | } |
679 | |
680 | return pkt_stream; |
681 | } |
682 | |
683 | static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) |
684 | { |
685 | return __pkt_stream_generate(nb_pkts, pkt_len, nb_start: 0, nb_off: 1); |
686 | } |
687 | |
688 | static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) |
689 | { |
690 | return pkt_stream_generate(nb_pkts: pkt_stream->nb_pkts, pkt_len: pkt_stream->pkts[0].len); |
691 | } |
692 | |
693 | static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) |
694 | { |
695 | struct pkt_stream *pkt_stream; |
696 | |
697 | pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); |
698 | test->ifobj_tx->xsk->pkt_stream = pkt_stream; |
699 | pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); |
700 | test->ifobj_rx->xsk->pkt_stream = pkt_stream; |
701 | } |
702 | |
703 | static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, |
704 | int offset) |
705 | { |
706 | struct pkt_stream *pkt_stream; |
707 | u32 i; |
708 | |
709 | pkt_stream = pkt_stream_clone(pkt_stream: ifobj->xsk->pkt_stream); |
710 | for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) |
711 | pkt_stream_pkt_set(pkt_stream, pkt: &pkt_stream->pkts[i], offset, len: pkt_len); |
712 | |
713 | ifobj->xsk->pkt_stream = pkt_stream; |
714 | } |
715 | |
716 | static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) |
717 | { |
718 | __pkt_stream_replace_half(ifobj: test->ifobj_tx, pkt_len, offset); |
719 | __pkt_stream_replace_half(ifobj: test->ifobj_rx, pkt_len, offset); |
720 | } |
721 | |
722 | static void pkt_stream_receive_half(struct test_spec *test) |
723 | { |
724 | struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; |
725 | u32 i; |
726 | |
727 | test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(nb_pkts: pkt_stream->nb_pkts, |
728 | pkt_len: pkt_stream->pkts[0].len); |
729 | pkt_stream = test->ifobj_rx->xsk->pkt_stream; |
730 | for (i = 1; i < pkt_stream->nb_pkts; i += 2) |
731 | pkt_stream->pkts[i].valid = false; |
732 | |
733 | pkt_stream->nb_valid_entries /= 2; |
734 | } |
735 | |
736 | static void pkt_stream_even_odd_sequence(struct test_spec *test) |
737 | { |
738 | struct pkt_stream *pkt_stream; |
739 | u32 i; |
740 | |
741 | for (i = 0; i < test->nb_sockets; i++) { |
742 | pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; |
743 | pkt_stream = __pkt_stream_generate(nb_pkts: pkt_stream->nb_pkts / 2, |
744 | pkt_len: pkt_stream->pkts[0].len, nb_start: i, nb_off: 2); |
745 | test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; |
746 | |
747 | pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; |
748 | pkt_stream = __pkt_stream_generate(nb_pkts: pkt_stream->nb_pkts / 2, |
749 | pkt_len: pkt_stream->pkts[0].len, nb_start: i, nb_off: 2); |
750 | test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; |
751 | } |
752 | } |
753 | |
754 | static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) |
755 | { |
756 | if (!pkt->valid) |
757 | return pkt->offset; |
758 | return pkt->offset + umem_alloc_buffer(umem); |
759 | } |
760 | |
761 | static void pkt_stream_cancel(struct pkt_stream *pkt_stream) |
762 | { |
763 | pkt_stream->current_pkt_nb--; |
764 | } |
765 | |
766 | static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, |
767 | u32 pkt_nb, u32 bytes_written) |
768 | { |
769 | void *data = xsk_umem__get_data(umem_area: umem->buffer, addr); |
770 | |
771 | if (len < MIN_PKT_SIZE) |
772 | return; |
773 | |
774 | if (!bytes_written) { |
775 | gen_eth_hdr(xsk, eth_hdr: data); |
776 | |
777 | len -= PKT_HDR_SIZE; |
778 | data += PKT_HDR_SIZE; |
779 | } else { |
780 | bytes_written -= PKT_HDR_SIZE; |
781 | } |
782 | |
783 | write_payload(dest: data, pkt_nb, start: bytes_written, size: len); |
784 | } |
785 | |
786 | static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, |
787 | u32 nb_frames, bool verbatim) |
788 | { |
789 | u32 i, len = 0, pkt_nb = 0, payload = 0; |
790 | struct pkt_stream *pkt_stream; |
791 | |
792 | pkt_stream = __pkt_stream_alloc(nb_pkts: nb_frames); |
793 | if (!pkt_stream) |
794 | exit_with_error(ENOMEM); |
795 | |
796 | for (i = 0; i < nb_frames; i++) { |
797 | struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; |
798 | struct pkt *frame = &frames[i]; |
799 | |
800 | pkt->offset = frame->offset; |
801 | if (verbatim) { |
802 | *pkt = *frame; |
803 | pkt->pkt_nb = payload; |
804 | if (!frame->valid || !pkt_continues(options: frame->options)) |
805 | payload++; |
806 | } else { |
807 | if (frame->valid) |
808 | len += frame->len; |
809 | if (frame->valid && pkt_continues(options: frame->options)) |
810 | continue; |
811 | |
812 | pkt->pkt_nb = pkt_nb; |
813 | pkt->len = len; |
814 | pkt->valid = frame->valid; |
815 | pkt->options = 0; |
816 | |
817 | len = 0; |
818 | } |
819 | |
820 | print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n" , |
821 | pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); |
822 | |
823 | if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) |
824 | pkt_stream->max_pkt_len = pkt->len; |
825 | |
826 | if (pkt->valid) |
827 | pkt_stream->nb_valid_entries++; |
828 | |
829 | pkt_nb++; |
830 | } |
831 | |
832 | pkt_stream->nb_pkts = pkt_nb; |
833 | pkt_stream->verbatim = verbatim; |
834 | return pkt_stream; |
835 | } |
836 | |
837 | static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) |
838 | { |
839 | struct pkt_stream *pkt_stream; |
840 | |
841 | pkt_stream = __pkt_stream_generate_custom(ifobj: test->ifobj_tx, frames: pkts, nb_frames: nb_pkts, verbatim: true); |
842 | test->ifobj_tx->xsk->pkt_stream = pkt_stream; |
843 | |
844 | pkt_stream = __pkt_stream_generate_custom(ifobj: test->ifobj_rx, frames: pkts, nb_frames: nb_pkts, verbatim: false); |
845 | test->ifobj_rx->xsk->pkt_stream = pkt_stream; |
846 | } |
847 | |
848 | static void pkt_print_data(u32 *data, u32 cnt) |
849 | { |
850 | u32 i; |
851 | |
852 | for (i = 0; i < cnt; i++) { |
853 | u32 seqnum, pkt_nb; |
854 | |
855 | seqnum = ntohl(*data) & 0xffff; |
856 | pkt_nb = ntohl(*data) >> 16; |
857 | ksft_print_msg(msg: "%u:%u " , pkt_nb, seqnum); |
858 | data++; |
859 | } |
860 | } |
861 | |
862 | static void pkt_dump(void *pkt, u32 len, bool ) |
863 | { |
864 | struct ethhdr *ethhdr = pkt; |
865 | u32 i, *data; |
866 | |
867 | if (eth_header) { |
868 | /*extract L2 frame */ |
869 | ksft_print_msg(msg: "DEBUG>> L2: dst mac: " ); |
870 | for (i = 0; i < ETH_ALEN; i++) |
871 | ksft_print_msg(msg: "%02X" , ethhdr->h_dest[i]); |
872 | |
873 | ksft_print_msg(msg: "\nDEBUG>> L2: src mac: " ); |
874 | for (i = 0; i < ETH_ALEN; i++) |
875 | ksft_print_msg(msg: "%02X" , ethhdr->h_source[i]); |
876 | |
877 | data = pkt + PKT_HDR_SIZE; |
878 | } else { |
879 | data = pkt; |
880 | } |
881 | |
882 | /*extract L5 frame */ |
883 | ksft_print_msg(msg: "\nDEBUG>> L5: seqnum: " ); |
884 | pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); |
885 | ksft_print_msg(msg: "...." ); |
886 | if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { |
887 | ksft_print_msg(msg: "\n.... " ); |
888 | pkt_print_data(data: data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, |
889 | PKT_DUMP_NB_TO_PRINT); |
890 | } |
891 | ksft_print_msg(msg: "\n---------------------------------------\n" ); |
892 | } |
893 | |
894 | static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) |
895 | { |
896 | u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; |
897 | u32 offset = addr % umem->frame_size, expected_offset; |
898 | int pkt_offset = pkt->valid ? pkt->offset : 0; |
899 | |
900 | if (!umem->unaligned_mode) |
901 | pkt_offset = 0; |
902 | |
903 | expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; |
904 | |
905 | if (offset == expected_offset) |
906 | return true; |
907 | |
908 | ksft_print_msg(msg: "[%s] expected [%u], got [%u]\n" , __func__, expected_offset, offset); |
909 | return false; |
910 | } |
911 | |
912 | static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) |
913 | { |
914 | void *data = xsk_umem__get_data(umem_area: buffer, addr); |
915 | struct xdp_info *meta = data - sizeof(struct xdp_info); |
916 | |
917 | if (meta->count != pkt->pkt_nb) { |
918 | ksft_print_msg(msg: "[%s] expected meta_count [%d], got meta_count [%llu]\n" , |
919 | __func__, pkt->pkt_nb, |
920 | (unsigned long long)meta->count); |
921 | return false; |
922 | } |
923 | |
924 | return true; |
925 | } |
926 | |
927 | static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, |
928 | u32 bytes_processed) |
929 | { |
930 | u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; |
931 | void *data = xsk_umem__get_data(umem_area: umem->buffer, addr); |
932 | |
933 | addr -= umem->base_addr; |
934 | |
935 | if (addr >= umem->num_frames * umem->frame_size || |
936 | addr + len > umem->num_frames * umem->frame_size) { |
937 | ksft_print_msg(msg: "Frag invalid addr: %llx len: %u\n" , |
938 | (unsigned long long)addr, len); |
939 | return false; |
940 | } |
941 | if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { |
942 | ksft_print_msg(msg: "Frag crosses frame boundary addr: %llx len: %u\n" , |
943 | (unsigned long long)addr, len); |
944 | return false; |
945 | } |
946 | |
947 | pkt_data = data; |
948 | if (!bytes_processed) { |
949 | pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); |
950 | len -= PKT_HDR_SIZE; |
951 | } else { |
952 | bytes_processed -= PKT_HDR_SIZE; |
953 | } |
954 | |
955 | expected_seqnum = bytes_processed / sizeof(*pkt_data); |
956 | seqnum = ntohl(*pkt_data) & 0xffff; |
957 | pkt_nb = ntohl(*pkt_data) >> 16; |
958 | |
959 | if (expected_pkt_nb != pkt_nb) { |
960 | ksft_print_msg(msg: "[%s] expected pkt_nb [%u], got pkt_nb [%u]\n" , |
961 | __func__, expected_pkt_nb, pkt_nb); |
962 | goto error; |
963 | } |
964 | if (expected_seqnum != seqnum) { |
965 | ksft_print_msg(msg: "[%s] expected seqnum at start [%u], got seqnum [%u]\n" , |
966 | __func__, expected_seqnum, seqnum); |
967 | goto error; |
968 | } |
969 | |
970 | words_to_end = len / sizeof(*pkt_data) - 1; |
971 | pkt_data += words_to_end; |
972 | seqnum = ntohl(*pkt_data) & 0xffff; |
973 | expected_seqnum += words_to_end; |
974 | if (expected_seqnum != seqnum) { |
975 | ksft_print_msg(msg: "[%s] expected seqnum at end [%u], got seqnum [%u]\n" , |
976 | __func__, expected_seqnum, seqnum); |
977 | goto error; |
978 | } |
979 | |
980 | return true; |
981 | |
982 | error: |
983 | pkt_dump(pkt: data, len, eth_header: !bytes_processed); |
984 | return false; |
985 | } |
986 | |
987 | static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) |
988 | { |
989 | if (pkt->len != len) { |
990 | ksft_print_msg(msg: "[%s] expected packet length [%d], got length [%d]\n" , |
991 | __func__, pkt->len, len); |
992 | pkt_dump(pkt: xsk_umem__get_data(umem_area: buffer, addr), len, eth_header: true); |
993 | return false; |
994 | } |
995 | |
996 | return true; |
997 | } |
998 | |
999 | static int kick_tx(struct xsk_socket_info *xsk) |
1000 | { |
1001 | int ret; |
1002 | |
1003 | ret = sendto(xsk_socket__fd(xsk: xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
1004 | if (ret >= 0) |
1005 | return TEST_PASS; |
1006 | if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { |
1007 | usleep(100); |
1008 | return TEST_PASS; |
1009 | } |
1010 | return TEST_FAILURE; |
1011 | } |
1012 | |
1013 | static int kick_rx(struct xsk_socket_info *xsk) |
1014 | { |
1015 | int ret; |
1016 | |
1017 | ret = recvfrom(xsk_socket__fd(xsk: xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); |
1018 | if (ret < 0) |
1019 | return TEST_FAILURE; |
1020 | |
1021 | return TEST_PASS; |
1022 | } |
1023 | |
1024 | static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) |
1025 | { |
1026 | unsigned int rcvd; |
1027 | u32 idx; |
1028 | int ret; |
1029 | |
1030 | if (xsk_ring_prod__needs_wakeup(r: &xsk->tx)) { |
1031 | ret = kick_tx(xsk); |
1032 | if (ret) |
1033 | return TEST_FAILURE; |
1034 | } |
1035 | |
1036 | rcvd = xsk_ring_cons__peek(cons: &xsk->umem->cq, nb: batch_size, idx: &idx); |
1037 | if (rcvd) { |
1038 | if (rcvd > xsk->outstanding_tx) { |
1039 | u64 addr = *xsk_ring_cons__comp_addr(comp: &xsk->umem->cq, idx: idx + rcvd - 1); |
1040 | |
1041 | ksft_print_msg(msg: "[%s] Too many packets completed\n" , __func__); |
1042 | ksft_print_msg(msg: "Last completion address: %llx\n" , |
1043 | (unsigned long long)addr); |
1044 | return TEST_FAILURE; |
1045 | } |
1046 | |
1047 | xsk_ring_cons__release(cons: &xsk->umem->cq, nb: rcvd); |
1048 | xsk->outstanding_tx -= rcvd; |
1049 | } |
1050 | |
1051 | return TEST_PASS; |
1052 | } |
1053 | |
1054 | static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) |
1055 | { |
1056 | u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; |
1057 | u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; |
1058 | struct pkt_stream *pkt_stream = xsk->pkt_stream; |
1059 | struct ifobject *ifobj = test->ifobj_rx; |
1060 | struct xsk_umem_info *umem = xsk->umem; |
1061 | struct pollfd fds = { }; |
1062 | struct pkt *pkt; |
1063 | u64 first_addr = 0; |
1064 | int ret; |
1065 | |
1066 | fds.fd = xsk_socket__fd(xsk: xsk->xsk); |
1067 | fds.events = POLLIN; |
1068 | |
1069 | ret = kick_rx(xsk); |
1070 | if (ret) |
1071 | return TEST_FAILURE; |
1072 | |
1073 | if (ifobj->use_poll) { |
1074 | ret = poll(&fds, 1, POLL_TMOUT); |
1075 | if (ret < 0) |
1076 | return TEST_FAILURE; |
1077 | |
1078 | if (!ret) { |
1079 | if (!is_umem_valid(ifobj: test->ifobj_tx)) |
1080 | return TEST_PASS; |
1081 | |
1082 | ksft_print_msg(msg: "ERROR: [%s] Poll timed out\n" , __func__); |
1083 | return TEST_CONTINUE; |
1084 | } |
1085 | |
1086 | if (!(fds.revents & POLLIN)) |
1087 | return TEST_CONTINUE; |
1088 | } |
1089 | |
1090 | rcvd = xsk_ring_cons__peek(cons: &xsk->rx, BATCH_SIZE, idx: &idx_rx); |
1091 | if (!rcvd) |
1092 | return TEST_CONTINUE; |
1093 | |
1094 | if (ifobj->use_fill_ring) { |
1095 | ret = xsk_ring_prod__reserve(prod: &umem->fq, nb: rcvd, idx: &idx_fq); |
1096 | while (ret != rcvd) { |
1097 | if (xsk_ring_prod__needs_wakeup(r: &umem->fq)) { |
1098 | ret = poll(&fds, 1, POLL_TMOUT); |
1099 | if (ret < 0) |
1100 | return TEST_FAILURE; |
1101 | } |
1102 | ret = xsk_ring_prod__reserve(prod: &umem->fq, nb: rcvd, idx: &idx_fq); |
1103 | } |
1104 | } |
1105 | |
1106 | while (frags_processed < rcvd) { |
1107 | const struct xdp_desc *desc = xsk_ring_cons__rx_desc(rx: &xsk->rx, idx: idx_rx++); |
1108 | u64 addr = desc->addr, orig; |
1109 | |
1110 | orig = xsk_umem__extract_addr(addr); |
1111 | addr = xsk_umem__add_offset_to_addr(addr); |
1112 | |
1113 | if (!nb_frags) { |
1114 | pkt = pkt_stream_get_next_rx_pkt(pkt_stream, pkts_sent: &pkts_sent); |
1115 | if (!pkt) { |
1116 | ksft_print_msg(msg: "[%s] received too many packets addr: %lx len %u\n" , |
1117 | __func__, addr, desc->len); |
1118 | return TEST_FAILURE; |
1119 | } |
1120 | } |
1121 | |
1122 | print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n" , |
1123 | addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); |
1124 | |
1125 | if (!is_frag_valid(umem, addr, len: desc->len, expected_pkt_nb: pkt->pkt_nb, bytes_processed: pkt_len) || |
1126 | !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && |
1127 | !is_metadata_correct(pkt, buffer: umem->buffer, addr))) |
1128 | return TEST_FAILURE; |
1129 | |
1130 | if (!nb_frags++) |
1131 | first_addr = addr; |
1132 | frags_processed++; |
1133 | pkt_len += desc->len; |
1134 | if (ifobj->use_fill_ring) |
1135 | *xsk_ring_prod__fill_addr(fill: &umem->fq, idx: idx_fq++) = orig; |
1136 | |
1137 | if (pkt_continues(options: desc->options)) |
1138 | continue; |
1139 | |
1140 | /* The complete packet has been received */ |
1141 | if (!is_pkt_valid(pkt, buffer: umem->buffer, addr: first_addr, len: pkt_len) || |
1142 | !is_offset_correct(umem, pkt, addr)) |
1143 | return TEST_FAILURE; |
1144 | |
1145 | pkt_stream->nb_rx_pkts++; |
1146 | nb_frags = 0; |
1147 | pkt_len = 0; |
1148 | } |
1149 | |
1150 | if (nb_frags) { |
1151 | /* In the middle of a packet. Start over from beginning of packet. */ |
1152 | idx_rx -= nb_frags; |
1153 | xsk_ring_cons__cancel(cons: &xsk->rx, nb: nb_frags); |
1154 | if (ifobj->use_fill_ring) { |
1155 | idx_fq -= nb_frags; |
1156 | xsk_ring_prod__cancel(prod: &umem->fq, nb: nb_frags); |
1157 | } |
1158 | frags_processed -= nb_frags; |
1159 | } |
1160 | |
1161 | if (ifobj->use_fill_ring) |
1162 | xsk_ring_prod__submit(prod: &umem->fq, nb: frags_processed); |
1163 | if (ifobj->release_rx) |
1164 | xsk_ring_cons__release(cons: &xsk->rx, nb: frags_processed); |
1165 | |
1166 | pthread_mutex_lock(&pacing_mutex); |
1167 | pkts_in_flight -= pkts_sent; |
1168 | pthread_mutex_unlock(&pacing_mutex); |
1169 | pkts_sent = 0; |
1170 | |
1171 | return TEST_CONTINUE; |
1172 | } |
1173 | |
1174 | bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, |
1175 | unsigned long *bitmap) |
1176 | { |
1177 | struct pkt_stream *pkt_stream = xsk->pkt_stream; |
1178 | |
1179 | if (!pkt_stream) { |
1180 | __set_bit(sock_num, bitmap); |
1181 | return false; |
1182 | } |
1183 | |
1184 | if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { |
1185 | __set_bit(sock_num, bitmap); |
1186 | if (bitmap_full(src: bitmap, nbits: test->nb_sockets)) |
1187 | return true; |
1188 | } |
1189 | |
1190 | return false; |
1191 | } |
1192 | |
1193 | static int receive_pkts(struct test_spec *test) |
1194 | { |
1195 | struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; |
1196 | DECLARE_BITMAP(bitmap, test->nb_sockets); |
1197 | struct xsk_socket_info *xsk; |
1198 | u32 sock_num = 0; |
1199 | int res, ret; |
1200 | |
1201 | ret = gettimeofday(&tv_now, NULL); |
1202 | if (ret) |
1203 | exit_with_error(errno); |
1204 | |
1205 | timeradd(&tv_now, &tv_timeout, &tv_end); |
1206 | |
1207 | while (1) { |
1208 | xsk = &test->ifobj_rx->xsk_arr[sock_num]; |
1209 | |
1210 | if ((all_packets_received(test, xsk, sock_num, bitmap))) |
1211 | break; |
1212 | |
1213 | res = __receive_pkts(test, xsk); |
1214 | if (!(res == TEST_PASS || res == TEST_CONTINUE)) |
1215 | return res; |
1216 | |
1217 | ret = gettimeofday(&tv_now, NULL); |
1218 | if (ret) |
1219 | exit_with_error(errno); |
1220 | |
1221 | if (timercmp(&tv_now, &tv_end, >)) { |
1222 | ksft_print_msg(msg: "ERROR: [%s] Receive loop timed out\n" , __func__); |
1223 | return TEST_FAILURE; |
1224 | } |
1225 | sock_num = (sock_num + 1) % test->nb_sockets; |
1226 | } |
1227 | |
1228 | return TEST_PASS; |
1229 | } |
1230 | |
1231 | static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) |
1232 | { |
1233 | u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; |
1234 | struct pkt_stream *pkt_stream = xsk->pkt_stream; |
1235 | struct xsk_umem_info *umem = ifobject->umem; |
1236 | bool use_poll = ifobject->use_poll; |
1237 | struct pollfd fds = { }; |
1238 | int ret; |
1239 | |
1240 | buffer_len = pkt_get_buffer_len(umem, len: pkt_stream->max_pkt_len); |
1241 | /* pkts_in_flight might be negative if many invalid packets are sent */ |
1242 | if (pkts_in_flight >= (int)((umem_size(umem) - BATCH_SIZE * buffer_len) / buffer_len)) { |
1243 | ret = kick_tx(xsk); |
1244 | if (ret) |
1245 | return TEST_FAILURE; |
1246 | return TEST_CONTINUE; |
1247 | } |
1248 | |
1249 | fds.fd = xsk_socket__fd(xsk: xsk->xsk); |
1250 | fds.events = POLLOUT; |
1251 | |
1252 | while (xsk_ring_prod__reserve(prod: &xsk->tx, BATCH_SIZE, idx: &idx) < BATCH_SIZE) { |
1253 | if (use_poll) { |
1254 | ret = poll(&fds, 1, POLL_TMOUT); |
1255 | if (timeout) { |
1256 | if (ret < 0) { |
1257 | ksft_print_msg("ERROR: [%s] Poll error %d\n" , |
1258 | __func__, errno); |
1259 | return TEST_FAILURE; |
1260 | } |
1261 | if (ret == 0) |
1262 | return TEST_PASS; |
1263 | break; |
1264 | } |
1265 | if (ret <= 0) { |
1266 | ksft_print_msg("ERROR: [%s] Poll error %d\n" , |
1267 | __func__, errno); |
1268 | return TEST_FAILURE; |
1269 | } |
1270 | } |
1271 | |
1272 | complete_pkts(xsk, BATCH_SIZE); |
1273 | } |
1274 | |
1275 | for (i = 0; i < BATCH_SIZE; i++) { |
1276 | struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); |
1277 | u32 nb_frags_left, nb_frags, bytes_written = 0; |
1278 | |
1279 | if (!pkt) |
1280 | break; |
1281 | |
1282 | nb_frags = pkt_nb_frags(frame_size: umem->frame_size, pkt_stream, pkt); |
1283 | if (nb_frags > BATCH_SIZE - i) { |
1284 | pkt_stream_cancel(pkt_stream); |
1285 | xsk_ring_prod__cancel(prod: &xsk->tx, BATCH_SIZE - i); |
1286 | break; |
1287 | } |
1288 | nb_frags_left = nb_frags; |
1289 | |
1290 | while (nb_frags_left--) { |
1291 | struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(tx: &xsk->tx, idx: idx + i); |
1292 | |
1293 | tx_desc->addr = pkt_get_addr(pkt, umem: ifobject->umem); |
1294 | if (pkt_stream->verbatim) { |
1295 | tx_desc->len = pkt->len; |
1296 | tx_desc->options = pkt->options; |
1297 | } else if (nb_frags_left) { |
1298 | tx_desc->len = umem->frame_size; |
1299 | tx_desc->options = XDP_PKT_CONTD; |
1300 | } else { |
1301 | tx_desc->len = pkt->len - bytes_written; |
1302 | tx_desc->options = 0; |
1303 | } |
1304 | if (pkt->valid) |
1305 | pkt_generate(xsk, umem, addr: tx_desc->addr, len: tx_desc->len, pkt_nb: pkt->pkt_nb, |
1306 | bytes_written); |
1307 | bytes_written += tx_desc->len; |
1308 | |
1309 | print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n" , |
1310 | tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); |
1311 | |
1312 | if (nb_frags_left) { |
1313 | i++; |
1314 | if (pkt_stream->verbatim) |
1315 | pkt = pkt_stream_get_next_tx_pkt(pkt_stream); |
1316 | } |
1317 | } |
1318 | |
1319 | if (pkt && pkt->valid) { |
1320 | valid_pkts++; |
1321 | valid_frags += nb_frags; |
1322 | } |
1323 | } |
1324 | |
1325 | pthread_mutex_lock(&pacing_mutex); |
1326 | pkts_in_flight += valid_pkts; |
1327 | pthread_mutex_unlock(&pacing_mutex); |
1328 | |
1329 | xsk_ring_prod__submit(prod: &xsk->tx, nb: i); |
1330 | xsk->outstanding_tx += valid_frags; |
1331 | |
1332 | if (use_poll) { |
1333 | ret = poll(&fds, 1, POLL_TMOUT); |
1334 | if (ret <= 0) { |
1335 | if (ret == 0 && timeout) |
1336 | return TEST_PASS; |
1337 | |
1338 | ksft_print_msg(msg: "ERROR: [%s] Poll error %d\n" , __func__, ret); |
1339 | return TEST_FAILURE; |
1340 | } |
1341 | } |
1342 | |
1343 | if (!timeout) { |
1344 | if (complete_pkts(xsk, batch_size: i)) |
1345 | return TEST_FAILURE; |
1346 | |
1347 | usleep(10); |
1348 | return TEST_PASS; |
1349 | } |
1350 | |
1351 | return TEST_CONTINUE; |
1352 | } |
1353 | |
1354 | static int wait_for_tx_completion(struct xsk_socket_info *xsk) |
1355 | { |
1356 | struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; |
1357 | int ret; |
1358 | |
1359 | ret = gettimeofday(&tv_now, NULL); |
1360 | if (ret) |
1361 | exit_with_error(errno); |
1362 | timeradd(&tv_now, &tv_timeout, &tv_end); |
1363 | |
1364 | while (xsk->outstanding_tx) { |
1365 | ret = gettimeofday(&tv_now, NULL); |
1366 | if (ret) |
1367 | exit_with_error(errno); |
1368 | if (timercmp(&tv_now, &tv_end, >)) { |
1369 | ksft_print_msg(msg: "ERROR: [%s] Transmission loop timed out\n" , __func__); |
1370 | return TEST_FAILURE; |
1371 | } |
1372 | |
1373 | complete_pkts(xsk, BATCH_SIZE); |
1374 | } |
1375 | |
1376 | return TEST_PASS; |
1377 | } |
1378 | |
1379 | bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) |
1380 | { |
1381 | return bitmap_full(src: bitmap, nbits: test->nb_sockets); |
1382 | } |
1383 | |
1384 | static int send_pkts(struct test_spec *test, struct ifobject *ifobject) |
1385 | { |
1386 | bool timeout = !is_umem_valid(ifobj: test->ifobj_rx); |
1387 | DECLARE_BITMAP(bitmap, test->nb_sockets); |
1388 | u32 i, ret; |
1389 | |
1390 | while (!(all_packets_sent(test, bitmap))) { |
1391 | for (i = 0; i < test->nb_sockets; i++) { |
1392 | struct pkt_stream *pkt_stream; |
1393 | |
1394 | pkt_stream = ifobject->xsk_arr[i].pkt_stream; |
1395 | if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { |
1396 | __set_bit(i, bitmap); |
1397 | continue; |
1398 | } |
1399 | ret = __send_pkts(ifobject, xsk: &ifobject->xsk_arr[i], timeout); |
1400 | if (ret == TEST_CONTINUE && !test->fail) |
1401 | continue; |
1402 | |
1403 | if ((ret || test->fail) && !timeout) |
1404 | return TEST_FAILURE; |
1405 | |
1406 | if (ret == TEST_PASS && timeout) |
1407 | return ret; |
1408 | |
1409 | ret = wait_for_tx_completion(xsk: &ifobject->xsk_arr[i]); |
1410 | if (ret) |
1411 | return TEST_FAILURE; |
1412 | } |
1413 | } |
1414 | |
1415 | return TEST_PASS; |
1416 | } |
1417 | |
1418 | static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) |
1419 | { |
1420 | int fd = xsk_socket__fd(xsk), err; |
1421 | socklen_t optlen, expected_len; |
1422 | |
1423 | optlen = sizeof(*stats); |
1424 | err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); |
1425 | if (err) { |
1426 | ksft_print_msg(msg: "[%s] getsockopt(XDP_STATISTICS) error %u %s\n" , |
1427 | __func__, -err, strerror(-err)); |
1428 | return TEST_FAILURE; |
1429 | } |
1430 | |
1431 | expected_len = sizeof(struct xdp_statistics); |
1432 | if (optlen != expected_len) { |
1433 | ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n" , |
1434 | __func__, expected_len, optlen); |
1435 | return TEST_FAILURE; |
1436 | } |
1437 | |
1438 | return TEST_PASS; |
1439 | } |
1440 | |
1441 | static int validate_rx_dropped(struct ifobject *ifobject) |
1442 | { |
1443 | struct xsk_socket *xsk = ifobject->xsk->xsk; |
1444 | struct xdp_statistics stats; |
1445 | int err; |
1446 | |
1447 | err = kick_rx(xsk: ifobject->xsk); |
1448 | if (err) |
1449 | return TEST_FAILURE; |
1450 | |
1451 | err = get_xsk_stats(xsk, stats: &stats); |
1452 | if (err) |
1453 | return TEST_FAILURE; |
1454 | |
1455 | /* The receiver calls getsockopt after receiving the last (valid) |
1456 | * packet which is not the final packet sent in this test (valid and |
1457 | * invalid packets are sent in alternating fashion with the final |
1458 | * packet being invalid). Since the last packet may or may not have |
1459 | * been dropped already, both outcomes must be allowed. |
1460 | */ |
1461 | if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || |
1462 | stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) |
1463 | return TEST_PASS; |
1464 | |
1465 | return TEST_FAILURE; |
1466 | } |
1467 | |
1468 | static int validate_rx_full(struct ifobject *ifobject) |
1469 | { |
1470 | struct xsk_socket *xsk = ifobject->xsk->xsk; |
1471 | struct xdp_statistics stats; |
1472 | int err; |
1473 | |
1474 | usleep(1000); |
1475 | err = kick_rx(xsk: ifobject->xsk); |
1476 | if (err) |
1477 | return TEST_FAILURE; |
1478 | |
1479 | err = get_xsk_stats(xsk, stats: &stats); |
1480 | if (err) |
1481 | return TEST_FAILURE; |
1482 | |
1483 | if (stats.rx_ring_full) |
1484 | return TEST_PASS; |
1485 | |
1486 | return TEST_FAILURE; |
1487 | } |
1488 | |
1489 | static int validate_fill_empty(struct ifobject *ifobject) |
1490 | { |
1491 | struct xsk_socket *xsk = ifobject->xsk->xsk; |
1492 | struct xdp_statistics stats; |
1493 | int err; |
1494 | |
1495 | usleep(1000); |
1496 | err = kick_rx(xsk: ifobject->xsk); |
1497 | if (err) |
1498 | return TEST_FAILURE; |
1499 | |
1500 | err = get_xsk_stats(xsk, stats: &stats); |
1501 | if (err) |
1502 | return TEST_FAILURE; |
1503 | |
1504 | if (stats.rx_fill_ring_empty_descs) |
1505 | return TEST_PASS; |
1506 | |
1507 | return TEST_FAILURE; |
1508 | } |
1509 | |
1510 | static int validate_tx_invalid_descs(struct ifobject *ifobject) |
1511 | { |
1512 | struct xsk_socket *xsk = ifobject->xsk->xsk; |
1513 | int fd = xsk_socket__fd(xsk); |
1514 | struct xdp_statistics stats; |
1515 | socklen_t optlen; |
1516 | int err; |
1517 | |
1518 | optlen = sizeof(stats); |
1519 | err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); |
1520 | if (err) { |
1521 | ksft_print_msg(msg: "[%s] getsockopt(XDP_STATISTICS) error %u %s\n" , |
1522 | __func__, -err, strerror(-err)); |
1523 | return TEST_FAILURE; |
1524 | } |
1525 | |
1526 | if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { |
1527 | ksft_print_msg(msg: "[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n" , |
1528 | __func__, |
1529 | (unsigned long long)stats.tx_invalid_descs, |
1530 | ifobject->xsk->pkt_stream->nb_pkts); |
1531 | return TEST_FAILURE; |
1532 | } |
1533 | |
1534 | return TEST_PASS; |
1535 | } |
1536 | |
1537 | static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, |
1538 | struct xsk_umem_info *umem, bool tx) |
1539 | { |
1540 | int i, ret; |
1541 | |
1542 | for (i = 0; i < test->nb_sockets; i++) { |
1543 | bool shared = (ifobject->shared_umem && tx) ? true : !!i; |
1544 | u32 ctr = 0; |
1545 | |
1546 | while (ctr++ < SOCK_RECONF_CTR) { |
1547 | ret = __xsk_configure_socket(xsk: &ifobject->xsk_arr[i], umem, |
1548 | ifobject, shared); |
1549 | if (!ret) |
1550 | break; |
1551 | |
1552 | /* Retry if it fails as xsk_socket__create() is asynchronous */ |
1553 | if (ctr >= SOCK_RECONF_CTR) |
1554 | exit_with_error(-ret); |
1555 | usleep(USLEEP_MAX); |
1556 | } |
1557 | if (ifobject->busy_poll) |
1558 | enable_busy_poll(xsk: &ifobject->xsk_arr[i]); |
1559 | } |
1560 | } |
1561 | |
1562 | static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) |
1563 | { |
1564 | xsk_configure_socket(test, ifobject, umem: test->ifobj_rx->umem, tx: true); |
1565 | ifobject->xsk = &ifobject->xsk_arr[0]; |
1566 | ifobject->xskmap = test->ifobj_rx->xskmap; |
1567 | memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); |
1568 | ifobject->umem->base_addr = 0; |
1569 | } |
1570 | |
1571 | static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, |
1572 | bool fill_up) |
1573 | { |
1574 | u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; |
1575 | u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; |
1576 | int ret; |
1577 | |
1578 | if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) |
1579 | buffers_to_fill = umem->num_frames; |
1580 | else |
1581 | buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS; |
1582 | |
1583 | ret = xsk_ring_prod__reserve(prod: &umem->fq, nb: buffers_to_fill, idx: &idx); |
1584 | if (ret != buffers_to_fill) |
1585 | exit_with_error(ENOSPC); |
1586 | |
1587 | while (filled < buffers_to_fill) { |
1588 | struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, pkts_sent: &nb_pkts); |
1589 | u64 addr; |
1590 | u32 i; |
1591 | |
1592 | for (i = 0; i < pkt_nb_frags(frame_size: rx_frame_size, pkt_stream, pkt); i++) { |
1593 | if (!pkt) { |
1594 | if (!fill_up) |
1595 | break; |
1596 | addr = filled * umem->frame_size + umem->base_addr; |
1597 | } else if (pkt->offset >= 0) { |
1598 | addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); |
1599 | } else { |
1600 | addr = pkt->offset + umem_alloc_buffer(umem); |
1601 | } |
1602 | |
1603 | *xsk_ring_prod__fill_addr(fill: &umem->fq, idx: idx++) = addr; |
1604 | if (++filled >= buffers_to_fill) |
1605 | break; |
1606 | } |
1607 | } |
1608 | xsk_ring_prod__submit(prod: &umem->fq, nb: filled); |
1609 | xsk_ring_prod__cancel(prod: &umem->fq, nb: buffers_to_fill - filled); |
1610 | |
1611 | pkt_stream_reset(pkt_stream); |
1612 | umem_reset_alloc(umem); |
1613 | } |
1614 | |
1615 | static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) |
1616 | { |
1617 | u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; |
1618 | int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; |
1619 | LIBBPF_OPTS(bpf_xdp_query_opts, opts); |
1620 | void *bufs; |
1621 | int ret; |
1622 | u32 i; |
1623 | |
1624 | if (ifobject->umem->unaligned_mode) |
1625 | mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; |
1626 | |
1627 | if (ifobject->shared_umem) |
1628 | umem_sz *= 2; |
1629 | |
1630 | bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); |
1631 | if (bufs == MAP_FAILED) |
1632 | exit_with_error(errno); |
1633 | |
1634 | ret = xsk_configure_umem(ifobj: ifobject, umem: ifobject->umem, buffer: bufs, size: umem_sz); |
1635 | if (ret) |
1636 | exit_with_error(-ret); |
1637 | |
1638 | xsk_configure_socket(test, ifobject, umem: ifobject->umem, tx: false); |
1639 | |
1640 | ifobject->xsk = &ifobject->xsk_arr[0]; |
1641 | |
1642 | if (!ifobject->rx_on) |
1643 | return; |
1644 | |
1645 | xsk_populate_fill_ring(umem: ifobject->umem, pkt_stream: ifobject->xsk->pkt_stream, fill_up: ifobject->use_fill_ring); |
1646 | |
1647 | for (i = 0; i < test->nb_sockets; i++) { |
1648 | ifobject->xsk = &ifobject->xsk_arr[i]; |
1649 | ret = xsk_update_xskmap(map: ifobject->xskmap, xsk: ifobject->xsk->xsk, index: i); |
1650 | if (ret) |
1651 | exit_with_error(errno); |
1652 | } |
1653 | } |
1654 | |
1655 | static void *worker_testapp_validate_tx(void *arg) |
1656 | { |
1657 | struct test_spec *test = (struct test_spec *)arg; |
1658 | struct ifobject *ifobject = test->ifobj_tx; |
1659 | int err; |
1660 | |
1661 | if (test->current_step == 1) { |
1662 | if (!ifobject->shared_umem) |
1663 | thread_common_ops(test, ifobject); |
1664 | else |
1665 | thread_common_ops_tx(test, ifobject); |
1666 | } |
1667 | |
1668 | err = send_pkts(test, ifobject); |
1669 | |
1670 | if (!err && ifobject->validation_func) |
1671 | err = ifobject->validation_func(ifobject); |
1672 | if (err) |
1673 | report_failure(test); |
1674 | |
1675 | pthread_exit(NULL); |
1676 | } |
1677 | |
1678 | static void *worker_testapp_validate_rx(void *arg) |
1679 | { |
1680 | struct test_spec *test = (struct test_spec *)arg; |
1681 | struct ifobject *ifobject = test->ifobj_rx; |
1682 | int err; |
1683 | |
1684 | if (test->current_step == 1) { |
1685 | thread_common_ops(test, ifobject); |
1686 | } else { |
1687 | xsk_clear_xskmap(map: ifobject->xskmap); |
1688 | err = xsk_update_xskmap(map: ifobject->xskmap, xsk: ifobject->xsk->xsk, index: 0); |
1689 | if (err) { |
1690 | ksft_print_msg(msg: "Error: Failed to update xskmap, error %s\n" , |
1691 | strerror(-err)); |
1692 | exit_with_error(-err); |
1693 | } |
1694 | } |
1695 | |
1696 | pthread_barrier_wait(&barr); |
1697 | |
1698 | err = receive_pkts(test); |
1699 | |
1700 | if (!err && ifobject->validation_func) |
1701 | err = ifobject->validation_func(ifobject); |
1702 | if (err) |
1703 | report_failure(test); |
1704 | |
1705 | pthread_exit(NULL); |
1706 | } |
1707 | |
1708 | static u64 ceil_u64(u64 a, u64 b) |
1709 | { |
1710 | return (a + b - 1) / b; |
1711 | } |
1712 | |
1713 | static void testapp_clean_xsk_umem(struct ifobject *ifobj) |
1714 | { |
1715 | u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; |
1716 | |
1717 | if (ifobj->shared_umem) |
1718 | umem_sz *= 2; |
1719 | |
1720 | umem_sz = ceil_u64(a: umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; |
1721 | xsk_umem__delete(umem: ifobj->umem->umem); |
1722 | munmap(ifobj->umem->buffer, umem_sz); |
1723 | } |
1724 | |
1725 | static void handler(int signum) |
1726 | { |
1727 | pthread_exit(NULL); |
1728 | } |
1729 | |
1730 | static bool xdp_prog_changed_rx(struct test_spec *test) |
1731 | { |
1732 | struct ifobject *ifobj = test->ifobj_rx; |
1733 | |
1734 | return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; |
1735 | } |
1736 | |
1737 | static bool xdp_prog_changed_tx(struct test_spec *test) |
1738 | { |
1739 | struct ifobject *ifobj = test->ifobj_tx; |
1740 | |
1741 | return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; |
1742 | } |
1743 | |
1744 | static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, |
1745 | struct bpf_map *xskmap, enum test_mode mode) |
1746 | { |
1747 | int err; |
1748 | |
1749 | xsk_detach_xdp_program(ifindex: ifobj->ifindex, xdp_flags: mode_to_xdp_flags(mode: ifobj->mode)); |
1750 | err = xsk_attach_xdp_program(prog: xdp_prog, ifindex: ifobj->ifindex, xdp_flags: mode_to_xdp_flags(mode)); |
1751 | if (err) { |
1752 | ksft_print_msg(msg: "Error attaching XDP program\n" ); |
1753 | exit_with_error(-err); |
1754 | } |
1755 | |
1756 | if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) |
1757 | if (!xsk_is_in_mode(ifindex: ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { |
1758 | ksft_print_msg(msg: "ERROR: XDP prog not in DRV mode\n" ); |
1759 | exit_with_error(EINVAL); |
1760 | } |
1761 | |
1762 | ifobj->xdp_prog = xdp_prog; |
1763 | ifobj->xskmap = xskmap; |
1764 | ifobj->mode = mode; |
1765 | } |
1766 | |
1767 | static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, |
1768 | struct ifobject *ifobj_tx) |
1769 | { |
1770 | if (xdp_prog_changed_rx(test)) |
1771 | xsk_reattach_xdp(ifobj: ifobj_rx, xdp_prog: test->xdp_prog_rx, xskmap: test->xskmap_rx, mode: test->mode); |
1772 | |
1773 | if (!ifobj_tx || ifobj_tx->shared_umem) |
1774 | return; |
1775 | |
1776 | if (xdp_prog_changed_tx(test)) |
1777 | xsk_reattach_xdp(ifobj: ifobj_tx, xdp_prog: test->xdp_prog_tx, xskmap: test->xskmap_tx, mode: test->mode); |
1778 | } |
1779 | |
1780 | static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, |
1781 | struct ifobject *ifobj2) |
1782 | { |
1783 | pthread_t t0, t1; |
1784 | int err; |
1785 | |
1786 | if (test->mtu > MAX_ETH_PKT_SIZE) { |
1787 | if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || |
1788 | (ifobj2 && !ifobj2->multi_buff_zc_supp))) { |
1789 | ksft_test_result_skip(msg: "Multi buffer for zero-copy not supported.\n" ); |
1790 | return TEST_SKIP; |
1791 | } |
1792 | if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || |
1793 | (ifobj2 && !ifobj2->multi_buff_supp))) { |
1794 | ksft_test_result_skip(msg: "Multi buffer not supported.\n" ); |
1795 | return TEST_SKIP; |
1796 | } |
1797 | } |
1798 | err = test_spec_set_mtu(test, mtu: test->mtu); |
1799 | if (err) { |
1800 | ksft_print_msg(msg: "Error, could not set mtu.\n" ); |
1801 | exit_with_error(err); |
1802 | } |
1803 | |
1804 | if (ifobj2) { |
1805 | if (pthread_barrier_init(&barr, NULL, 2)) |
1806 | exit_with_error(errno); |
1807 | pkt_stream_reset(pkt_stream: ifobj2->xsk->pkt_stream); |
1808 | } |
1809 | |
1810 | test->current_step++; |
1811 | pkt_stream_reset(pkt_stream: ifobj1->xsk->pkt_stream); |
1812 | pkts_in_flight = 0; |
1813 | |
1814 | signal(SIGUSR1, handler); |
1815 | /*Spawn RX thread */ |
1816 | pthread_create(&t0, NULL, ifobj1->func_ptr, test); |
1817 | |
1818 | if (ifobj2) { |
1819 | pthread_barrier_wait(&barr); |
1820 | if (pthread_barrier_destroy(&barr)) |
1821 | exit_with_error(errno); |
1822 | |
1823 | /*Spawn TX thread */ |
1824 | pthread_create(&t1, NULL, ifobj2->func_ptr, test); |
1825 | |
1826 | pthread_join(t1, NULL); |
1827 | } |
1828 | |
1829 | if (!ifobj2) |
1830 | pthread_kill(t0, SIGUSR1); |
1831 | else |
1832 | pthread_join(t0, NULL); |
1833 | |
1834 | if (test->total_steps == test->current_step || test->fail) { |
1835 | u32 i; |
1836 | |
1837 | if (ifobj2) |
1838 | for (i = 0; i < test->nb_sockets; i++) |
1839 | xsk_socket__delete(xsk: ifobj2->xsk_arr[i].xsk); |
1840 | |
1841 | for (i = 0; i < test->nb_sockets; i++) |
1842 | xsk_socket__delete(xsk: ifobj1->xsk_arr[i].xsk); |
1843 | |
1844 | testapp_clean_xsk_umem(ifobj: ifobj1); |
1845 | if (ifobj2 && !ifobj2->shared_umem) |
1846 | testapp_clean_xsk_umem(ifobj: ifobj2); |
1847 | } |
1848 | |
1849 | return !!test->fail; |
1850 | } |
1851 | |
1852 | static int testapp_validate_traffic(struct test_spec *test) |
1853 | { |
1854 | struct ifobject *ifobj_rx = test->ifobj_rx; |
1855 | struct ifobject *ifobj_tx = test->ifobj_tx; |
1856 | |
1857 | if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || |
1858 | (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { |
1859 | ksft_test_result_skip(msg: "No huge pages present.\n" ); |
1860 | return TEST_SKIP; |
1861 | } |
1862 | |
1863 | xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); |
1864 | return __testapp_validate_traffic(test, ifobj1: ifobj_rx, ifobj2: ifobj_tx); |
1865 | } |
1866 | |
1867 | static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) |
1868 | { |
1869 | return __testapp_validate_traffic(test, ifobj1: ifobj, NULL); |
1870 | } |
1871 | |
1872 | static int testapp_teardown(struct test_spec *test) |
1873 | { |
1874 | int i; |
1875 | |
1876 | for (i = 0; i < MAX_TEARDOWN_ITER; i++) { |
1877 | if (testapp_validate_traffic(test)) |
1878 | return TEST_FAILURE; |
1879 | test_spec_reset(test); |
1880 | } |
1881 | |
1882 | return TEST_PASS; |
1883 | } |
1884 | |
1885 | static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) |
1886 | { |
1887 | thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; |
1888 | struct ifobject *tmp_ifobj = (*ifobj1); |
1889 | |
1890 | (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; |
1891 | (*ifobj2)->func_ptr = tmp_func_ptr; |
1892 | |
1893 | *ifobj1 = *ifobj2; |
1894 | *ifobj2 = tmp_ifobj; |
1895 | } |
1896 | |
1897 | static int testapp_bidirectional(struct test_spec *test) |
1898 | { |
1899 | int res; |
1900 | |
1901 | test->ifobj_tx->rx_on = true; |
1902 | test->ifobj_rx->tx_on = true; |
1903 | test->total_steps = 2; |
1904 | if (testapp_validate_traffic(test)) |
1905 | return TEST_FAILURE; |
1906 | |
1907 | print_verbose("Switching Tx/Rx direction\n" ); |
1908 | swap_directions(ifobj1: &test->ifobj_rx, ifobj2: &test->ifobj_tx); |
1909 | res = __testapp_validate_traffic(test, ifobj1: test->ifobj_rx, ifobj2: test->ifobj_tx); |
1910 | |
1911 | swap_directions(ifobj1: &test->ifobj_rx, ifobj2: &test->ifobj_tx); |
1912 | return res; |
1913 | } |
1914 | |
1915 | static int swap_xsk_resources(struct test_spec *test) |
1916 | { |
1917 | int ret; |
1918 | |
1919 | test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; |
1920 | test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; |
1921 | test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; |
1922 | test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; |
1923 | test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; |
1924 | test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; |
1925 | |
1926 | ret = xsk_update_xskmap(map: test->ifobj_rx->xskmap, xsk: test->ifobj_rx->xsk->xsk, index: 0); |
1927 | if (ret) |
1928 | return TEST_FAILURE; |
1929 | |
1930 | return TEST_PASS; |
1931 | } |
1932 | |
1933 | static int testapp_xdp_prog_cleanup(struct test_spec *test) |
1934 | { |
1935 | test->total_steps = 2; |
1936 | test->nb_sockets = 2; |
1937 | if (testapp_validate_traffic(test)) |
1938 | return TEST_FAILURE; |
1939 | |
1940 | if (swap_xsk_resources(test)) |
1941 | return TEST_FAILURE; |
1942 | return testapp_validate_traffic(test); |
1943 | } |
1944 | |
1945 | static int testapp_headroom(struct test_spec *test) |
1946 | { |
1947 | test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; |
1948 | return testapp_validate_traffic(test); |
1949 | } |
1950 | |
1951 | static int testapp_stats_rx_dropped(struct test_spec *test) |
1952 | { |
1953 | if (test->mode == TEST_MODE_ZC) { |
1954 | ksft_test_result_skip(msg: "Can not run RX_DROPPED test for ZC mode\n" ); |
1955 | return TEST_SKIP; |
1956 | } |
1957 | |
1958 | pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, offset: 0); |
1959 | test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - |
1960 | XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; |
1961 | pkt_stream_receive_half(test); |
1962 | test->ifobj_rx->validation_func = validate_rx_dropped; |
1963 | return testapp_validate_traffic(test); |
1964 | } |
1965 | |
1966 | static int testapp_stats_tx_invalid_descs(struct test_spec *test) |
1967 | { |
1968 | pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, offset: 0); |
1969 | test->ifobj_tx->validation_func = validate_tx_invalid_descs; |
1970 | return testapp_validate_traffic(test); |
1971 | } |
1972 | |
1973 | static int testapp_stats_rx_full(struct test_spec *test) |
1974 | { |
1975 | pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); |
1976 | test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); |
1977 | |
1978 | test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; |
1979 | test->ifobj_rx->release_rx = false; |
1980 | test->ifobj_rx->validation_func = validate_rx_full; |
1981 | return testapp_validate_traffic(test); |
1982 | } |
1983 | |
1984 | static int testapp_stats_fill_empty(struct test_spec *test) |
1985 | { |
1986 | pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); |
1987 | test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); |
1988 | |
1989 | test->ifobj_rx->use_fill_ring = false; |
1990 | test->ifobj_rx->validation_func = validate_fill_empty; |
1991 | return testapp_validate_traffic(test); |
1992 | } |
1993 | |
1994 | static int testapp_send_receive_unaligned(struct test_spec *test) |
1995 | { |
1996 | test->ifobj_tx->umem->unaligned_mode = true; |
1997 | test->ifobj_rx->umem->unaligned_mode = true; |
1998 | /* Let half of the packets straddle a 4K buffer boundary */ |
1999 | pkt_stream_replace_half(test, MIN_PKT_SIZE, offset: -MIN_PKT_SIZE / 2); |
2000 | |
2001 | return testapp_validate_traffic(test); |
2002 | } |
2003 | |
2004 | static int testapp_send_receive_unaligned_mb(struct test_spec *test) |
2005 | { |
2006 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2007 | test->ifobj_tx->umem->unaligned_mode = true; |
2008 | test->ifobj_rx->umem->unaligned_mode = true; |
2009 | pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); |
2010 | return testapp_validate_traffic(test); |
2011 | } |
2012 | |
2013 | static int testapp_single_pkt(struct test_spec *test) |
2014 | { |
2015 | struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; |
2016 | |
2017 | pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); |
2018 | return testapp_validate_traffic(test); |
2019 | } |
2020 | |
2021 | static int testapp_send_receive_mb(struct test_spec *test) |
2022 | { |
2023 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2024 | pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); |
2025 | |
2026 | return testapp_validate_traffic(test); |
2027 | } |
2028 | |
2029 | static int testapp_invalid_desc_mb(struct test_spec *test) |
2030 | { |
2031 | struct xsk_umem_info *umem = test->ifobj_tx->umem; |
2032 | u64 umem_size = umem->num_frames * umem->frame_size; |
2033 | struct pkt pkts[] = { |
2034 | /* Valid packet for synch to start with */ |
2035 | {0, MIN_PKT_SIZE, 0, true, 0}, |
2036 | /* Zero frame len is not legal */ |
2037 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2038 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2039 | {0, 0, 0, false, 0}, |
2040 | /* Invalid address in the second frame */ |
2041 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2042 | {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2043 | /* Invalid len in the middle */ |
2044 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2045 | {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2046 | /* Invalid options in the middle */ |
2047 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2048 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, |
2049 | /* Transmit 2 frags, receive 3 */ |
2050 | {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, |
2051 | {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, |
2052 | /* Middle frame crosses chunk boundary with small length */ |
2053 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, |
2054 | {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, |
2055 | /* Valid packet for synch so that something is received */ |
2056 | {0, MIN_PKT_SIZE, 0, true, 0}}; |
2057 | |
2058 | if (umem->unaligned_mode) { |
2059 | /* Crossing a chunk boundary allowed */ |
2060 | pkts[12].valid = true; |
2061 | pkts[13].valid = true; |
2062 | } |
2063 | |
2064 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2065 | pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); |
2066 | return testapp_validate_traffic(test); |
2067 | } |
2068 | |
2069 | static int testapp_invalid_desc(struct test_spec *test) |
2070 | { |
2071 | struct xsk_umem_info *umem = test->ifobj_tx->umem; |
2072 | u64 umem_size = umem->num_frames * umem->frame_size; |
2073 | struct pkt pkts[] = { |
2074 | /* Zero packet address allowed */ |
2075 | {0, MIN_PKT_SIZE, 0, true}, |
2076 | /* Allowed packet */ |
2077 | {0, MIN_PKT_SIZE, 0, true}, |
2078 | /* Straddling the start of umem */ |
2079 | {-2, MIN_PKT_SIZE, 0, false}, |
2080 | /* Packet too large */ |
2081 | {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, |
2082 | /* Up to end of umem allowed */ |
2083 | {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, |
2084 | /* After umem ends */ |
2085 | {umem_size, MIN_PKT_SIZE, 0, false}, |
2086 | /* Straddle the end of umem */ |
2087 | {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, |
2088 | /* Straddle a 4K boundary */ |
2089 | {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, |
2090 | /* Straddle a 2K boundary */ |
2091 | {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, |
2092 | /* Valid packet for synch so that something is received */ |
2093 | {0, MIN_PKT_SIZE, 0, true}}; |
2094 | |
2095 | if (umem->unaligned_mode) { |
2096 | /* Crossing a page boundary allowed */ |
2097 | pkts[7].valid = true; |
2098 | } |
2099 | if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { |
2100 | /* Crossing a 2K frame size boundary not allowed */ |
2101 | pkts[8].valid = false; |
2102 | } |
2103 | |
2104 | if (test->ifobj_tx->shared_umem) { |
2105 | pkts[4].offset += umem_size; |
2106 | pkts[5].offset += umem_size; |
2107 | pkts[6].offset += umem_size; |
2108 | } |
2109 | |
2110 | pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); |
2111 | return testapp_validate_traffic(test); |
2112 | } |
2113 | |
2114 | static int testapp_xdp_drop(struct test_spec *test) |
2115 | { |
2116 | struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; |
2117 | struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; |
2118 | |
2119 | test_spec_set_xdp_prog(test, xdp_prog_rx: skel_rx->progs.xsk_xdp_drop, xdp_prog_tx: skel_tx->progs.xsk_xdp_drop, |
2120 | xskmap_rx: skel_rx->maps.xsk, xskmap_tx: skel_tx->maps.xsk); |
2121 | |
2122 | pkt_stream_receive_half(test); |
2123 | return testapp_validate_traffic(test); |
2124 | } |
2125 | |
2126 | static int testapp_xdp_metadata_copy(struct test_spec *test) |
2127 | { |
2128 | struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; |
2129 | struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; |
2130 | struct bpf_map *data_map; |
2131 | int count = 0; |
2132 | int key = 0; |
2133 | |
2134 | test_spec_set_xdp_prog(test, xdp_prog_rx: skel_rx->progs.xsk_xdp_populate_metadata, |
2135 | xdp_prog_tx: skel_tx->progs.xsk_xdp_populate_metadata, |
2136 | xskmap_rx: skel_rx->maps.xsk, xskmap_tx: skel_tx->maps.xsk); |
2137 | test->ifobj_rx->use_metadata = true; |
2138 | |
2139 | data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss" ); |
2140 | if (!data_map || !bpf_map__is_internal(data_map)) { |
2141 | ksft_print_msg(msg: "Error: could not find bss section of XDP program\n" ); |
2142 | return TEST_FAILURE; |
2143 | } |
2144 | |
2145 | if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) { |
2146 | ksft_print_msg(msg: "Error: could not update count element\n" ); |
2147 | return TEST_FAILURE; |
2148 | } |
2149 | |
2150 | return testapp_validate_traffic(test); |
2151 | } |
2152 | |
2153 | static int testapp_xdp_shared_umem(struct test_spec *test) |
2154 | { |
2155 | struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; |
2156 | struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; |
2157 | |
2158 | test->total_steps = 1; |
2159 | test->nb_sockets = 2; |
2160 | |
2161 | test_spec_set_xdp_prog(test, xdp_prog_rx: skel_rx->progs.xsk_xdp_shared_umem, |
2162 | xdp_prog_tx: skel_tx->progs.xsk_xdp_shared_umem, |
2163 | xskmap_rx: skel_rx->maps.xsk, xskmap_tx: skel_tx->maps.xsk); |
2164 | |
2165 | pkt_stream_even_odd_sequence(test); |
2166 | |
2167 | return testapp_validate_traffic(test); |
2168 | } |
2169 | |
2170 | static int testapp_poll_txq_tmout(struct test_spec *test) |
2171 | { |
2172 | test->ifobj_tx->use_poll = true; |
2173 | /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ |
2174 | test->ifobj_tx->umem->frame_size = 2048; |
2175 | pkt_stream_replace(test, nb_pkts: 2 * DEFAULT_PKT_CNT, pkt_len: 2048); |
2176 | return testapp_validate_traffic_single_thread(test, ifobj: test->ifobj_tx); |
2177 | } |
2178 | |
2179 | static int testapp_poll_rxq_tmout(struct test_spec *test) |
2180 | { |
2181 | test->ifobj_rx->use_poll = true; |
2182 | return testapp_validate_traffic_single_thread(test, ifobj: test->ifobj_rx); |
2183 | } |
2184 | |
2185 | static int testapp_too_many_frags(struct test_spec *test) |
2186 | { |
2187 | struct pkt pkts[2 * XSK_DESC__MAX_SKB_FRAGS + 2] = {}; |
2188 | u32 max_frags, i; |
2189 | |
2190 | if (test->mode == TEST_MODE_ZC) |
2191 | max_frags = test->ifobj_tx->xdp_zc_max_segs; |
2192 | else |
2193 | max_frags = XSK_DESC__MAX_SKB_FRAGS; |
2194 | |
2195 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2196 | |
2197 | /* Valid packet for synch */ |
2198 | pkts[0].len = MIN_PKT_SIZE; |
2199 | pkts[0].valid = true; |
2200 | |
2201 | /* One valid packet with the max amount of frags */ |
2202 | for (i = 1; i < max_frags + 1; i++) { |
2203 | pkts[i].len = MIN_PKT_SIZE; |
2204 | pkts[i].options = XDP_PKT_CONTD; |
2205 | pkts[i].valid = true; |
2206 | } |
2207 | pkts[max_frags].options = 0; |
2208 | |
2209 | /* An invalid packet with the max amount of frags but signals packet |
2210 | * continues on the last frag |
2211 | */ |
2212 | for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { |
2213 | pkts[i].len = MIN_PKT_SIZE; |
2214 | pkts[i].options = XDP_PKT_CONTD; |
2215 | pkts[i].valid = false; |
2216 | } |
2217 | |
2218 | /* Valid packet for synch */ |
2219 | pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; |
2220 | pkts[2 * max_frags + 1].valid = true; |
2221 | |
2222 | pkt_stream_generate_custom(test, pkts, nb_pkts: 2 * max_frags + 2); |
2223 | return testapp_validate_traffic(test); |
2224 | } |
2225 | |
2226 | static int xsk_load_xdp_programs(struct ifobject *ifobj) |
2227 | { |
2228 | ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); |
2229 | if (libbpf_get_error(ifobj->xdp_progs)) |
2230 | return libbpf_get_error(ifobj->xdp_progs); |
2231 | |
2232 | return 0; |
2233 | } |
2234 | |
2235 | static void xsk_unload_xdp_programs(struct ifobject *ifobj) |
2236 | { |
2237 | xsk_xdp_progs__destroy(ifobj->xdp_progs); |
2238 | } |
2239 | |
2240 | /* Simple test */ |
2241 | static bool hugepages_present(void) |
2242 | { |
2243 | size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; |
2244 | void *bufs; |
2245 | |
2246 | bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, |
2247 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); |
2248 | if (bufs == MAP_FAILED) |
2249 | return false; |
2250 | |
2251 | mmap_sz = ceil_u64(a: mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; |
2252 | munmap(bufs, mmap_sz); |
2253 | return true; |
2254 | } |
2255 | |
2256 | static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) |
2257 | { |
2258 | LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); |
2259 | int err; |
2260 | |
2261 | ifobj->func_ptr = func_ptr; |
2262 | |
2263 | err = xsk_load_xdp_programs(ifobj); |
2264 | if (err) { |
2265 | ksft_print_msg(msg: "Error loading XDP program\n" ); |
2266 | exit_with_error(err); |
2267 | } |
2268 | |
2269 | if (hugepages_present()) |
2270 | ifobj->unaligned_supp = true; |
2271 | |
2272 | err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); |
2273 | if (err) { |
2274 | ksft_print_msg(msg: "Error querying XDP capabilities\n" ); |
2275 | exit_with_error(-err); |
2276 | } |
2277 | if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) |
2278 | ifobj->multi_buff_supp = true; |
2279 | if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { |
2280 | if (query_opts.xdp_zc_max_segs > 1) { |
2281 | ifobj->multi_buff_zc_supp = true; |
2282 | ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; |
2283 | } else { |
2284 | ifobj->xdp_zc_max_segs = 0; |
2285 | } |
2286 | } |
2287 | } |
2288 | |
2289 | static int testapp_send_receive(struct test_spec *test) |
2290 | { |
2291 | return testapp_validate_traffic(test); |
2292 | } |
2293 | |
2294 | static int testapp_send_receive_2k_frame(struct test_spec *test) |
2295 | { |
2296 | test->ifobj_tx->umem->frame_size = 2048; |
2297 | test->ifobj_rx->umem->frame_size = 2048; |
2298 | pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); |
2299 | return testapp_validate_traffic(test); |
2300 | } |
2301 | |
2302 | static int testapp_poll_rx(struct test_spec *test) |
2303 | { |
2304 | test->ifobj_rx->use_poll = true; |
2305 | return testapp_validate_traffic(test); |
2306 | } |
2307 | |
2308 | static int testapp_poll_tx(struct test_spec *test) |
2309 | { |
2310 | test->ifobj_tx->use_poll = true; |
2311 | return testapp_validate_traffic(test); |
2312 | } |
2313 | |
2314 | static int testapp_aligned_inv_desc(struct test_spec *test) |
2315 | { |
2316 | return testapp_invalid_desc(test); |
2317 | } |
2318 | |
2319 | static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) |
2320 | { |
2321 | test->ifobj_tx->umem->frame_size = 2048; |
2322 | test->ifobj_rx->umem->frame_size = 2048; |
2323 | return testapp_invalid_desc(test); |
2324 | } |
2325 | |
2326 | static int testapp_unaligned_inv_desc(struct test_spec *test) |
2327 | { |
2328 | test->ifobj_tx->umem->unaligned_mode = true; |
2329 | test->ifobj_rx->umem->unaligned_mode = true; |
2330 | return testapp_invalid_desc(test); |
2331 | } |
2332 | |
2333 | static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) |
2334 | { |
2335 | u64 page_size, umem_size; |
2336 | |
2337 | /* Odd frame size so the UMEM doesn't end near a page boundary. */ |
2338 | test->ifobj_tx->umem->frame_size = 4001; |
2339 | test->ifobj_rx->umem->frame_size = 4001; |
2340 | test->ifobj_tx->umem->unaligned_mode = true; |
2341 | test->ifobj_rx->umem->unaligned_mode = true; |
2342 | /* This test exists to test descriptors that staddle the end of |
2343 | * the UMEM but not a page. |
2344 | */ |
2345 | page_size = sysconf(_SC_PAGESIZE); |
2346 | umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; |
2347 | assert(umem_size % page_size > MIN_PKT_SIZE); |
2348 | assert(umem_size % page_size < page_size - MIN_PKT_SIZE); |
2349 | |
2350 | return testapp_invalid_desc(test); |
2351 | } |
2352 | |
2353 | static int testapp_aligned_inv_desc_mb(struct test_spec *test) |
2354 | { |
2355 | return testapp_invalid_desc_mb(test); |
2356 | } |
2357 | |
2358 | static int testapp_unaligned_inv_desc_mb(struct test_spec *test) |
2359 | { |
2360 | test->ifobj_tx->umem->unaligned_mode = true; |
2361 | test->ifobj_rx->umem->unaligned_mode = true; |
2362 | return testapp_invalid_desc_mb(test); |
2363 | } |
2364 | |
2365 | static int testapp_xdp_metadata(struct test_spec *test) |
2366 | { |
2367 | return testapp_xdp_metadata_copy(test); |
2368 | } |
2369 | |
2370 | static int testapp_xdp_metadata_mb(struct test_spec *test) |
2371 | { |
2372 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2373 | return testapp_xdp_metadata_copy(test); |
2374 | } |
2375 | |
2376 | static void run_pkt_test(struct test_spec *test) |
2377 | { |
2378 | int ret; |
2379 | |
2380 | ret = test->test_func(test); |
2381 | |
2382 | if (ret == TEST_PASS) |
2383 | ksft_test_result_pass(msg: "PASS: %s %s%s\n" , mode_string(test), busy_poll_string(test), |
2384 | test->name); |
2385 | pkt_stream_restore_default(test); |
2386 | } |
2387 | |
2388 | static struct ifobject *ifobject_create(void) |
2389 | { |
2390 | struct ifobject *ifobj; |
2391 | |
2392 | ifobj = calloc(1, sizeof(struct ifobject)); |
2393 | if (!ifobj) |
2394 | return NULL; |
2395 | |
2396 | ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); |
2397 | if (!ifobj->xsk_arr) |
2398 | goto out_xsk_arr; |
2399 | |
2400 | ifobj->umem = calloc(1, sizeof(*ifobj->umem)); |
2401 | if (!ifobj->umem) |
2402 | goto out_umem; |
2403 | |
2404 | return ifobj; |
2405 | |
2406 | out_umem: |
2407 | free(ifobj->xsk_arr); |
2408 | out_xsk_arr: |
2409 | free(ifobj); |
2410 | return NULL; |
2411 | } |
2412 | |
2413 | static void ifobject_delete(struct ifobject *ifobj) |
2414 | { |
2415 | free(ifobj->umem); |
2416 | free(ifobj->xsk_arr); |
2417 | free(ifobj); |
2418 | } |
2419 | |
2420 | static bool is_xdp_supported(int ifindex) |
2421 | { |
2422 | int flags = XDP_FLAGS_DRV_MODE; |
2423 | |
2424 | LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); |
2425 | struct bpf_insn insns[2] = { |
2426 | BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), |
2427 | BPF_EXIT_INSN() |
2428 | }; |
2429 | int prog_fd, insn_cnt = ARRAY_SIZE(insns); |
2430 | int err; |
2431 | |
2432 | prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL" , insns, insn_cnt, NULL); |
2433 | if (prog_fd < 0) |
2434 | return false; |
2435 | |
2436 | err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); |
2437 | if (err) { |
2438 | close(prog_fd); |
2439 | return false; |
2440 | } |
2441 | |
2442 | bpf_xdp_detach(ifindex, flags, NULL); |
2443 | close(prog_fd); |
2444 | |
2445 | return true; |
2446 | } |
2447 | |
2448 | static const struct test_spec tests[] = { |
2449 | {.name = "SEND_RECEIVE" , .test_func = testapp_send_receive}, |
2450 | {.name = "SEND_RECEIVE_2K_FRAME" , .test_func = testapp_send_receive_2k_frame}, |
2451 | {.name = "SEND_RECEIVE_SINGLE_PKT" , .test_func = testapp_single_pkt}, |
2452 | {.name = "POLL_RX" , .test_func = testapp_poll_rx}, |
2453 | {.name = "POLL_TX" , .test_func = testapp_poll_tx}, |
2454 | {.name = "POLL_RXQ_FULL" , .test_func = testapp_poll_rxq_tmout}, |
2455 | {.name = "POLL_TXQ_FULL" , .test_func = testapp_poll_txq_tmout}, |
2456 | {.name = "SEND_RECEIVE_UNALIGNED" , .test_func = testapp_send_receive_unaligned}, |
2457 | {.name = "ALIGNED_INV_DESC" , .test_func = testapp_aligned_inv_desc}, |
2458 | {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE" , .test_func = testapp_aligned_inv_desc_2k_frame}, |
2459 | {.name = "UNALIGNED_INV_DESC" , .test_func = testapp_unaligned_inv_desc}, |
2460 | {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE" , |
2461 | .test_func = testapp_unaligned_inv_desc_4001_frame}, |
2462 | {.name = "UMEM_HEADROOM" , .test_func = testapp_headroom}, |
2463 | {.name = "TEARDOWN" , .test_func = testapp_teardown}, |
2464 | {.name = "BIDIRECTIONAL" , .test_func = testapp_bidirectional}, |
2465 | {.name = "STAT_RX_DROPPED" , .test_func = testapp_stats_rx_dropped}, |
2466 | {.name = "STAT_TX_INVALID" , .test_func = testapp_stats_tx_invalid_descs}, |
2467 | {.name = "STAT_RX_FULL" , .test_func = testapp_stats_rx_full}, |
2468 | {.name = "STAT_FILL_EMPTY" , .test_func = testapp_stats_fill_empty}, |
2469 | {.name = "XDP_PROG_CLEANUP" , .test_func = testapp_xdp_prog_cleanup}, |
2470 | {.name = "XDP_DROP_HALF" , .test_func = testapp_xdp_drop}, |
2471 | {.name = "XDP_SHARED_UMEM" , .test_func = testapp_xdp_shared_umem}, |
2472 | {.name = "XDP_METADATA_COPY" , .test_func = testapp_xdp_metadata}, |
2473 | {.name = "XDP_METADATA_COPY_MULTI_BUFF" , .test_func = testapp_xdp_metadata_mb}, |
2474 | {.name = "SEND_RECEIVE_9K_PACKETS" , .test_func = testapp_send_receive_mb}, |
2475 | {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS" , |
2476 | .test_func = testapp_send_receive_unaligned_mb}, |
2477 | {.name = "ALIGNED_INV_DESC_MULTI_BUFF" , .test_func = testapp_aligned_inv_desc_mb}, |
2478 | {.name = "UNALIGNED_INV_DESC_MULTI_BUFF" , .test_func = testapp_unaligned_inv_desc_mb}, |
2479 | {.name = "TOO_MANY_FRAGS" , .test_func = testapp_too_many_frags}, |
2480 | }; |
2481 | |
2482 | static void print_tests(void) |
2483 | { |
2484 | u32 i; |
2485 | |
2486 | printf("Tests:\n" ); |
2487 | for (i = 0; i < ARRAY_SIZE(tests); i++) |
2488 | printf("%u: %s\n" , i, tests[i].name); |
2489 | } |
2490 | |
2491 | int main(int argc, char **argv) |
2492 | { |
2493 | struct pkt_stream *rx_pkt_stream_default; |
2494 | struct pkt_stream *tx_pkt_stream_default; |
2495 | struct ifobject *ifobj_tx, *ifobj_rx; |
2496 | u32 i, j, failed_tests = 0, nb_tests; |
2497 | int modes = TEST_MODE_SKB + 1; |
2498 | struct test_spec test; |
2499 | bool shared_netdev; |
2500 | |
2501 | /* Use libbpf 1.0 API mode */ |
2502 | libbpf_set_strict_mode(LIBBPF_STRICT_ALL); |
2503 | |
2504 | ifobj_tx = ifobject_create(); |
2505 | if (!ifobj_tx) |
2506 | exit_with_error(ENOMEM); |
2507 | ifobj_rx = ifobject_create(); |
2508 | if (!ifobj_rx) |
2509 | exit_with_error(ENOMEM); |
2510 | |
2511 | setlocale(LC_ALL, "" ); |
2512 | |
2513 | parse_command_line(ifobj_tx, ifobj_rx, argc, argv); |
2514 | |
2515 | if (opt_print_tests) { |
2516 | print_tests(); |
2517 | ksft_exit_xpass(); |
2518 | } |
2519 | if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) { |
2520 | ksft_print_msg(msg: "Error: test %u does not exist.\n" , opt_run_test); |
2521 | ksft_exit_xfail(); |
2522 | } |
2523 | |
2524 | shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); |
2525 | ifobj_tx->shared_umem = shared_netdev; |
2526 | ifobj_rx->shared_umem = shared_netdev; |
2527 | |
2528 | if (!validate_interface(ifobj: ifobj_tx) || !validate_interface(ifobj: ifobj_rx)) |
2529 | print_usage(argv); |
2530 | |
2531 | if (is_xdp_supported(ifindex: ifobj_tx->ifindex)) { |
2532 | modes++; |
2533 | if (ifobj_zc_avail(ifobject: ifobj_tx)) |
2534 | modes++; |
2535 | } |
2536 | |
2537 | init_iface(ifobj: ifobj_rx, func_ptr: worker_testapp_validate_rx); |
2538 | init_iface(ifobj: ifobj_tx, func_ptr: worker_testapp_validate_tx); |
2539 | |
2540 | test_spec_init(test: &test, ifobj_tx, ifobj_rx, mode: 0, test_to_run: &tests[0]); |
2541 | tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); |
2542 | rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); |
2543 | if (!tx_pkt_stream_default || !rx_pkt_stream_default) |
2544 | exit_with_error(ENOMEM); |
2545 | test.tx_pkt_stream_default = tx_pkt_stream_default; |
2546 | test.rx_pkt_stream_default = rx_pkt_stream_default; |
2547 | |
2548 | if (opt_run_test == RUN_ALL_TESTS) |
2549 | nb_tests = ARRAY_SIZE(tests); |
2550 | else |
2551 | nb_tests = 1; |
2552 | if (opt_mode == TEST_MODE_ALL) { |
2553 | ksft_set_plan(plan: modes * nb_tests); |
2554 | } else { |
2555 | if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) { |
2556 | ksft_print_msg(msg: "Error: XDP_DRV mode not supported.\n" ); |
2557 | ksft_exit_xfail(); |
2558 | } |
2559 | if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) { |
2560 | ksft_print_msg(msg: "Error: zero-copy mode not supported.\n" ); |
2561 | ksft_exit_xfail(); |
2562 | } |
2563 | |
2564 | ksft_set_plan(plan: nb_tests); |
2565 | } |
2566 | |
2567 | for (i = 0; i < modes; i++) { |
2568 | if (opt_mode != TEST_MODE_ALL && i != opt_mode) |
2569 | continue; |
2570 | |
2571 | for (j = 0; j < ARRAY_SIZE(tests); j++) { |
2572 | if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) |
2573 | continue; |
2574 | |
2575 | test_spec_init(test: &test, ifobj_tx, ifobj_rx, mode: i, test_to_run: &tests[j]); |
2576 | run_pkt_test(test: &test); |
2577 | usleep(USLEEP_MAX); |
2578 | |
2579 | if (test.fail) |
2580 | failed_tests++; |
2581 | } |
2582 | } |
2583 | |
2584 | pkt_stream_delete(pkt_stream: tx_pkt_stream_default); |
2585 | pkt_stream_delete(pkt_stream: rx_pkt_stream_default); |
2586 | xsk_unload_xdp_programs(ifobj: ifobj_tx); |
2587 | xsk_unload_xdp_programs(ifobj: ifobj_rx); |
2588 | ifobject_delete(ifobj: ifobj_tx); |
2589 | ifobject_delete(ifobj: ifobj_rx); |
2590 | |
2591 | if (failed_tests) |
2592 | ksft_exit_fail(); |
2593 | else |
2594 | ksft_exit_pass(); |
2595 | } |
2596 | |