1 | // SPDX-License-Identifier: GPL-2.0 |
2 | // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io |
3 | #include <stdio.h> |
4 | #include <stdlib.h> |
5 | #include <sys/socket.h> |
6 | #include <sys/ioctl.h> |
7 | #include <sys/select.h> |
8 | #include <netinet/in.h> |
9 | #include <arpa/inet.h> |
10 | #include <unistd.h> |
11 | #include <string.h> |
12 | #include <errno.h> |
13 | #include <stdbool.h> |
14 | #include <signal.h> |
15 | #include <fcntl.h> |
16 | #include <sys/wait.h> |
17 | #include <time.h> |
18 | #include <sched.h> |
19 | |
20 | #include <sys/time.h> |
21 | #include <sys/types.h> |
22 | #include <sys/sendfile.h> |
23 | |
24 | #include <linux/netlink.h> |
25 | #include <linux/socket.h> |
26 | #include <linux/sock_diag.h> |
27 | #include <linux/bpf.h> |
28 | #include <linux/if_link.h> |
29 | #include <linux/tls.h> |
30 | #include <assert.h> |
31 | #include <libgen.h> |
32 | |
33 | #include <getopt.h> |
34 | |
35 | #include <bpf/bpf.h> |
36 | #include <bpf/libbpf.h> |
37 | |
38 | #include "bpf_util.h" |
39 | #include "cgroup_helpers.h" |
40 | |
41 | int running; |
42 | static void running_handler(int a); |
43 | |
44 | #ifndef TCP_ULP |
45 | # define TCP_ULP 31 |
46 | #endif |
47 | #ifndef SOL_TLS |
48 | # define SOL_TLS 282 |
49 | #endif |
50 | |
51 | /* randomly selected ports for testing on lo */ |
52 | #define S1_PORT 10000 |
53 | #define S2_PORT 10001 |
54 | |
55 | #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" |
56 | #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" |
57 | #define CG_PATH "/sockmap" |
58 | |
59 | /* global sockets */ |
60 | int s1, s2, c1, c2, p1, p2; |
61 | int test_cnt; |
62 | int passed; |
63 | int failed; |
64 | int map_fd[9]; |
65 | struct bpf_map *maps[9]; |
66 | int prog_fd[11]; |
67 | |
68 | int txmsg_pass; |
69 | int txmsg_redir; |
70 | int txmsg_drop; |
71 | int txmsg_apply; |
72 | int txmsg_cork; |
73 | int txmsg_start; |
74 | int txmsg_end; |
75 | int txmsg_start_push; |
76 | int txmsg_end_push; |
77 | int txmsg_start_pop; |
78 | int txmsg_pop; |
79 | int txmsg_ingress; |
80 | int txmsg_redir_skb; |
81 | int txmsg_ktls_skb; |
82 | int txmsg_ktls_skb_drop; |
83 | int txmsg_ktls_skb_redir; |
84 | int ktls; |
85 | int peek_flag; |
86 | int skb_use_parser; |
87 | int txmsg_omit_skb_parser; |
88 | |
89 | static const struct option long_options[] = { |
90 | {"help" , no_argument, NULL, 'h' }, |
91 | {"cgroup" , required_argument, NULL, 'c' }, |
92 | {"rate" , required_argument, NULL, 'r' }, |
93 | {"verbose" , optional_argument, NULL, 'v' }, |
94 | {"iov_count" , required_argument, NULL, 'i' }, |
95 | {"length" , required_argument, NULL, 'l' }, |
96 | {"test" , required_argument, NULL, 't' }, |
97 | {"data_test" , no_argument, NULL, 'd' }, |
98 | {"txmsg" , no_argument, &txmsg_pass, 1 }, |
99 | {"txmsg_redir" , no_argument, &txmsg_redir, 1 }, |
100 | {"txmsg_drop" , no_argument, &txmsg_drop, 1 }, |
101 | {"txmsg_apply" , required_argument, NULL, 'a'}, |
102 | {"txmsg_cork" , required_argument, NULL, 'k'}, |
103 | {"txmsg_start" , required_argument, NULL, 's'}, |
104 | {"txmsg_end" , required_argument, NULL, 'e'}, |
105 | {"txmsg_start_push" , required_argument, NULL, 'p'}, |
106 | {"txmsg_end_push" , required_argument, NULL, 'q'}, |
107 | {"txmsg_start_pop" , required_argument, NULL, 'w'}, |
108 | {"txmsg_pop" , required_argument, NULL, 'x'}, |
109 | {"txmsg_ingress" , no_argument, &txmsg_ingress, 1 }, |
110 | {"txmsg_redir_skb" , no_argument, &txmsg_redir_skb, 1 }, |
111 | {"ktls" , no_argument, &ktls, 1 }, |
112 | {"peek" , no_argument, &peek_flag, 1 }, |
113 | {"txmsg_omit_skb_parser" , no_argument, &txmsg_omit_skb_parser, 1}, |
114 | {"whitelist" , required_argument, NULL, 'n' }, |
115 | {"blacklist" , required_argument, NULL, 'b' }, |
116 | {0, 0, NULL, 0 } |
117 | }; |
118 | |
119 | struct test_env { |
120 | const char *type; |
121 | const char *subtest; |
122 | const char *prepend; |
123 | |
124 | int test_num; |
125 | int subtest_num; |
126 | |
127 | int succ_cnt; |
128 | int fail_cnt; |
129 | int fail_last; |
130 | }; |
131 | |
132 | struct test_env env; |
133 | |
134 | struct sockmap_options { |
135 | int verbose; |
136 | bool base; |
137 | bool sendpage; |
138 | bool data_test; |
139 | bool drop_expected; |
140 | bool check_recved_len; |
141 | bool tx_wait_mem; |
142 | int iov_count; |
143 | int iov_length; |
144 | int rate; |
145 | char *map; |
146 | char *whitelist; |
147 | char *blacklist; |
148 | char *prepend; |
149 | }; |
150 | |
151 | struct _test { |
152 | char *title; |
153 | void (*tester)(int cg_fd, struct sockmap_options *opt); |
154 | }; |
155 | |
156 | static void test_start(void) |
157 | { |
158 | env.subtest_num++; |
159 | } |
160 | |
161 | static void test_fail(void) |
162 | { |
163 | env.fail_cnt++; |
164 | } |
165 | |
166 | static void test_pass(void) |
167 | { |
168 | env.succ_cnt++; |
169 | } |
170 | |
171 | static void test_reset(void) |
172 | { |
173 | txmsg_start = txmsg_end = 0; |
174 | txmsg_start_pop = txmsg_pop = 0; |
175 | txmsg_start_push = txmsg_end_push = 0; |
176 | txmsg_pass = txmsg_drop = txmsg_redir = 0; |
177 | txmsg_apply = txmsg_cork = 0; |
178 | txmsg_ingress = txmsg_redir_skb = 0; |
179 | txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; |
180 | txmsg_omit_skb_parser = 0; |
181 | skb_use_parser = 0; |
182 | } |
183 | |
184 | static int test_start_subtest(const struct _test *t, struct sockmap_options *o) |
185 | { |
186 | env.type = o->map; |
187 | env.subtest = t->title; |
188 | env.prepend = o->prepend; |
189 | env.test_num++; |
190 | env.subtest_num = 0; |
191 | env.fail_last = env.fail_cnt; |
192 | test_reset(); |
193 | return 0; |
194 | } |
195 | |
196 | static void test_end_subtest(void) |
197 | { |
198 | int error = env.fail_cnt - env.fail_last; |
199 | int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); |
200 | |
201 | if (!error) |
202 | test_pass(); |
203 | |
204 | fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n" , |
205 | env.test_num, env.subtest_num, |
206 | !type ? "sockmap" : "sockhash" , |
207 | env.prepend ? : "" , |
208 | env.subtest, error ? "FAIL" : "OK" ); |
209 | } |
210 | |
211 | static void test_print_results(void) |
212 | { |
213 | fprintf(stdout, "Pass: %d Fail: %d\n" , |
214 | env.succ_cnt, env.fail_cnt); |
215 | } |
216 | |
217 | static void usage(char *argv[]) |
218 | { |
219 | int i; |
220 | |
221 | printf(" Usage: %s --cgroup <cgroup_path>\n" , argv[0]); |
222 | printf(" options:\n" ); |
223 | for (i = 0; long_options[i].name != 0; i++) { |
224 | printf(" --%-12s" , long_options[i].name); |
225 | if (long_options[i].flag != NULL) |
226 | printf(" flag (internal value:%d)\n" , |
227 | *long_options[i].flag); |
228 | else |
229 | printf(" -%c\n" , long_options[i].val); |
230 | } |
231 | printf("\n" ); |
232 | } |
233 | |
234 | char *sock_to_string(int s) |
235 | { |
236 | if (s == c1) |
237 | return "client1" ; |
238 | else if (s == c2) |
239 | return "client2" ; |
240 | else if (s == s1) |
241 | return "server1" ; |
242 | else if (s == s2) |
243 | return "server2" ; |
244 | else if (s == p1) |
245 | return "peer1" ; |
246 | else if (s == p2) |
247 | return "peer2" ; |
248 | else |
249 | return "unknown" ; |
250 | } |
251 | |
252 | static int sockmap_init_ktls(int verbose, int s) |
253 | { |
254 | struct tls12_crypto_info_aes_gcm_128 tls_tx = { |
255 | .info = { |
256 | .version = TLS_1_2_VERSION, |
257 | .cipher_type = TLS_CIPHER_AES_GCM_128, |
258 | }, |
259 | }; |
260 | struct tls12_crypto_info_aes_gcm_128 tls_rx = { |
261 | .info = { |
262 | .version = TLS_1_2_VERSION, |
263 | .cipher_type = TLS_CIPHER_AES_GCM_128, |
264 | }, |
265 | }; |
266 | int so_buf = 6553500; |
267 | int err; |
268 | |
269 | err = setsockopt(s, 6, TCP_ULP, "tls" , sizeof("tls" )); |
270 | if (err) { |
271 | fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n" , sock_to_string(s), err); |
272 | return -EINVAL; |
273 | } |
274 | err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); |
275 | if (err) { |
276 | fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n" , sock_to_string(s), err); |
277 | return -EINVAL; |
278 | } |
279 | err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); |
280 | if (err) { |
281 | fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n" , sock_to_string(s), err); |
282 | return -EINVAL; |
283 | } |
284 | err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); |
285 | if (err) { |
286 | fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n" , sock_to_string(s), err); |
287 | return -EINVAL; |
288 | } |
289 | err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); |
290 | if (err) { |
291 | fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n" , sock_to_string(s), err); |
292 | return -EINVAL; |
293 | } |
294 | |
295 | if (verbose) |
296 | fprintf(stdout, "socket(%s) kTLS enabled\n" , sock_to_string(s)); |
297 | return 0; |
298 | } |
299 | static int sockmap_init_sockets(int verbose) |
300 | { |
301 | int i, err, one = 1; |
302 | struct sockaddr_in addr; |
303 | int *fds[4] = {&s1, &s2, &c1, &c2}; |
304 | |
305 | s1 = s2 = p1 = p2 = c1 = c2 = 0; |
306 | |
307 | /* Init sockets */ |
308 | for (i = 0; i < 4; i++) { |
309 | *fds[i] = socket(AF_INET, SOCK_STREAM, 0); |
310 | if (*fds[i] < 0) { |
311 | perror("socket s1 failed()" ); |
312 | return errno; |
313 | } |
314 | } |
315 | |
316 | /* Allow reuse */ |
317 | for (i = 0; i < 2; i++) { |
318 | err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, |
319 | (char *)&one, sizeof(one)); |
320 | if (err) { |
321 | perror("setsockopt failed()" ); |
322 | return errno; |
323 | } |
324 | } |
325 | |
326 | /* Non-blocking sockets */ |
327 | for (i = 0; i < 2; i++) { |
328 | err = ioctl(*fds[i], FIONBIO, (char *)&one); |
329 | if (err < 0) { |
330 | perror("ioctl s1 failed()" ); |
331 | return errno; |
332 | } |
333 | } |
334 | |
335 | /* Bind server sockets */ |
336 | memset(&addr, 0, sizeof(struct sockaddr_in)); |
337 | addr.sin_family = AF_INET; |
338 | addr.sin_addr.s_addr = inet_addr("127.0.0.1" ); |
339 | |
340 | addr.sin_port = htons(S1_PORT); |
341 | err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); |
342 | if (err < 0) { |
343 | perror("bind s1 failed()" ); |
344 | return errno; |
345 | } |
346 | |
347 | addr.sin_port = htons(S2_PORT); |
348 | err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); |
349 | if (err < 0) { |
350 | perror("bind s2 failed()" ); |
351 | return errno; |
352 | } |
353 | |
354 | /* Listen server sockets */ |
355 | addr.sin_port = htons(S1_PORT); |
356 | err = listen(s1, 32); |
357 | if (err < 0) { |
358 | perror("listen s1 failed()" ); |
359 | return errno; |
360 | } |
361 | |
362 | addr.sin_port = htons(S2_PORT); |
363 | err = listen(s2, 32); |
364 | if (err < 0) { |
365 | perror("listen s1 failed()" ); |
366 | return errno; |
367 | } |
368 | |
369 | /* Initiate Connect */ |
370 | addr.sin_port = htons(S1_PORT); |
371 | err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); |
372 | if (err < 0 && errno != EINPROGRESS) { |
373 | perror("connect c1 failed()" ); |
374 | return errno; |
375 | } |
376 | |
377 | addr.sin_port = htons(S2_PORT); |
378 | err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); |
379 | if (err < 0 && errno != EINPROGRESS) { |
380 | perror("connect c2 failed()" ); |
381 | return errno; |
382 | } else if (err < 0) { |
383 | err = 0; |
384 | } |
385 | |
386 | /* Accept Connecrtions */ |
387 | p1 = accept(s1, NULL, NULL); |
388 | if (p1 < 0) { |
389 | perror("accept s1 failed()" ); |
390 | return errno; |
391 | } |
392 | |
393 | p2 = accept(s2, NULL, NULL); |
394 | if (p2 < 0) { |
395 | perror("accept s1 failed()" ); |
396 | return errno; |
397 | } |
398 | |
399 | if (verbose > 1) { |
400 | printf("connected sockets: c1 <-> p1, c2 <-> p2\n" ); |
401 | printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n" , |
402 | c1, s1, c2, s2); |
403 | } |
404 | return 0; |
405 | } |
406 | |
407 | struct msg_stats { |
408 | size_t bytes_sent; |
409 | size_t bytes_recvd; |
410 | struct timespec start; |
411 | struct timespec end; |
412 | }; |
413 | |
414 | static int msg_loop_sendpage(int fd, int iov_length, int cnt, |
415 | struct msg_stats *s, |
416 | struct sockmap_options *opt) |
417 | { |
418 | bool drop = opt->drop_expected; |
419 | unsigned char k = 0; |
420 | FILE *file; |
421 | int i, fp; |
422 | |
423 | file = tmpfile(); |
424 | if (!file) { |
425 | perror("create file for sendpage" ); |
426 | return 1; |
427 | } |
428 | for (i = 0; i < iov_length * cnt; i++, k++) |
429 | fwrite(&k, sizeof(char), 1, file); |
430 | fflush(file); |
431 | fseek(file, 0, SEEK_SET); |
432 | |
433 | fp = fileno(file); |
434 | |
435 | clock_gettime(CLOCK_MONOTONIC, &s->start); |
436 | for (i = 0; i < cnt; i++) { |
437 | int sent; |
438 | |
439 | errno = 0; |
440 | sent = sendfile(fd, fp, NULL, iov_length); |
441 | |
442 | if (!drop && sent < 0) { |
443 | perror("sendpage loop error" ); |
444 | fclose(file); |
445 | return sent; |
446 | } else if (drop && sent >= 0) { |
447 | printf("sendpage loop error expected: %i errno %i\n" , |
448 | sent, errno); |
449 | fclose(file); |
450 | return -EIO; |
451 | } |
452 | |
453 | if (sent > 0) |
454 | s->bytes_sent += sent; |
455 | } |
456 | clock_gettime(CLOCK_MONOTONIC, &s->end); |
457 | fclose(file); |
458 | return 0; |
459 | } |
460 | |
461 | static void msg_free_iov(struct msghdr *msg) |
462 | { |
463 | int i; |
464 | |
465 | for (i = 0; i < msg->msg_iovlen; i++) |
466 | free(msg->msg_iov[i].iov_base); |
467 | free(msg->msg_iov); |
468 | msg->msg_iov = NULL; |
469 | msg->msg_iovlen = 0; |
470 | } |
471 | |
472 | static int msg_alloc_iov(struct msghdr *msg, |
473 | int iov_count, int iov_length, |
474 | bool data, bool xmit) |
475 | { |
476 | unsigned char k = 0; |
477 | struct iovec *iov; |
478 | int i; |
479 | |
480 | iov = calloc(iov_count, sizeof(struct iovec)); |
481 | if (!iov) |
482 | return errno; |
483 | |
484 | for (i = 0; i < iov_count; i++) { |
485 | unsigned char *d = calloc(iov_length, sizeof(char)); |
486 | |
487 | if (!d) { |
488 | fprintf(stderr, "iov_count %i/%i OOM\n" , i, iov_count); |
489 | goto unwind_iov; |
490 | } |
491 | iov[i].iov_base = d; |
492 | iov[i].iov_len = iov_length; |
493 | |
494 | if (data && xmit) { |
495 | int j; |
496 | |
497 | for (j = 0; j < iov_length; j++) |
498 | d[j] = k++; |
499 | } |
500 | } |
501 | |
502 | msg->msg_iov = iov; |
503 | msg->msg_iovlen = iov_count; |
504 | |
505 | return 0; |
506 | unwind_iov: |
507 | for (i--; i >= 0 ; i--) |
508 | free(msg->msg_iov[i].iov_base); |
509 | return -ENOMEM; |
510 | } |
511 | |
512 | static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) |
513 | { |
514 | int i, j = 0, bytes_cnt = 0; |
515 | unsigned char k = 0; |
516 | |
517 | for (i = 0; i < msg->msg_iovlen; i++) { |
518 | unsigned char *d = msg->msg_iov[i].iov_base; |
519 | |
520 | /* Special case test for skb ingress + ktls */ |
521 | if (i == 0 && txmsg_ktls_skb) { |
522 | if (msg->msg_iov[i].iov_len < 4) |
523 | return -EIO; |
524 | if (memcmp(p: d, q: "PASS" , size: 4) != 0) { |
525 | fprintf(stderr, |
526 | "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n" , |
527 | i, 0, d[0], d[1], d[2], d[3]); |
528 | return -EIO; |
529 | } |
530 | j = 4; /* advance index past PASS header */ |
531 | } |
532 | |
533 | for (; j < msg->msg_iov[i].iov_len && size; j++) { |
534 | if (d[j] != k++) { |
535 | fprintf(stderr, |
536 | "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n" , |
537 | i, j, d[j], k - 1, d[j+1], k); |
538 | return -EIO; |
539 | } |
540 | bytes_cnt++; |
541 | if (bytes_cnt == chunk_sz) { |
542 | k = 0; |
543 | bytes_cnt = 0; |
544 | } |
545 | size--; |
546 | } |
547 | } |
548 | return 0; |
549 | } |
550 | |
551 | static int msg_loop(int fd, int iov_count, int iov_length, int cnt, |
552 | struct msg_stats *s, bool tx, |
553 | struct sockmap_options *opt) |
554 | { |
555 | struct msghdr msg = {0}, msg_peek = {0}; |
556 | int err, i, flags = MSG_NOSIGNAL; |
557 | bool drop = opt->drop_expected; |
558 | bool data = opt->data_test; |
559 | int iov_alloc_length = iov_length; |
560 | |
561 | if (!tx && opt->check_recved_len) |
562 | iov_alloc_length *= 2; |
563 | |
564 | err = msg_alloc_iov(msg: &msg, iov_count, iov_length: iov_alloc_length, data, xmit: tx); |
565 | if (err) |
566 | goto out_errno; |
567 | if (peek_flag) { |
568 | err = msg_alloc_iov(msg: &msg_peek, iov_count, iov_length, data, xmit: tx); |
569 | if (err) |
570 | goto out_errno; |
571 | } |
572 | |
573 | if (tx) { |
574 | clock_gettime(CLOCK_MONOTONIC, &s->start); |
575 | for (i = 0; i < cnt; i++) { |
576 | int sent; |
577 | |
578 | errno = 0; |
579 | sent = sendmsg(fd, &msg, flags); |
580 | |
581 | if (!drop && sent < 0) { |
582 | if (opt->tx_wait_mem && errno == EACCES) { |
583 | errno = 0; |
584 | goto out_errno; |
585 | } |
586 | perror("sendmsg loop error" ); |
587 | goto out_errno; |
588 | } else if (drop && sent >= 0) { |
589 | fprintf(stderr, |
590 | "sendmsg loop error expected: %i errno %i\n" , |
591 | sent, errno); |
592 | errno = -EIO; |
593 | goto out_errno; |
594 | } |
595 | if (sent > 0) |
596 | s->bytes_sent += sent; |
597 | } |
598 | clock_gettime(CLOCK_MONOTONIC, &s->end); |
599 | } else { |
600 | int slct, recvp = 0, recv, max_fd = fd; |
601 | float total_bytes, txmsg_pop_total; |
602 | int fd_flags = O_NONBLOCK; |
603 | struct timeval timeout; |
604 | fd_set w; |
605 | |
606 | fcntl(fd, fd_flags); |
607 | /* Account for pop bytes noting each iteration of apply will |
608 | * call msg_pop_data helper so we need to account for this |
609 | * by calculating the number of apply iterations. Note user |
610 | * of the tool can create cases where no data is sent by |
611 | * manipulating pop/push/pull/etc. For example txmsg_apply 1 |
612 | * with txmsg_pop 1 will try to apply 1B at a time but each |
613 | * iteration will then pop 1B so no data will ever be sent. |
614 | * This is really only useful for testing edge cases in code |
615 | * paths. |
616 | */ |
617 | total_bytes = (float)iov_count * (float)iov_length * (float)cnt; |
618 | if (txmsg_apply) |
619 | txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); |
620 | else |
621 | txmsg_pop_total = txmsg_pop * cnt; |
622 | total_bytes -= txmsg_pop_total; |
623 | err = clock_gettime(CLOCK_MONOTONIC, &s->start); |
624 | if (err < 0) |
625 | perror("recv start time" ); |
626 | while (s->bytes_recvd < total_bytes) { |
627 | if (txmsg_cork) { |
628 | timeout.tv_sec = 0; |
629 | timeout.tv_usec = 300000; |
630 | } else { |
631 | timeout.tv_sec = 3; |
632 | timeout.tv_usec = 0; |
633 | } |
634 | |
635 | /* FD sets */ |
636 | FD_ZERO(&w); |
637 | FD_SET(fd, &w); |
638 | |
639 | slct = select(max_fd + 1, &w, NULL, NULL, &timeout); |
640 | if (slct == -1) { |
641 | perror("select()" ); |
642 | clock_gettime(CLOCK_MONOTONIC, &s->end); |
643 | goto out_errno; |
644 | } else if (!slct) { |
645 | if (opt->verbose) |
646 | fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n" , s->bytes_recvd, total_bytes, txmsg_pop_total); |
647 | errno = -EIO; |
648 | clock_gettime(CLOCK_MONOTONIC, &s->end); |
649 | goto out_errno; |
650 | } |
651 | |
652 | if (opt->tx_wait_mem) { |
653 | FD_ZERO(&w); |
654 | FD_SET(fd, &w); |
655 | slct = select(max_fd + 1, NULL, NULL, &w, &timeout); |
656 | errno = 0; |
657 | close(fd); |
658 | goto out_errno; |
659 | } |
660 | |
661 | errno = 0; |
662 | if (peek_flag) { |
663 | flags |= MSG_PEEK; |
664 | recvp = recvmsg(fd, &msg_peek, flags); |
665 | if (recvp < 0) { |
666 | if (errno != EWOULDBLOCK) { |
667 | clock_gettime(CLOCK_MONOTONIC, &s->end); |
668 | goto out_errno; |
669 | } |
670 | } |
671 | flags = 0; |
672 | } |
673 | |
674 | recv = recvmsg(fd, &msg, flags); |
675 | if (recv < 0) { |
676 | if (errno != EWOULDBLOCK) { |
677 | clock_gettime(CLOCK_MONOTONIC, &s->end); |
678 | perror("recv failed()" ); |
679 | goto out_errno; |
680 | } |
681 | } |
682 | |
683 | s->bytes_recvd += recv; |
684 | |
685 | if (opt->check_recved_len && s->bytes_recvd > total_bytes) { |
686 | errno = EMSGSIZE; |
687 | fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n" , |
688 | s->bytes_recvd, total_bytes); |
689 | goto out_errno; |
690 | } |
691 | |
692 | if (data) { |
693 | int chunk_sz = opt->sendpage ? |
694 | iov_length * cnt : |
695 | iov_length * iov_count; |
696 | |
697 | errno = msg_verify_data(&msg, recv, chunk_sz); |
698 | if (errno) { |
699 | perror("data verify msg failed" ); |
700 | goto out_errno; |
701 | } |
702 | if (recvp) { |
703 | errno = msg_verify_data(&msg_peek, |
704 | recvp, |
705 | chunk_sz); |
706 | if (errno) { |
707 | perror("data verify msg_peek failed" ); |
708 | goto out_errno; |
709 | } |
710 | } |
711 | } |
712 | } |
713 | clock_gettime(CLOCK_MONOTONIC, &s->end); |
714 | } |
715 | |
716 | msg_free_iov(msg: &msg); |
717 | msg_free_iov(msg: &msg_peek); |
718 | return err; |
719 | out_errno: |
720 | msg_free_iov(msg: &msg); |
721 | msg_free_iov(msg: &msg_peek); |
722 | return errno; |
723 | } |
724 | |
725 | static float giga = 1000000000; |
726 | |
727 | static inline float sentBps(struct msg_stats s) |
728 | { |
729 | return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); |
730 | } |
731 | |
732 | static inline float recvdBps(struct msg_stats s) |
733 | { |
734 | return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); |
735 | } |
736 | |
737 | static int sendmsg_test(struct sockmap_options *opt) |
738 | { |
739 | float sent_Bps = 0, recvd_Bps = 0; |
740 | int rx_fd, txpid, rxpid, err = 0; |
741 | struct msg_stats s = {0}; |
742 | int iov_count = opt->iov_count; |
743 | int iov_buf = opt->iov_length; |
744 | int rx_status, tx_status; |
745 | int cnt = opt->rate; |
746 | |
747 | errno = 0; |
748 | |
749 | if (opt->base) |
750 | rx_fd = p1; |
751 | else |
752 | rx_fd = p2; |
753 | |
754 | if (ktls) { |
755 | /* Redirecting into non-TLS socket which sends into a TLS |
756 | * socket is not a valid test. So in this case lets not |
757 | * enable kTLS but still run the test. |
758 | */ |
759 | if (!txmsg_redir || txmsg_ingress) { |
760 | err = sockmap_init_ktls(verbose: opt->verbose, s: rx_fd); |
761 | if (err) |
762 | return err; |
763 | } |
764 | err = sockmap_init_ktls(verbose: opt->verbose, s: c1); |
765 | if (err) |
766 | return err; |
767 | } |
768 | |
769 | if (opt->tx_wait_mem) { |
770 | struct timeval timeout; |
771 | int rxtx_buf_len = 1024; |
772 | |
773 | timeout.tv_sec = 3; |
774 | timeout.tv_usec = 0; |
775 | |
776 | err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); |
777 | err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); |
778 | err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); |
779 | if (err) { |
780 | perror("setsockopt failed()" ); |
781 | return errno; |
782 | } |
783 | } |
784 | |
785 | rxpid = fork(); |
786 | if (rxpid == 0) { |
787 | if (txmsg_pop || txmsg_start_pop) |
788 | iov_buf -= (txmsg_pop - txmsg_start_pop + 1); |
789 | if (opt->drop_expected || txmsg_ktls_skb_drop) |
790 | _exit(0); |
791 | |
792 | if (!iov_buf) /* zero bytes sent case */ |
793 | _exit(0); |
794 | |
795 | if (opt->sendpage) |
796 | iov_count = 1; |
797 | err = msg_loop(fd: rx_fd, iov_count, iov_length: iov_buf, |
798 | cnt, s: &s, tx: false, opt); |
799 | if (opt->verbose > 1) |
800 | fprintf(stderr, |
801 | "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n" , |
802 | iov_count, iov_buf, cnt, err); |
803 | if (s.end.tv_sec - s.start.tv_sec) { |
804 | sent_Bps = sentBps(s); |
805 | recvd_Bps = recvdBps(s); |
806 | } |
807 | if (opt->verbose > 1) |
808 | fprintf(stdout, |
809 | "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n" , |
810 | s.bytes_sent, sent_Bps, sent_Bps/giga, |
811 | s.bytes_recvd, recvd_Bps, recvd_Bps/giga, |
812 | peek_flag ? "(peek_msg)" : "" ); |
813 | if (err && txmsg_cork) |
814 | err = 0; |
815 | exit(err ? 1 : 0); |
816 | } else if (rxpid == -1) { |
817 | perror("msg_loop_rx" ); |
818 | return errno; |
819 | } |
820 | |
821 | if (opt->tx_wait_mem) |
822 | close(c2); |
823 | |
824 | txpid = fork(); |
825 | if (txpid == 0) { |
826 | if (opt->sendpage) |
827 | err = msg_loop_sendpage(fd: c1, iov_length: iov_buf, cnt, s: &s, opt); |
828 | else |
829 | err = msg_loop(fd: c1, iov_count, iov_length: iov_buf, |
830 | cnt, s: &s, tx: true, opt); |
831 | |
832 | if (err) |
833 | fprintf(stderr, |
834 | "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n" , |
835 | iov_count, iov_buf, cnt, err); |
836 | if (s.end.tv_sec - s.start.tv_sec) { |
837 | sent_Bps = sentBps(s); |
838 | recvd_Bps = recvdBps(s); |
839 | } |
840 | if (opt->verbose > 1) |
841 | fprintf(stdout, |
842 | "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n" , |
843 | s.bytes_sent, sent_Bps, sent_Bps/giga, |
844 | s.bytes_recvd, recvd_Bps, recvd_Bps/giga); |
845 | exit(err ? 1 : 0); |
846 | } else if (txpid == -1) { |
847 | perror("msg_loop_tx" ); |
848 | return errno; |
849 | } |
850 | |
851 | assert(waitpid(rxpid, &rx_status, 0) == rxpid); |
852 | assert(waitpid(txpid, &tx_status, 0) == txpid); |
853 | if (WIFEXITED(rx_status)) { |
854 | err = WEXITSTATUS(rx_status); |
855 | if (err) { |
856 | fprintf(stderr, "rx thread exited with err %d.\n" , err); |
857 | goto out; |
858 | } |
859 | } |
860 | if (WIFEXITED(tx_status)) { |
861 | err = WEXITSTATUS(tx_status); |
862 | if (err) |
863 | fprintf(stderr, "tx thread exited with err %d.\n" , err); |
864 | } |
865 | out: |
866 | return err; |
867 | } |
868 | |
869 | static int forever_ping_pong(int rate, struct sockmap_options *opt) |
870 | { |
871 | struct timeval timeout; |
872 | char buf[1024] = {0}; |
873 | int sc; |
874 | |
875 | timeout.tv_sec = 10; |
876 | timeout.tv_usec = 0; |
877 | |
878 | /* Ping/Pong data from client to server */ |
879 | sc = send(c1, buf, sizeof(buf), 0); |
880 | if (sc < 0) { |
881 | perror("send failed()" ); |
882 | return sc; |
883 | } |
884 | |
885 | do { |
886 | int s, rc, i, max_fd = p2; |
887 | fd_set w; |
888 | |
889 | /* FD sets */ |
890 | FD_ZERO(&w); |
891 | FD_SET(c1, &w); |
892 | FD_SET(c2, &w); |
893 | FD_SET(p1, &w); |
894 | FD_SET(p2, &w); |
895 | |
896 | s = select(max_fd + 1, &w, NULL, NULL, &timeout); |
897 | if (s == -1) { |
898 | perror("select()" ); |
899 | break; |
900 | } else if (!s) { |
901 | fprintf(stderr, "unexpected timeout\n" ); |
902 | break; |
903 | } |
904 | |
905 | for (i = 0; i <= max_fd && s > 0; ++i) { |
906 | if (!FD_ISSET(i, &w)) |
907 | continue; |
908 | |
909 | s--; |
910 | |
911 | rc = recv(i, buf, sizeof(buf), 0); |
912 | if (rc < 0) { |
913 | if (errno != EWOULDBLOCK) { |
914 | perror("recv failed()" ); |
915 | return rc; |
916 | } |
917 | } |
918 | |
919 | if (rc == 0) { |
920 | close(i); |
921 | break; |
922 | } |
923 | |
924 | sc = send(i, buf, rc, 0); |
925 | if (sc < 0) { |
926 | perror("send failed()" ); |
927 | return sc; |
928 | } |
929 | } |
930 | |
931 | if (rate) |
932 | sleep(rate); |
933 | |
934 | if (opt->verbose) { |
935 | printf("." ); |
936 | fflush(stdout); |
937 | |
938 | } |
939 | } while (running); |
940 | |
941 | return 0; |
942 | } |
943 | |
944 | enum { |
945 | SELFTESTS, |
946 | PING_PONG, |
947 | SENDMSG, |
948 | BASE, |
949 | BASE_SENDPAGE, |
950 | SENDPAGE, |
951 | }; |
952 | |
953 | static int run_options(struct sockmap_options *options, int cg_fd, int test) |
954 | { |
955 | int i, key, next_key, err, tx_prog_fd = -1, zero = 0; |
956 | |
957 | /* If base test skip BPF setup */ |
958 | if (test == BASE || test == BASE_SENDPAGE) |
959 | goto run; |
960 | |
961 | /* Attach programs to sockmap */ |
962 | if (!txmsg_omit_skb_parser) { |
963 | err = bpf_prog_attach(prog_fd[0], map_fd[0], |
964 | BPF_SK_SKB_STREAM_PARSER, 0); |
965 | if (err) { |
966 | fprintf(stderr, |
967 | "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n" , |
968 | prog_fd[0], map_fd[0], err, strerror(errno)); |
969 | return err; |
970 | } |
971 | } |
972 | |
973 | err = bpf_prog_attach(prog_fd[1], map_fd[0], |
974 | BPF_SK_SKB_STREAM_VERDICT, 0); |
975 | if (err) { |
976 | fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n" , |
977 | err, strerror(errno)); |
978 | return err; |
979 | } |
980 | |
981 | /* Attach programs to TLS sockmap */ |
982 | if (txmsg_ktls_skb) { |
983 | if (!txmsg_omit_skb_parser) { |
984 | err = bpf_prog_attach(prog_fd[0], map_fd[8], |
985 | BPF_SK_SKB_STREAM_PARSER, 0); |
986 | if (err) { |
987 | fprintf(stderr, |
988 | "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n" , |
989 | prog_fd[0], map_fd[8], err, strerror(errno)); |
990 | return err; |
991 | } |
992 | } |
993 | |
994 | err = bpf_prog_attach(prog_fd[2], map_fd[8], |
995 | BPF_SK_SKB_STREAM_VERDICT, 0); |
996 | if (err) { |
997 | fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n" , |
998 | err, strerror(errno)); |
999 | return err; |
1000 | } |
1001 | } |
1002 | |
1003 | /* Attach to cgroups */ |
1004 | err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0); |
1005 | if (err) { |
1006 | fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n" , |
1007 | err, strerror(errno)); |
1008 | return err; |
1009 | } |
1010 | |
1011 | run: |
1012 | err = sockmap_init_sockets(verbose: options->verbose); |
1013 | if (err) { |
1014 | fprintf(stderr, "ERROR: test socket failed: %d\n" , err); |
1015 | goto out; |
1016 | } |
1017 | |
1018 | /* Attach txmsg program to sockmap */ |
1019 | if (txmsg_pass) |
1020 | tx_prog_fd = prog_fd[4]; |
1021 | else if (txmsg_redir) |
1022 | tx_prog_fd = prog_fd[5]; |
1023 | else if (txmsg_apply) |
1024 | tx_prog_fd = prog_fd[6]; |
1025 | else if (txmsg_cork) |
1026 | tx_prog_fd = prog_fd[7]; |
1027 | else if (txmsg_drop) |
1028 | tx_prog_fd = prog_fd[8]; |
1029 | else |
1030 | tx_prog_fd = 0; |
1031 | |
1032 | if (tx_prog_fd) { |
1033 | int redir_fd, i = 0; |
1034 | |
1035 | err = bpf_prog_attach(tx_prog_fd, |
1036 | map_fd[1], BPF_SK_MSG_VERDICT, 0); |
1037 | if (err) { |
1038 | fprintf(stderr, |
1039 | "ERROR: bpf_prog_attach (txmsg): %d (%s)\n" , |
1040 | err, strerror(errno)); |
1041 | goto out; |
1042 | } |
1043 | |
1044 | err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); |
1045 | if (err) { |
1046 | fprintf(stderr, |
1047 | "ERROR: bpf_map_update_elem (txmsg): %d (%s\n" , |
1048 | err, strerror(errno)); |
1049 | goto out; |
1050 | } |
1051 | |
1052 | if (txmsg_redir) |
1053 | redir_fd = c2; |
1054 | else |
1055 | redir_fd = c1; |
1056 | |
1057 | err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); |
1058 | if (err) { |
1059 | fprintf(stderr, |
1060 | "ERROR: bpf_map_update_elem (txmsg): %d (%s\n" , |
1061 | err, strerror(errno)); |
1062 | goto out; |
1063 | } |
1064 | |
1065 | if (txmsg_apply) { |
1066 | err = bpf_map_update_elem(map_fd[3], |
1067 | &i, &txmsg_apply, BPF_ANY); |
1068 | if (err) { |
1069 | fprintf(stderr, |
1070 | "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n" , |
1071 | err, strerror(errno)); |
1072 | goto out; |
1073 | } |
1074 | } |
1075 | |
1076 | if (txmsg_cork) { |
1077 | err = bpf_map_update_elem(map_fd[4], |
1078 | &i, &txmsg_cork, BPF_ANY); |
1079 | if (err) { |
1080 | fprintf(stderr, |
1081 | "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n" , |
1082 | err, strerror(errno)); |
1083 | goto out; |
1084 | } |
1085 | } |
1086 | |
1087 | if (txmsg_start) { |
1088 | err = bpf_map_update_elem(map_fd[5], |
1089 | &i, &txmsg_start, BPF_ANY); |
1090 | if (err) { |
1091 | fprintf(stderr, |
1092 | "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n" , |
1093 | err, strerror(errno)); |
1094 | goto out; |
1095 | } |
1096 | } |
1097 | |
1098 | if (txmsg_end) { |
1099 | i = 1; |
1100 | err = bpf_map_update_elem(map_fd[5], |
1101 | &i, &txmsg_end, BPF_ANY); |
1102 | if (err) { |
1103 | fprintf(stderr, |
1104 | "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n" , |
1105 | err, strerror(errno)); |
1106 | goto out; |
1107 | } |
1108 | } |
1109 | |
1110 | if (txmsg_start_push) { |
1111 | i = 2; |
1112 | err = bpf_map_update_elem(map_fd[5], |
1113 | &i, &txmsg_start_push, BPF_ANY); |
1114 | if (err) { |
1115 | fprintf(stderr, |
1116 | "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n" , |
1117 | err, strerror(errno)); |
1118 | goto out; |
1119 | } |
1120 | } |
1121 | |
1122 | if (txmsg_end_push) { |
1123 | i = 3; |
1124 | err = bpf_map_update_elem(map_fd[5], |
1125 | &i, &txmsg_end_push, BPF_ANY); |
1126 | if (err) { |
1127 | fprintf(stderr, |
1128 | "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n" , |
1129 | txmsg_end_push, i, err, strerror(errno)); |
1130 | goto out; |
1131 | } |
1132 | } |
1133 | |
1134 | if (txmsg_start_pop) { |
1135 | i = 4; |
1136 | err = bpf_map_update_elem(map_fd[5], |
1137 | &i, &txmsg_start_pop, BPF_ANY); |
1138 | if (err) { |
1139 | fprintf(stderr, |
1140 | "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n" , |
1141 | txmsg_start_pop, i, err, strerror(errno)); |
1142 | goto out; |
1143 | } |
1144 | } else { |
1145 | i = 4; |
1146 | bpf_map_update_elem(map_fd[5], |
1147 | &i, &txmsg_start_pop, BPF_ANY); |
1148 | } |
1149 | |
1150 | if (txmsg_pop) { |
1151 | i = 5; |
1152 | err = bpf_map_update_elem(map_fd[5], |
1153 | &i, &txmsg_pop, BPF_ANY); |
1154 | if (err) { |
1155 | fprintf(stderr, |
1156 | "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n" , |
1157 | txmsg_pop, i, err, strerror(errno)); |
1158 | goto out; |
1159 | } |
1160 | } else { |
1161 | i = 5; |
1162 | bpf_map_update_elem(map_fd[5], |
1163 | &i, &txmsg_pop, BPF_ANY); |
1164 | |
1165 | } |
1166 | |
1167 | if (txmsg_ingress) { |
1168 | int in = BPF_F_INGRESS; |
1169 | |
1170 | i = 0; |
1171 | err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); |
1172 | if (err) { |
1173 | fprintf(stderr, |
1174 | "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n" , |
1175 | err, strerror(errno)); |
1176 | } |
1177 | i = 1; |
1178 | err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); |
1179 | if (err) { |
1180 | fprintf(stderr, |
1181 | "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n" , |
1182 | err, strerror(errno)); |
1183 | } |
1184 | err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); |
1185 | if (err) { |
1186 | fprintf(stderr, |
1187 | "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n" , |
1188 | err, strerror(errno)); |
1189 | } |
1190 | |
1191 | i = 2; |
1192 | err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); |
1193 | if (err) { |
1194 | fprintf(stderr, |
1195 | "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n" , |
1196 | err, strerror(errno)); |
1197 | } |
1198 | } |
1199 | |
1200 | if (txmsg_ktls_skb) { |
1201 | int ingress = BPF_F_INGRESS; |
1202 | |
1203 | i = 0; |
1204 | err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); |
1205 | if (err) { |
1206 | fprintf(stderr, |
1207 | "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n" , |
1208 | err, strerror(errno)); |
1209 | } |
1210 | |
1211 | if (txmsg_ktls_skb_redir) { |
1212 | i = 1; |
1213 | err = bpf_map_update_elem(map_fd[7], |
1214 | &i, &ingress, BPF_ANY); |
1215 | if (err) { |
1216 | fprintf(stderr, |
1217 | "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n" , |
1218 | err, strerror(errno)); |
1219 | } |
1220 | } |
1221 | |
1222 | if (txmsg_ktls_skb_drop) { |
1223 | i = 1; |
1224 | err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); |
1225 | } |
1226 | } |
1227 | |
1228 | if (txmsg_redir_skb) { |
1229 | int skb_fd = (test == SENDMSG || test == SENDPAGE) ? |
1230 | p2 : p1; |
1231 | int ingress = BPF_F_INGRESS; |
1232 | |
1233 | i = 0; |
1234 | err = bpf_map_update_elem(map_fd[7], |
1235 | &i, &ingress, BPF_ANY); |
1236 | if (err) { |
1237 | fprintf(stderr, |
1238 | "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n" , |
1239 | err, strerror(errno)); |
1240 | } |
1241 | |
1242 | i = 3; |
1243 | err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); |
1244 | if (err) { |
1245 | fprintf(stderr, |
1246 | "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n" , |
1247 | err, strerror(errno)); |
1248 | } |
1249 | } |
1250 | } |
1251 | |
1252 | if (skb_use_parser) { |
1253 | i = 2; |
1254 | err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); |
1255 | } |
1256 | |
1257 | if (txmsg_drop) |
1258 | options->drop_expected = true; |
1259 | |
1260 | if (test == PING_PONG) |
1261 | err = forever_ping_pong(rate: options->rate, opt: options); |
1262 | else if (test == SENDMSG) { |
1263 | options->base = false; |
1264 | options->sendpage = false; |
1265 | err = sendmsg_test(opt: options); |
1266 | } else if (test == SENDPAGE) { |
1267 | options->base = false; |
1268 | options->sendpage = true; |
1269 | err = sendmsg_test(opt: options); |
1270 | } else if (test == BASE) { |
1271 | options->base = true; |
1272 | options->sendpage = false; |
1273 | err = sendmsg_test(opt: options); |
1274 | } else if (test == BASE_SENDPAGE) { |
1275 | options->base = true; |
1276 | options->sendpage = true; |
1277 | err = sendmsg_test(opt: options); |
1278 | } else |
1279 | fprintf(stderr, "unknown test\n" ); |
1280 | out: |
1281 | /* Detatch and zero all the maps */ |
1282 | bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS); |
1283 | bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); |
1284 | bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); |
1285 | bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER); |
1286 | bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT); |
1287 | |
1288 | if (tx_prog_fd >= 0) |
1289 | bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); |
1290 | |
1291 | for (i = 0; i < 8; i++) { |
1292 | key = next_key = 0; |
1293 | bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); |
1294 | while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { |
1295 | bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); |
1296 | key = next_key; |
1297 | } |
1298 | } |
1299 | |
1300 | close(s1); |
1301 | close(s2); |
1302 | close(p1); |
1303 | close(p2); |
1304 | close(c1); |
1305 | close(c2); |
1306 | return err; |
1307 | } |
1308 | |
1309 | static char *test_to_str(int test) |
1310 | { |
1311 | switch (test) { |
1312 | case SENDMSG: |
1313 | return "sendmsg" ; |
1314 | case SENDPAGE: |
1315 | return "sendpage" ; |
1316 | } |
1317 | return "unknown" ; |
1318 | } |
1319 | |
1320 | static void append_str(char *dst, const char *src, size_t dst_cap) |
1321 | { |
1322 | size_t avail = dst_cap - strlen(dst); |
1323 | |
1324 | if (avail <= 1) /* just zero byte could be written */ |
1325 | return; |
1326 | |
1327 | strncat(p: dst, q: src, count: avail - 1); /* strncat() adds + 1 for zero byte */ |
1328 | } |
1329 | |
1330 | #define OPTSTRING 60 |
1331 | static void test_options(char *options) |
1332 | { |
1333 | char tstr[OPTSTRING]; |
1334 | |
1335 | memset(options, 0, OPTSTRING); |
1336 | |
1337 | if (txmsg_pass) |
1338 | append_str(dst: options, src: "pass," , OPTSTRING); |
1339 | if (txmsg_redir) |
1340 | append_str(dst: options, src: "redir," , OPTSTRING); |
1341 | if (txmsg_drop) |
1342 | append_str(dst: options, src: "drop," , OPTSTRING); |
1343 | if (txmsg_apply) { |
1344 | snprintf(buf: tstr, OPTSTRING, fmt: "apply %d," , txmsg_apply); |
1345 | append_str(dst: options, src: tstr, OPTSTRING); |
1346 | } |
1347 | if (txmsg_cork) { |
1348 | snprintf(buf: tstr, OPTSTRING, fmt: "cork %d," , txmsg_cork); |
1349 | append_str(dst: options, src: tstr, OPTSTRING); |
1350 | } |
1351 | if (txmsg_start) { |
1352 | snprintf(buf: tstr, OPTSTRING, fmt: "start %d," , txmsg_start); |
1353 | append_str(dst: options, src: tstr, OPTSTRING); |
1354 | } |
1355 | if (txmsg_end) { |
1356 | snprintf(buf: tstr, OPTSTRING, fmt: "end %d," , txmsg_end); |
1357 | append_str(dst: options, src: tstr, OPTSTRING); |
1358 | } |
1359 | if (txmsg_start_pop) { |
1360 | snprintf(buf: tstr, OPTSTRING, fmt: "pop (%d,%d)," , |
1361 | txmsg_start_pop, txmsg_start_pop + txmsg_pop); |
1362 | append_str(dst: options, src: tstr, OPTSTRING); |
1363 | } |
1364 | if (txmsg_ingress) |
1365 | append_str(dst: options, src: "ingress," , OPTSTRING); |
1366 | if (txmsg_redir_skb) |
1367 | append_str(dst: options, src: "redir_skb," , OPTSTRING); |
1368 | if (txmsg_ktls_skb) |
1369 | append_str(dst: options, src: "ktls_skb," , OPTSTRING); |
1370 | if (ktls) |
1371 | append_str(dst: options, src: "ktls," , OPTSTRING); |
1372 | if (peek_flag) |
1373 | append_str(dst: options, src: "peek," , OPTSTRING); |
1374 | } |
1375 | |
1376 | static int __test_exec(int cgrp, int test, struct sockmap_options *opt) |
1377 | { |
1378 | char *options = calloc(OPTSTRING, sizeof(char)); |
1379 | int err; |
1380 | |
1381 | if (test == SENDPAGE) |
1382 | opt->sendpage = true; |
1383 | else |
1384 | opt->sendpage = false; |
1385 | |
1386 | if (txmsg_drop) |
1387 | opt->drop_expected = true; |
1388 | else |
1389 | opt->drop_expected = false; |
1390 | |
1391 | test_options(options); |
1392 | |
1393 | if (opt->verbose) { |
1394 | fprintf(stdout, |
1395 | " [TEST %i]: (%i, %i, %i, %s, %s): " , |
1396 | test_cnt, opt->rate, opt->iov_count, opt->iov_length, |
1397 | test_to_str(test), options); |
1398 | fflush(stdout); |
1399 | } |
1400 | err = run_options(options: opt, cg_fd: cgrp, test); |
1401 | if (opt->verbose) |
1402 | fprintf(stdout, " %s\n" , !err ? "PASS" : "FAILED" ); |
1403 | test_cnt++; |
1404 | !err ? passed++ : failed++; |
1405 | free(options); |
1406 | return err; |
1407 | } |
1408 | |
1409 | static void test_exec(int cgrp, struct sockmap_options *opt) |
1410 | { |
1411 | int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); |
1412 | int err; |
1413 | |
1414 | if (type == 0) { |
1415 | test_start(); |
1416 | err = __test_exec(cgrp, test: SENDMSG, opt); |
1417 | if (err) |
1418 | test_fail(); |
1419 | } else { |
1420 | test_start(); |
1421 | err = __test_exec(cgrp, test: SENDPAGE, opt); |
1422 | if (err) |
1423 | test_fail(); |
1424 | } |
1425 | } |
1426 | |
1427 | static void test_send_one(struct sockmap_options *opt, int cgrp) |
1428 | { |
1429 | opt->iov_length = 1; |
1430 | opt->iov_count = 1; |
1431 | opt->rate = 1; |
1432 | test_exec(cgrp, opt); |
1433 | |
1434 | opt->iov_length = 1; |
1435 | opt->iov_count = 1024; |
1436 | opt->rate = 1; |
1437 | test_exec(cgrp, opt); |
1438 | |
1439 | opt->iov_length = 1024; |
1440 | opt->iov_count = 1; |
1441 | opt->rate = 1; |
1442 | test_exec(cgrp, opt); |
1443 | |
1444 | } |
1445 | |
1446 | static void test_send_many(struct sockmap_options *opt, int cgrp) |
1447 | { |
1448 | opt->iov_length = 3; |
1449 | opt->iov_count = 1; |
1450 | opt->rate = 512; |
1451 | test_exec(cgrp, opt); |
1452 | |
1453 | opt->rate = 100; |
1454 | opt->iov_count = 1; |
1455 | opt->iov_length = 5; |
1456 | test_exec(cgrp, opt); |
1457 | } |
1458 | |
1459 | static void test_send_large(struct sockmap_options *opt, int cgrp) |
1460 | { |
1461 | opt->iov_length = 256; |
1462 | opt->iov_count = 1024; |
1463 | opt->rate = 2; |
1464 | test_exec(cgrp, opt); |
1465 | } |
1466 | |
1467 | static void test_send(struct sockmap_options *opt, int cgrp) |
1468 | { |
1469 | test_send_one(opt, cgrp); |
1470 | test_send_many(opt, cgrp); |
1471 | test_send_large(opt, cgrp); |
1472 | sched_yield(); |
1473 | } |
1474 | |
1475 | static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) |
1476 | { |
1477 | /* Test small and large iov_count values with pass/redir/apply/cork */ |
1478 | txmsg_pass = 1; |
1479 | test_send(opt, cgrp); |
1480 | } |
1481 | |
1482 | static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) |
1483 | { |
1484 | txmsg_redir = 1; |
1485 | test_send(opt, cgrp); |
1486 | } |
1487 | |
1488 | static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) |
1489 | { |
1490 | txmsg_redir = 1; |
1491 | opt->tx_wait_mem = true; |
1492 | test_send_large(opt, cgrp); |
1493 | opt->tx_wait_mem = false; |
1494 | } |
1495 | |
1496 | static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) |
1497 | { |
1498 | txmsg_drop = 1; |
1499 | test_send(opt, cgrp); |
1500 | } |
1501 | |
1502 | static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) |
1503 | { |
1504 | txmsg_pass = txmsg_drop = 0; |
1505 | txmsg_ingress = txmsg_redir = 1; |
1506 | test_send(opt, cgrp); |
1507 | } |
1508 | |
1509 | static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) |
1510 | { |
1511 | bool data = opt->data_test; |
1512 | int k = ktls; |
1513 | |
1514 | opt->data_test = true; |
1515 | ktls = 1; |
1516 | |
1517 | txmsg_pass = txmsg_drop = 0; |
1518 | txmsg_ingress = txmsg_redir = 0; |
1519 | txmsg_ktls_skb = 1; |
1520 | txmsg_pass = 1; |
1521 | |
1522 | /* Using data verification so ensure iov layout is |
1523 | * expected from test receiver side. e.g. has enough |
1524 | * bytes to write test code. |
1525 | */ |
1526 | opt->iov_length = 100; |
1527 | opt->iov_count = 1; |
1528 | opt->rate = 1; |
1529 | test_exec(cgrp, opt); |
1530 | |
1531 | txmsg_ktls_skb_drop = 1; |
1532 | test_exec(cgrp, opt); |
1533 | |
1534 | txmsg_ktls_skb_drop = 0; |
1535 | txmsg_ktls_skb_redir = 1; |
1536 | test_exec(cgrp, opt); |
1537 | txmsg_ktls_skb_redir = 0; |
1538 | |
1539 | /* Tests that omit skb_parser */ |
1540 | txmsg_omit_skb_parser = 1; |
1541 | ktls = 0; |
1542 | txmsg_ktls_skb = 0; |
1543 | test_exec(cgrp, opt); |
1544 | |
1545 | txmsg_ktls_skb_drop = 1; |
1546 | test_exec(cgrp, opt); |
1547 | txmsg_ktls_skb_drop = 0; |
1548 | |
1549 | txmsg_ktls_skb_redir = 1; |
1550 | test_exec(cgrp, opt); |
1551 | |
1552 | ktls = 1; |
1553 | test_exec(cgrp, opt); |
1554 | txmsg_omit_skb_parser = 0; |
1555 | |
1556 | opt->data_test = data; |
1557 | ktls = k; |
1558 | } |
1559 | |
1560 | /* Test cork with hung data. This tests poor usage patterns where |
1561 | * cork can leave data on the ring if user program is buggy and |
1562 | * doesn't flush them somehow. They do take some time however |
1563 | * because they wait for a timeout. Test pass, redir and cork with |
1564 | * apply logic. Use cork size of 4097 with send_large to avoid |
1565 | * aligning cork size with send size. |
1566 | */ |
1567 | static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) |
1568 | { |
1569 | txmsg_pass = 1; |
1570 | txmsg_redir = 0; |
1571 | txmsg_cork = 4097; |
1572 | txmsg_apply = 4097; |
1573 | test_send_large(opt, cgrp); |
1574 | |
1575 | txmsg_pass = 0; |
1576 | txmsg_redir = 1; |
1577 | txmsg_apply = 0; |
1578 | txmsg_cork = 4097; |
1579 | test_send_large(opt, cgrp); |
1580 | |
1581 | txmsg_pass = 0; |
1582 | txmsg_redir = 1; |
1583 | txmsg_apply = 4097; |
1584 | txmsg_cork = 4097; |
1585 | test_send_large(opt, cgrp); |
1586 | } |
1587 | |
1588 | static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) |
1589 | { |
1590 | /* Test basic start/end */ |
1591 | txmsg_start = 1; |
1592 | txmsg_end = 2; |
1593 | test_send(opt, cgrp); |
1594 | |
1595 | /* Test >4k pull */ |
1596 | txmsg_start = 4096; |
1597 | txmsg_end = 9182; |
1598 | test_send_large(opt, cgrp); |
1599 | |
1600 | /* Test pull + redirect */ |
1601 | txmsg_redir = 0; |
1602 | txmsg_start = 1; |
1603 | txmsg_end = 2; |
1604 | test_send(opt, cgrp); |
1605 | |
1606 | /* Test pull + cork */ |
1607 | txmsg_redir = 0; |
1608 | txmsg_cork = 512; |
1609 | txmsg_start = 1; |
1610 | txmsg_end = 2; |
1611 | test_send_many(opt, cgrp); |
1612 | |
1613 | /* Test pull + cork + redirect */ |
1614 | txmsg_redir = 1; |
1615 | txmsg_cork = 512; |
1616 | txmsg_start = 1; |
1617 | txmsg_end = 2; |
1618 | test_send_many(opt, cgrp); |
1619 | } |
1620 | |
1621 | static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) |
1622 | { |
1623 | /* Test basic pop */ |
1624 | txmsg_start_pop = 1; |
1625 | txmsg_pop = 2; |
1626 | test_send_many(opt, cgrp); |
1627 | |
1628 | /* Test pop with >4k */ |
1629 | txmsg_start_pop = 4096; |
1630 | txmsg_pop = 4096; |
1631 | test_send_large(opt, cgrp); |
1632 | |
1633 | /* Test pop + redirect */ |
1634 | txmsg_redir = 1; |
1635 | txmsg_start_pop = 1; |
1636 | txmsg_pop = 2; |
1637 | test_send_many(opt, cgrp); |
1638 | |
1639 | /* Test pop + cork */ |
1640 | txmsg_redir = 0; |
1641 | txmsg_cork = 512; |
1642 | txmsg_start_pop = 1; |
1643 | txmsg_pop = 2; |
1644 | test_send_many(opt, cgrp); |
1645 | |
1646 | /* Test pop + redirect + cork */ |
1647 | txmsg_redir = 1; |
1648 | txmsg_cork = 4; |
1649 | txmsg_start_pop = 1; |
1650 | txmsg_pop = 2; |
1651 | test_send_many(opt, cgrp); |
1652 | } |
1653 | |
1654 | static void test_txmsg_push(int cgrp, struct sockmap_options *opt) |
1655 | { |
1656 | /* Test basic push */ |
1657 | txmsg_start_push = 1; |
1658 | txmsg_end_push = 1; |
1659 | test_send(opt, cgrp); |
1660 | |
1661 | /* Test push 4kB >4k */ |
1662 | txmsg_start_push = 4096; |
1663 | txmsg_end_push = 4096; |
1664 | test_send_large(opt, cgrp); |
1665 | |
1666 | /* Test push + redirect */ |
1667 | txmsg_redir = 1; |
1668 | txmsg_start_push = 1; |
1669 | txmsg_end_push = 2; |
1670 | test_send_many(opt, cgrp); |
1671 | |
1672 | /* Test push + cork */ |
1673 | txmsg_redir = 0; |
1674 | txmsg_cork = 512; |
1675 | txmsg_start_push = 1; |
1676 | txmsg_end_push = 2; |
1677 | test_send_many(opt, cgrp); |
1678 | } |
1679 | |
1680 | static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) |
1681 | { |
1682 | txmsg_start_push = 1; |
1683 | txmsg_end_push = 10; |
1684 | txmsg_start_pop = 5; |
1685 | txmsg_pop = 4; |
1686 | test_send_large(opt, cgrp); |
1687 | } |
1688 | |
1689 | static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) |
1690 | { |
1691 | txmsg_pass = 1; |
1692 | txmsg_redir = 0; |
1693 | txmsg_ingress = 0; |
1694 | txmsg_apply = 1; |
1695 | txmsg_cork = 0; |
1696 | test_send_one(opt, cgrp); |
1697 | |
1698 | txmsg_pass = 0; |
1699 | txmsg_redir = 1; |
1700 | txmsg_ingress = 0; |
1701 | txmsg_apply = 1; |
1702 | txmsg_cork = 0; |
1703 | test_send_one(opt, cgrp); |
1704 | |
1705 | txmsg_pass = 0; |
1706 | txmsg_redir = 1; |
1707 | txmsg_ingress = 1; |
1708 | txmsg_apply = 1; |
1709 | txmsg_cork = 0; |
1710 | test_send_one(opt, cgrp); |
1711 | |
1712 | txmsg_pass = 1; |
1713 | txmsg_redir = 0; |
1714 | txmsg_ingress = 0; |
1715 | txmsg_apply = 1024; |
1716 | txmsg_cork = 0; |
1717 | test_send_large(opt, cgrp); |
1718 | |
1719 | txmsg_pass = 0; |
1720 | txmsg_redir = 1; |
1721 | txmsg_ingress = 0; |
1722 | txmsg_apply = 1024; |
1723 | txmsg_cork = 0; |
1724 | test_send_large(opt, cgrp); |
1725 | |
1726 | txmsg_pass = 0; |
1727 | txmsg_redir = 1; |
1728 | txmsg_ingress = 1; |
1729 | txmsg_apply = 1024; |
1730 | txmsg_cork = 0; |
1731 | test_send_large(opt, cgrp); |
1732 | } |
1733 | |
1734 | static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) |
1735 | { |
1736 | txmsg_pass = 1; |
1737 | txmsg_redir = 0; |
1738 | txmsg_apply = 0; |
1739 | txmsg_cork = 1; |
1740 | test_send(opt, cgrp); |
1741 | |
1742 | txmsg_pass = 1; |
1743 | txmsg_redir = 0; |
1744 | txmsg_apply = 1; |
1745 | txmsg_cork = 1; |
1746 | test_send(opt, cgrp); |
1747 | } |
1748 | |
1749 | static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) |
1750 | { |
1751 | txmsg_pass = 1; |
1752 | skb_use_parser = 512; |
1753 | if (ktls == 1) |
1754 | skb_use_parser = 570; |
1755 | opt->iov_length = 256; |
1756 | opt->iov_count = 1; |
1757 | opt->rate = 2; |
1758 | test_exec(cgrp, opt); |
1759 | } |
1760 | |
1761 | static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) |
1762 | { |
1763 | if (ktls == 1) |
1764 | return; |
1765 | skb_use_parser = 10; |
1766 | opt->iov_length = 20; |
1767 | opt->iov_count = 1; |
1768 | opt->rate = 1; |
1769 | opt->check_recved_len = true; |
1770 | test_exec(cgrp, opt); |
1771 | opt->check_recved_len = false; |
1772 | } |
1773 | |
1774 | char *map_names[] = { |
1775 | "sock_map" , |
1776 | "sock_map_txmsg" , |
1777 | "sock_map_redir" , |
1778 | "sock_apply_bytes" , |
1779 | "sock_cork_bytes" , |
1780 | "sock_bytes" , |
1781 | "sock_redir_flags" , |
1782 | "sock_skb_opts" , |
1783 | "tls_sock_map" , |
1784 | }; |
1785 | |
1786 | int prog_attach_type[] = { |
1787 | BPF_SK_SKB_STREAM_PARSER, |
1788 | BPF_SK_SKB_STREAM_VERDICT, |
1789 | BPF_SK_SKB_STREAM_VERDICT, |
1790 | BPF_CGROUP_SOCK_OPS, |
1791 | BPF_SK_MSG_VERDICT, |
1792 | BPF_SK_MSG_VERDICT, |
1793 | BPF_SK_MSG_VERDICT, |
1794 | BPF_SK_MSG_VERDICT, |
1795 | BPF_SK_MSG_VERDICT, |
1796 | BPF_SK_MSG_VERDICT, |
1797 | BPF_SK_MSG_VERDICT, |
1798 | }; |
1799 | |
1800 | int prog_type[] = { |
1801 | BPF_PROG_TYPE_SK_SKB, |
1802 | BPF_PROG_TYPE_SK_SKB, |
1803 | BPF_PROG_TYPE_SK_SKB, |
1804 | BPF_PROG_TYPE_SOCK_OPS, |
1805 | BPF_PROG_TYPE_SK_MSG, |
1806 | BPF_PROG_TYPE_SK_MSG, |
1807 | BPF_PROG_TYPE_SK_MSG, |
1808 | BPF_PROG_TYPE_SK_MSG, |
1809 | BPF_PROG_TYPE_SK_MSG, |
1810 | BPF_PROG_TYPE_SK_MSG, |
1811 | BPF_PROG_TYPE_SK_MSG, |
1812 | }; |
1813 | |
1814 | static int populate_progs(char *bpf_file) |
1815 | { |
1816 | struct bpf_program *prog; |
1817 | struct bpf_object *obj; |
1818 | int i = 0; |
1819 | long err; |
1820 | |
1821 | obj = bpf_object__open(bpf_file); |
1822 | err = libbpf_get_error(obj); |
1823 | if (err) { |
1824 | char err_buf[256]; |
1825 | |
1826 | libbpf_strerror(err, err_buf, sizeof(err_buf)); |
1827 | printf("Unable to load eBPF objects in file '%s' : %s\n" , |
1828 | bpf_file, err_buf); |
1829 | return -1; |
1830 | } |
1831 | |
1832 | bpf_object__for_each_program(prog, obj) { |
1833 | bpf_program__set_type(prog, prog_type[i]); |
1834 | bpf_program__set_expected_attach_type(prog, |
1835 | prog_attach_type[i]); |
1836 | i++; |
1837 | } |
1838 | |
1839 | i = bpf_object__load(obj); |
1840 | i = 0; |
1841 | bpf_object__for_each_program(prog, obj) { |
1842 | prog_fd[i] = bpf_program__fd(prog); |
1843 | i++; |
1844 | } |
1845 | |
1846 | for (i = 0; i < ARRAY_SIZE(map_fd); i++) { |
1847 | maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); |
1848 | map_fd[i] = bpf_map__fd(maps[i]); |
1849 | if (map_fd[i] < 0) { |
1850 | fprintf(stderr, "load_bpf_file: (%i) %s\n" , |
1851 | map_fd[i], strerror(errno)); |
1852 | return -1; |
1853 | } |
1854 | } |
1855 | |
1856 | return 0; |
1857 | } |
1858 | |
1859 | struct _test test[] = { |
1860 | {"txmsg test passthrough" , test_txmsg_pass}, |
1861 | {"txmsg test redirect" , test_txmsg_redir}, |
1862 | {"txmsg test redirect wait send mem" , test_txmsg_redir_wait_sndmem}, |
1863 | {"txmsg test drop" , test_txmsg_drop}, |
1864 | {"txmsg test ingress redirect" , test_txmsg_ingress_redir}, |
1865 | {"txmsg test skb" , test_txmsg_skb}, |
1866 | {"txmsg test apply" , test_txmsg_apply}, |
1867 | {"txmsg test cork" , test_txmsg_cork}, |
1868 | {"txmsg test hanging corks" , test_txmsg_cork_hangs}, |
1869 | {"txmsg test push_data" , test_txmsg_push}, |
1870 | {"txmsg test pull-data" , test_txmsg_pull}, |
1871 | {"txmsg test pop-data" , test_txmsg_pop}, |
1872 | {"txmsg test push/pop data" , test_txmsg_push_pop}, |
1873 | {"txmsg test ingress parser" , test_txmsg_ingress_parser}, |
1874 | {"txmsg test ingress parser2" , test_txmsg_ingress_parser2}, |
1875 | }; |
1876 | |
1877 | static int check_whitelist(struct _test *t, struct sockmap_options *opt) |
1878 | { |
1879 | char *entry, *ptr; |
1880 | |
1881 | if (!opt->whitelist) |
1882 | return 0; |
1883 | ptr = strdup(opt->whitelist); |
1884 | if (!ptr) |
1885 | return -ENOMEM; |
1886 | entry = strtok(ptr, "," ); |
1887 | while (entry) { |
1888 | if ((opt->prepend && strstr(opt->prepend, entry) != 0) || |
1889 | strstr(opt->map, entry) != 0 || |
1890 | strstr(t->title, entry) != 0) |
1891 | return 0; |
1892 | entry = strtok(NULL, "," ); |
1893 | } |
1894 | return -EINVAL; |
1895 | } |
1896 | |
1897 | static int check_blacklist(struct _test *t, struct sockmap_options *opt) |
1898 | { |
1899 | char *entry, *ptr; |
1900 | |
1901 | if (!opt->blacklist) |
1902 | return -EINVAL; |
1903 | ptr = strdup(opt->blacklist); |
1904 | if (!ptr) |
1905 | return -ENOMEM; |
1906 | entry = strtok(ptr, "," ); |
1907 | while (entry) { |
1908 | if ((opt->prepend && strstr(opt->prepend, entry) != 0) || |
1909 | strstr(opt->map, entry) != 0 || |
1910 | strstr(t->title, entry) != 0) |
1911 | return 0; |
1912 | entry = strtok(NULL, "," ); |
1913 | } |
1914 | return -EINVAL; |
1915 | } |
1916 | |
1917 | static int __test_selftests(int cg_fd, struct sockmap_options *opt) |
1918 | { |
1919 | int i, err; |
1920 | |
1921 | err = populate_progs(bpf_file: opt->map); |
1922 | if (err < 0) { |
1923 | fprintf(stderr, "ERROR: (%i) load bpf failed\n" , err); |
1924 | return err; |
1925 | } |
1926 | |
1927 | /* Tests basic commands and APIs */ |
1928 | for (i = 0; i < ARRAY_SIZE(test); i++) { |
1929 | struct _test t = test[i]; |
1930 | |
1931 | if (check_whitelist(t: &t, opt) != 0) |
1932 | continue; |
1933 | if (check_blacklist(t: &t, opt) == 0) |
1934 | continue; |
1935 | |
1936 | test_start_subtest(t: &t, o: opt); |
1937 | t.tester(cg_fd, opt); |
1938 | test_end_subtest(); |
1939 | } |
1940 | |
1941 | return err; |
1942 | } |
1943 | |
1944 | static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) |
1945 | { |
1946 | opt->map = BPF_SOCKMAP_FILENAME; |
1947 | __test_selftests(cg_fd, opt); |
1948 | } |
1949 | |
1950 | static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) |
1951 | { |
1952 | opt->map = BPF_SOCKHASH_FILENAME; |
1953 | __test_selftests(cg_fd, opt); |
1954 | } |
1955 | |
1956 | static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) |
1957 | { |
1958 | opt->map = BPF_SOCKHASH_FILENAME; |
1959 | opt->prepend = "ktls" ; |
1960 | ktls = 1; |
1961 | __test_selftests(cg_fd, opt); |
1962 | ktls = 0; |
1963 | } |
1964 | |
1965 | static int test_selftest(int cg_fd, struct sockmap_options *opt) |
1966 | { |
1967 | |
1968 | test_selftests_sockmap(cg_fd, opt); |
1969 | test_selftests_sockhash(cg_fd, opt); |
1970 | test_selftests_ktls(cg_fd, opt); |
1971 | test_print_results(); |
1972 | return 0; |
1973 | } |
1974 | |
1975 | int main(int argc, char **argv) |
1976 | { |
1977 | int iov_count = 1, length = 1024, rate = 1; |
1978 | struct sockmap_options options = {0}; |
1979 | int opt, longindex, err, cg_fd = 0; |
1980 | char *bpf_file = BPF_SOCKMAP_FILENAME; |
1981 | int test = SELFTESTS; |
1982 | bool cg_created = 0; |
1983 | |
1984 | while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:" , |
1985 | long_options, &longindex)) != -1) { |
1986 | switch (opt) { |
1987 | case 's': |
1988 | txmsg_start = atoi(optarg); |
1989 | break; |
1990 | case 'e': |
1991 | txmsg_end = atoi(optarg); |
1992 | break; |
1993 | case 'p': |
1994 | txmsg_start_push = atoi(optarg); |
1995 | break; |
1996 | case 'q': |
1997 | txmsg_end_push = atoi(optarg); |
1998 | break; |
1999 | case 'w': |
2000 | txmsg_start_pop = atoi(optarg); |
2001 | break; |
2002 | case 'x': |
2003 | txmsg_pop = atoi(optarg); |
2004 | break; |
2005 | case 'a': |
2006 | txmsg_apply = atoi(optarg); |
2007 | break; |
2008 | case 'k': |
2009 | txmsg_cork = atoi(optarg); |
2010 | break; |
2011 | case 'c': |
2012 | cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); |
2013 | if (cg_fd < 0) { |
2014 | fprintf(stderr, |
2015 | "ERROR: (%i) open cg path failed: %s\n" , |
2016 | cg_fd, optarg); |
2017 | return cg_fd; |
2018 | } |
2019 | break; |
2020 | case 'r': |
2021 | rate = atoi(optarg); |
2022 | break; |
2023 | case 'v': |
2024 | options.verbose = 1; |
2025 | if (optarg) |
2026 | options.verbose = atoi(optarg); |
2027 | break; |
2028 | case 'i': |
2029 | iov_count = atoi(optarg); |
2030 | break; |
2031 | case 'l': |
2032 | length = atoi(optarg); |
2033 | break; |
2034 | case 'd': |
2035 | options.data_test = true; |
2036 | break; |
2037 | case 't': |
2038 | if (strcmp(optarg, "ping" ) == 0) { |
2039 | test = PING_PONG; |
2040 | } else if (strcmp(optarg, "sendmsg" ) == 0) { |
2041 | test = SENDMSG; |
2042 | } else if (strcmp(optarg, "base" ) == 0) { |
2043 | test = BASE; |
2044 | } else if (strcmp(optarg, "base_sendpage" ) == 0) { |
2045 | test = BASE_SENDPAGE; |
2046 | } else if (strcmp(optarg, "sendpage" ) == 0) { |
2047 | test = SENDPAGE; |
2048 | } else { |
2049 | usage(argv); |
2050 | return -1; |
2051 | } |
2052 | break; |
2053 | case 'n': |
2054 | options.whitelist = strdup(optarg); |
2055 | if (!options.whitelist) |
2056 | return -ENOMEM; |
2057 | break; |
2058 | case 'b': |
2059 | options.blacklist = strdup(optarg); |
2060 | if (!options.blacklist) |
2061 | return -ENOMEM; |
2062 | case 0: |
2063 | break; |
2064 | case 'h': |
2065 | default: |
2066 | usage(argv); |
2067 | return -1; |
2068 | } |
2069 | } |
2070 | |
2071 | if (!cg_fd) { |
2072 | cg_fd = cgroup_setup_and_join(CG_PATH); |
2073 | if (cg_fd < 0) |
2074 | return cg_fd; |
2075 | cg_created = 1; |
2076 | } |
2077 | |
2078 | /* Use libbpf 1.0 API mode */ |
2079 | libbpf_set_strict_mode(LIBBPF_STRICT_ALL); |
2080 | |
2081 | if (test == SELFTESTS) { |
2082 | err = test_selftest(cg_fd, opt: &options); |
2083 | goto out; |
2084 | } |
2085 | |
2086 | err = populate_progs(bpf_file); |
2087 | if (err) { |
2088 | fprintf(stderr, "populate program: (%s) %s\n" , |
2089 | bpf_file, strerror(errno)); |
2090 | return 1; |
2091 | } |
2092 | running = 1; |
2093 | |
2094 | /* catch SIGINT */ |
2095 | signal(SIGINT, running_handler); |
2096 | |
2097 | options.iov_count = iov_count; |
2098 | options.iov_length = length; |
2099 | options.rate = rate; |
2100 | |
2101 | err = run_options(options: &options, cg_fd, test); |
2102 | out: |
2103 | if (options.whitelist) |
2104 | free(options.whitelist); |
2105 | if (options.blacklist) |
2106 | free(options.blacklist); |
2107 | if (cg_created) |
2108 | cleanup_cgroup_environment(); |
2109 | close(cg_fd); |
2110 | return err; |
2111 | } |
2112 | |
2113 | void running_handler(int a) |
2114 | { |
2115 | running = 0; |
2116 | } |
2117 | |