1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | #define _GNU_SOURCE |
4 | |
5 | #include <arpa/inet.h> |
6 | #include <errno.h> |
7 | #include <error.h> |
8 | #include <linux/errqueue.h> |
9 | #include <linux/net_tstamp.h> |
10 | #include <netinet/if_ether.h> |
11 | #include <netinet/in.h> |
12 | #include <netinet/ip.h> |
13 | #include <netinet/ip6.h> |
14 | #include <netinet/udp.h> |
15 | #include <poll.h> |
16 | #include <sched.h> |
17 | #include <signal.h> |
18 | #include <stdbool.h> |
19 | #include <stdio.h> |
20 | #include <stdlib.h> |
21 | #include <string.h> |
22 | #include <sys/socket.h> |
23 | #include <sys/time.h> |
24 | #include <sys/poll.h> |
25 | #include <sys/types.h> |
26 | #include <unistd.h> |
27 | |
28 | #include "../kselftest.h" |
29 | |
30 | #ifndef ETH_MAX_MTU |
31 | #define ETH_MAX_MTU 0xFFFFU |
32 | #endif |
33 | |
34 | #ifndef UDP_SEGMENT |
35 | #define UDP_SEGMENT 103 |
36 | #endif |
37 | |
38 | #ifndef SO_ZEROCOPY |
39 | #define SO_ZEROCOPY 60 |
40 | #endif |
41 | |
42 | #ifndef SO_EE_ORIGIN_ZEROCOPY |
43 | #define SO_EE_ORIGIN_ZEROCOPY 5 |
44 | #endif |
45 | |
46 | #ifndef MSG_ZEROCOPY |
47 | #define MSG_ZEROCOPY 0x4000000 |
48 | #endif |
49 | |
50 | #ifndef ENOTSUPP |
51 | #define ENOTSUPP 524 |
52 | #endif |
53 | |
54 | #define NUM_PKT 100 |
55 | |
56 | static bool cfg_cache_trash; |
57 | static int cfg_cpu = -1; |
58 | static int cfg_connected = true; |
59 | static int cfg_family = PF_UNSPEC; |
60 | static uint16_t cfg_mss; |
61 | static int cfg_payload_len = (1472 * 42); |
62 | static int cfg_port = 8000; |
63 | static int cfg_runtime_ms = -1; |
64 | static bool cfg_poll; |
65 | static int cfg_poll_loop_timeout_ms = 2000; |
66 | static bool cfg_segment; |
67 | static bool cfg_sendmmsg; |
68 | static bool cfg_tcp; |
69 | static uint32_t cfg_tx_ts = SOF_TIMESTAMPING_TX_SOFTWARE; |
70 | static bool cfg_tx_tstamp; |
71 | static bool cfg_audit; |
72 | static bool cfg_verbose; |
73 | static bool cfg_zerocopy; |
74 | static int cfg_msg_nr; |
75 | static uint16_t cfg_gso_size; |
76 | static unsigned long total_num_msgs; |
77 | static unsigned long total_num_sends; |
78 | static unsigned long stat_tx_ts; |
79 | static unsigned long stat_tx_ts_errors; |
80 | static unsigned long tstart; |
81 | static unsigned long tend; |
82 | static unsigned long stat_zcopies; |
83 | |
84 | static socklen_t cfg_alen; |
85 | static struct sockaddr_storage cfg_dst_addr; |
86 | |
87 | static bool interrupted; |
88 | static char buf[NUM_PKT][ETH_MAX_MTU]; |
89 | |
90 | static void sigint_handler(int signum) |
91 | { |
92 | if (signum == SIGINT) |
93 | interrupted = true; |
94 | } |
95 | |
96 | static unsigned long gettimeofday_ms(void) |
97 | { |
98 | struct timeval tv; |
99 | |
100 | gettimeofday(&tv, NULL); |
101 | return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); |
102 | } |
103 | |
104 | static int set_cpu(int cpu) |
105 | { |
106 | cpu_set_t mask; |
107 | |
108 | CPU_ZERO(&mask); |
109 | CPU_SET(cpu, &mask); |
110 | if (sched_setaffinity(0, sizeof(mask), &mask)) |
111 | error(1, 0, "setaffinity %d" , cpu); |
112 | |
113 | return 0; |
114 | } |
115 | |
116 | static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr) |
117 | { |
118 | struct sockaddr_in6 *addr6 = (void *) sockaddr; |
119 | struct sockaddr_in *addr4 = (void *) sockaddr; |
120 | |
121 | switch (domain) { |
122 | case PF_INET: |
123 | addr4->sin_family = AF_INET; |
124 | addr4->sin_port = htons(cfg_port); |
125 | if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) |
126 | error(1, 0, "ipv4 parse error: %s" , str_addr); |
127 | break; |
128 | case PF_INET6: |
129 | addr6->sin6_family = AF_INET6; |
130 | addr6->sin6_port = htons(cfg_port); |
131 | if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) |
132 | error(1, 0, "ipv6 parse error: %s" , str_addr); |
133 | break; |
134 | default: |
135 | error(1, 0, "illegal domain" ); |
136 | } |
137 | } |
138 | |
139 | static void flush_cmsg(struct cmsghdr *cmsg) |
140 | { |
141 | struct sock_extended_err *err; |
142 | struct scm_timestamping *tss; |
143 | __u32 lo; |
144 | __u32 hi; |
145 | int i; |
146 | |
147 | switch (cmsg->cmsg_level) { |
148 | case SOL_SOCKET: |
149 | if (cmsg->cmsg_type == SO_TIMESTAMPING) { |
150 | i = (cfg_tx_ts == SOF_TIMESTAMPING_TX_HARDWARE) ? 2 : 0; |
151 | tss = (struct scm_timestamping *)CMSG_DATA(cmsg); |
152 | if (tss->ts[i].tv_sec == 0) |
153 | stat_tx_ts_errors++; |
154 | } else { |
155 | error(1, 0, "unknown SOL_SOCKET cmsg type=%u\n" , |
156 | cmsg->cmsg_type); |
157 | } |
158 | break; |
159 | case SOL_IP: |
160 | case SOL_IPV6: |
161 | switch (cmsg->cmsg_type) { |
162 | case IP_RECVERR: |
163 | case IPV6_RECVERR: |
164 | { |
165 | err = (struct sock_extended_err *)CMSG_DATA(cmsg); |
166 | switch (err->ee_origin) { |
167 | case SO_EE_ORIGIN_TIMESTAMPING: |
168 | /* Got a TX timestamp from error queue */ |
169 | stat_tx_ts++; |
170 | break; |
171 | case SO_EE_ORIGIN_ICMP: |
172 | case SO_EE_ORIGIN_ICMP6: |
173 | if (cfg_verbose) |
174 | fprintf(stderr, |
175 | "received ICMP error: type=%u, code=%u\n" , |
176 | err->ee_type, err->ee_code); |
177 | break; |
178 | case SO_EE_ORIGIN_ZEROCOPY: |
179 | { |
180 | lo = err->ee_info; |
181 | hi = err->ee_data; |
182 | /* range of IDs acknowledged */ |
183 | stat_zcopies += hi - lo + 1; |
184 | break; |
185 | } |
186 | case SO_EE_ORIGIN_LOCAL: |
187 | if (cfg_verbose) |
188 | fprintf(stderr, |
189 | "received packet with local origin: %u\n" , |
190 | err->ee_origin); |
191 | break; |
192 | default: |
193 | error(0, 1, "received packet with origin: %u" , |
194 | err->ee_origin); |
195 | } |
196 | break; |
197 | } |
198 | default: |
199 | error(0, 1, "unknown IP msg type=%u\n" , |
200 | cmsg->cmsg_type); |
201 | break; |
202 | } |
203 | break; |
204 | default: |
205 | error(0, 1, "unknown cmsg level=%u\n" , |
206 | cmsg->cmsg_level); |
207 | } |
208 | } |
209 | |
210 | static void flush_errqueue_recv(int fd) |
211 | { |
212 | char control[CMSG_SPACE(sizeof(struct scm_timestamping)) + |
213 | CMSG_SPACE(sizeof(struct sock_extended_err)) + |
214 | CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0}; |
215 | struct msghdr msg = {0}; |
216 | struct cmsghdr *cmsg; |
217 | int ret; |
218 | |
219 | while (1) { |
220 | msg.msg_control = control; |
221 | msg.msg_controllen = sizeof(control); |
222 | ret = recvmsg(fd, &msg, MSG_ERRQUEUE); |
223 | if (ret == -1 && errno == EAGAIN) |
224 | break; |
225 | if (ret == -1) |
226 | error(1, errno, "errqueue" ); |
227 | if (msg.msg_flags != MSG_ERRQUEUE) |
228 | error(1, 0, "errqueue: flags 0x%x\n" , msg.msg_flags); |
229 | if (cfg_audit) { |
230 | for (cmsg = CMSG_FIRSTHDR(&msg); |
231 | cmsg; |
232 | cmsg = CMSG_NXTHDR(&msg, cmsg)) |
233 | flush_cmsg(cmsg); |
234 | } |
235 | msg.msg_flags = 0; |
236 | } |
237 | } |
238 | |
239 | static void flush_errqueue(int fd, const bool do_poll, |
240 | unsigned long poll_timeout, const bool poll_err) |
241 | { |
242 | if (do_poll) { |
243 | struct pollfd fds = {0}; |
244 | int ret; |
245 | |
246 | fds.fd = fd; |
247 | ret = poll(&fds, 1, poll_timeout); |
248 | if (ret == 0) { |
249 | if ((cfg_verbose) && (poll_err)) |
250 | fprintf(stderr, "poll timeout\n" ); |
251 | } else if (ret < 0) { |
252 | error(1, errno, "poll" ); |
253 | } |
254 | } |
255 | |
256 | flush_errqueue_recv(fd); |
257 | } |
258 | |
259 | static void flush_errqueue_retry(int fd, unsigned long num_sends) |
260 | { |
261 | unsigned long tnow, tstop; |
262 | bool first_try = true; |
263 | |
264 | tnow = gettimeofday_ms(); |
265 | tstop = tnow + cfg_poll_loop_timeout_ms; |
266 | do { |
267 | flush_errqueue(fd, do_poll: true, poll_timeout: tstop - tnow, poll_err: first_try); |
268 | first_try = false; |
269 | tnow = gettimeofday_ms(); |
270 | } while ((stat_zcopies != num_sends) && (tnow < tstop)); |
271 | } |
272 | |
273 | static int send_tcp(int fd, char *data) |
274 | { |
275 | int ret, done = 0, count = 0; |
276 | |
277 | while (done < cfg_payload_len) { |
278 | ret = send(fd, data + done, cfg_payload_len - done, |
279 | cfg_zerocopy ? MSG_ZEROCOPY : 0); |
280 | if (ret == -1) |
281 | error(1, errno, "write" ); |
282 | |
283 | done += ret; |
284 | count++; |
285 | } |
286 | |
287 | return count; |
288 | } |
289 | |
290 | static int send_udp(int fd, char *data) |
291 | { |
292 | int ret, total_len, len, count = 0; |
293 | |
294 | total_len = cfg_payload_len; |
295 | |
296 | while (total_len) { |
297 | len = total_len < cfg_mss ? total_len : cfg_mss; |
298 | |
299 | ret = sendto(fd, data, len, cfg_zerocopy ? MSG_ZEROCOPY : 0, |
300 | cfg_connected ? NULL : (void *)&cfg_dst_addr, |
301 | cfg_connected ? 0 : cfg_alen); |
302 | if (ret == -1) |
303 | error(1, errno, "write" ); |
304 | if (ret != len) |
305 | error(1, errno, "write: %uB != %uB\n" , ret, len); |
306 | |
307 | total_len -= len; |
308 | count++; |
309 | } |
310 | |
311 | return count; |
312 | } |
313 | |
314 | static void send_ts_cmsg(struct cmsghdr *cm) |
315 | { |
316 | uint32_t *valp; |
317 | |
318 | cm->cmsg_level = SOL_SOCKET; |
319 | cm->cmsg_type = SO_TIMESTAMPING; |
320 | cm->cmsg_len = CMSG_LEN(sizeof(cfg_tx_ts)); |
321 | valp = (void *)CMSG_DATA(cm); |
322 | *valp = cfg_tx_ts; |
323 | } |
324 | |
325 | static int send_udp_sendmmsg(int fd, char *data) |
326 | { |
327 | char control[CMSG_SPACE(sizeof(cfg_tx_ts))] = {0}; |
328 | const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN; |
329 | struct mmsghdr mmsgs[max_nr_msg]; |
330 | struct iovec iov[max_nr_msg]; |
331 | unsigned int off = 0, left; |
332 | size_t msg_controllen = 0; |
333 | int i = 0, ret; |
334 | |
335 | memset(mmsgs, 0, sizeof(mmsgs)); |
336 | |
337 | if (cfg_tx_tstamp) { |
338 | struct msghdr msg = {0}; |
339 | struct cmsghdr *cmsg; |
340 | |
341 | msg.msg_control = control; |
342 | msg.msg_controllen = sizeof(control); |
343 | cmsg = CMSG_FIRSTHDR(&msg); |
344 | send_ts_cmsg(cm: cmsg); |
345 | msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts)); |
346 | } |
347 | |
348 | left = cfg_payload_len; |
349 | while (left) { |
350 | if (i == max_nr_msg) |
351 | error(1, 0, "sendmmsg: exceeds max_nr_msg" ); |
352 | |
353 | iov[i].iov_base = data + off; |
354 | iov[i].iov_len = cfg_mss < left ? cfg_mss : left; |
355 | |
356 | mmsgs[i].msg_hdr.msg_iov = iov + i; |
357 | mmsgs[i].msg_hdr.msg_iovlen = 1; |
358 | |
359 | mmsgs[i].msg_hdr.msg_name = (void *)&cfg_dst_addr; |
360 | mmsgs[i].msg_hdr.msg_namelen = cfg_alen; |
361 | if (msg_controllen) { |
362 | mmsgs[i].msg_hdr.msg_control = control; |
363 | mmsgs[i].msg_hdr.msg_controllen = msg_controllen; |
364 | } |
365 | |
366 | off += iov[i].iov_len; |
367 | left -= iov[i].iov_len; |
368 | i++; |
369 | } |
370 | |
371 | ret = sendmmsg(fd, mmsgs, i, cfg_zerocopy ? MSG_ZEROCOPY : 0); |
372 | if (ret == -1) |
373 | error(1, errno, "sendmmsg" ); |
374 | |
375 | return ret; |
376 | } |
377 | |
378 | static void send_udp_segment_cmsg(struct cmsghdr *cm) |
379 | { |
380 | uint16_t *valp; |
381 | |
382 | cm->cmsg_level = SOL_UDP; |
383 | cm->cmsg_type = UDP_SEGMENT; |
384 | cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size)); |
385 | valp = (void *)CMSG_DATA(cm); |
386 | *valp = cfg_gso_size; |
387 | } |
388 | |
389 | static int send_udp_segment(int fd, char *data) |
390 | { |
391 | char control[CMSG_SPACE(sizeof(cfg_gso_size)) + |
392 | CMSG_SPACE(sizeof(cfg_tx_ts))] = {0}; |
393 | struct msghdr msg = {0}; |
394 | struct iovec iov = {0}; |
395 | size_t msg_controllen; |
396 | struct cmsghdr *cmsg; |
397 | int ret; |
398 | |
399 | iov.iov_base = data; |
400 | iov.iov_len = cfg_payload_len; |
401 | |
402 | msg.msg_iov = &iov; |
403 | msg.msg_iovlen = 1; |
404 | |
405 | msg.msg_control = control; |
406 | msg.msg_controllen = sizeof(control); |
407 | cmsg = CMSG_FIRSTHDR(&msg); |
408 | send_udp_segment_cmsg(cm: cmsg); |
409 | msg_controllen = CMSG_SPACE(sizeof(cfg_mss)); |
410 | if (cfg_tx_tstamp) { |
411 | cmsg = CMSG_NXTHDR(&msg, cmsg); |
412 | send_ts_cmsg(cm: cmsg); |
413 | msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts)); |
414 | } |
415 | |
416 | msg.msg_controllen = msg_controllen; |
417 | msg.msg_name = (void *)&cfg_dst_addr; |
418 | msg.msg_namelen = cfg_alen; |
419 | |
420 | ret = sendmsg(fd, &msg, cfg_zerocopy ? MSG_ZEROCOPY : 0); |
421 | if (ret == -1) |
422 | error(1, errno, "sendmsg" ); |
423 | if (ret != iov.iov_len) |
424 | error(1, 0, "sendmsg: %u != %llu\n" , ret, |
425 | (unsigned long long)iov.iov_len); |
426 | |
427 | return 1; |
428 | } |
429 | |
430 | static void usage(const char *filepath) |
431 | { |
432 | error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] " |
433 | "[-L secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]" , |
434 | filepath); |
435 | } |
436 | |
437 | static void parse_opts(int argc, char **argv) |
438 | { |
439 | const char *bind_addr = NULL; |
440 | int max_len, hdrlen; |
441 | int c; |
442 | |
443 | while ((c = getopt(argc, argv, "46acC:D:Hl:L:mM:p:s:PS:tTuvz" )) != -1) { |
444 | switch (c) { |
445 | case '4': |
446 | if (cfg_family != PF_UNSPEC) |
447 | error(1, 0, "Pass one of -4 or -6" ); |
448 | cfg_family = PF_INET; |
449 | cfg_alen = sizeof(struct sockaddr_in); |
450 | break; |
451 | case '6': |
452 | if (cfg_family != PF_UNSPEC) |
453 | error(1, 0, "Pass one of -4 or -6" ); |
454 | cfg_family = PF_INET6; |
455 | cfg_alen = sizeof(struct sockaddr_in6); |
456 | break; |
457 | case 'a': |
458 | cfg_audit = true; |
459 | break; |
460 | case 'c': |
461 | cfg_cache_trash = true; |
462 | break; |
463 | case 'C': |
464 | cfg_cpu = strtol(optarg, NULL, 0); |
465 | break; |
466 | case 'D': |
467 | bind_addr = optarg; |
468 | break; |
469 | case 'l': |
470 | cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000; |
471 | break; |
472 | case 'L': |
473 | cfg_poll_loop_timeout_ms = strtoul(optarg, NULL, 10) * 1000; |
474 | break; |
475 | case 'm': |
476 | cfg_sendmmsg = true; |
477 | break; |
478 | case 'M': |
479 | cfg_msg_nr = strtoul(optarg, NULL, 10); |
480 | break; |
481 | case 'p': |
482 | cfg_port = strtoul(optarg, NULL, 0); |
483 | break; |
484 | case 'P': |
485 | cfg_poll = true; |
486 | break; |
487 | case 's': |
488 | cfg_payload_len = strtoul(optarg, NULL, 0); |
489 | break; |
490 | case 'S': |
491 | cfg_gso_size = strtoul(optarg, NULL, 0); |
492 | cfg_segment = true; |
493 | break; |
494 | case 'H': |
495 | cfg_tx_ts = SOF_TIMESTAMPING_TX_HARDWARE; |
496 | cfg_tx_tstamp = true; |
497 | break; |
498 | case 't': |
499 | cfg_tcp = true; |
500 | break; |
501 | case 'T': |
502 | cfg_tx_tstamp = true; |
503 | break; |
504 | case 'u': |
505 | cfg_connected = false; |
506 | break; |
507 | case 'v': |
508 | cfg_verbose = true; |
509 | break; |
510 | case 'z': |
511 | cfg_zerocopy = true; |
512 | break; |
513 | default: |
514 | exit(1); |
515 | } |
516 | } |
517 | |
518 | if (!bind_addr) |
519 | bind_addr = cfg_family == PF_INET6 ? "::" : "0.0.0.0" ; |
520 | |
521 | setup_sockaddr(domain: cfg_family, str_addr: bind_addr, sockaddr: &cfg_dst_addr); |
522 | |
523 | if (optind != argc) |
524 | usage(filepath: argv[0]); |
525 | |
526 | if (cfg_family == PF_UNSPEC) |
527 | error(1, 0, "must pass one of -4 or -6" ); |
528 | if (cfg_tcp && !cfg_connected) |
529 | error(1, 0, "connectionless tcp makes no sense" ); |
530 | if (cfg_segment && cfg_sendmmsg) |
531 | error(1, 0, "cannot combine segment offload and sendmmsg" ); |
532 | if (cfg_tx_tstamp && !(cfg_segment || cfg_sendmmsg)) |
533 | error(1, 0, "Options -T and -H require either -S or -m option" ); |
534 | |
535 | if (cfg_family == PF_INET) |
536 | hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr); |
537 | else |
538 | hdrlen = sizeof(struct ip6_hdr) + sizeof(struct udphdr); |
539 | |
540 | cfg_mss = ETH_DATA_LEN - hdrlen; |
541 | max_len = ETH_MAX_MTU - hdrlen; |
542 | if (!cfg_gso_size) |
543 | cfg_gso_size = cfg_mss; |
544 | |
545 | if (cfg_payload_len > max_len) |
546 | error(1, 0, "payload length %u exceeds max %u" , |
547 | cfg_payload_len, max_len); |
548 | } |
549 | |
550 | static void set_pmtu_discover(int fd, bool is_ipv4) |
551 | { |
552 | int level, name, val; |
553 | |
554 | if (is_ipv4) { |
555 | level = SOL_IP; |
556 | name = IP_MTU_DISCOVER; |
557 | val = IP_PMTUDISC_DO; |
558 | } else { |
559 | level = SOL_IPV6; |
560 | name = IPV6_MTU_DISCOVER; |
561 | val = IPV6_PMTUDISC_DO; |
562 | } |
563 | |
564 | if (setsockopt(fd, level, name, &val, sizeof(val))) |
565 | error(1, errno, "setsockopt path mtu" ); |
566 | } |
567 | |
568 | static void set_tx_timestamping(int fd) |
569 | { |
570 | int val = SOF_TIMESTAMPING_OPT_CMSG | SOF_TIMESTAMPING_OPT_ID | |
571 | SOF_TIMESTAMPING_OPT_TSONLY; |
572 | |
573 | if (cfg_tx_ts == SOF_TIMESTAMPING_TX_SOFTWARE) |
574 | val |= SOF_TIMESTAMPING_SOFTWARE; |
575 | else |
576 | val |= SOF_TIMESTAMPING_RAW_HARDWARE; |
577 | |
578 | if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val))) |
579 | error(1, errno, "setsockopt tx timestamping" ); |
580 | } |
581 | |
582 | static void print_audit_report(unsigned long num_msgs, unsigned long num_sends) |
583 | { |
584 | unsigned long tdelta; |
585 | |
586 | tdelta = tend - tstart; |
587 | if (!tdelta) |
588 | return; |
589 | |
590 | fprintf(stderr, "Summary over %lu.%03lu seconds...\n" , |
591 | tdelta / 1000, tdelta % 1000); |
592 | fprintf(stderr, |
593 | "sum %s tx: %6lu MB/s %10lu calls (%lu/s) %10lu msgs (%lu/s)\n" , |
594 | cfg_tcp ? "tcp" : "udp" , |
595 | ((num_msgs * cfg_payload_len) >> 10) / tdelta, |
596 | num_sends, num_sends * 1000 / tdelta, |
597 | num_msgs, num_msgs * 1000 / tdelta); |
598 | |
599 | if (cfg_tx_tstamp) { |
600 | if (stat_tx_ts_errors) |
601 | error(1, 0, |
602 | "Expected clean TX Timestamps: %9lu msgs received %6lu errors" , |
603 | stat_tx_ts, stat_tx_ts_errors); |
604 | if (stat_tx_ts != num_sends) |
605 | error(1, 0, |
606 | "Unexpected number of TX Timestamps: %9lu expected %9lu received" , |
607 | num_sends, stat_tx_ts); |
608 | fprintf(stderr, |
609 | "Tx Timestamps: %19lu received %17lu errors\n" , |
610 | stat_tx_ts, stat_tx_ts_errors); |
611 | } |
612 | |
613 | if (cfg_zerocopy) { |
614 | if (stat_zcopies != num_sends) |
615 | error(1, 0, "Unexpected number of Zerocopy completions: %9lu expected %9lu received" , |
616 | num_sends, stat_zcopies); |
617 | fprintf(stderr, |
618 | "Zerocopy acks: %19lu\n" , |
619 | stat_zcopies); |
620 | } |
621 | } |
622 | |
623 | static void print_report(unsigned long num_msgs, unsigned long num_sends) |
624 | { |
625 | fprintf(stderr, |
626 | "%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n" , |
627 | cfg_tcp ? "tcp" : "udp" , |
628 | (num_msgs * cfg_payload_len) >> 20, |
629 | num_sends, num_msgs); |
630 | |
631 | if (cfg_audit) { |
632 | total_num_msgs += num_msgs; |
633 | total_num_sends += num_sends; |
634 | } |
635 | } |
636 | |
637 | int main(int argc, char **argv) |
638 | { |
639 | unsigned long num_msgs, num_sends; |
640 | unsigned long tnow, treport, tstop; |
641 | int fd, i, val, ret; |
642 | |
643 | parse_opts(argc, argv); |
644 | |
645 | if (cfg_cpu > 0) |
646 | set_cpu(cfg_cpu); |
647 | |
648 | for (i = 0; i < sizeof(buf[0]); i++) |
649 | buf[0][i] = 'a' + (i % 26); |
650 | for (i = 1; i < NUM_PKT; i++) |
651 | memcpy(buf[i], buf[0], sizeof(buf[0])); |
652 | |
653 | signal(SIGINT, sigint_handler); |
654 | |
655 | fd = socket(cfg_family, cfg_tcp ? SOCK_STREAM : SOCK_DGRAM, 0); |
656 | if (fd == -1) |
657 | error(1, errno, "socket" ); |
658 | |
659 | if (cfg_zerocopy) { |
660 | val = 1; |
661 | |
662 | ret = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, |
663 | &val, sizeof(val)); |
664 | if (ret) { |
665 | if (errno == ENOPROTOOPT || errno == ENOTSUPP) { |
666 | fprintf(stderr, "SO_ZEROCOPY not supported" ); |
667 | exit(KSFT_SKIP); |
668 | } |
669 | error(1, errno, "setsockopt zerocopy" ); |
670 | } |
671 | } |
672 | |
673 | if (cfg_connected && |
674 | connect(fd, (void *)&cfg_dst_addr, cfg_alen)) |
675 | error(1, errno, "connect" ); |
676 | |
677 | if (cfg_segment) |
678 | set_pmtu_discover(fd, is_ipv4: cfg_family == PF_INET); |
679 | |
680 | if (cfg_tx_tstamp) |
681 | set_tx_timestamping(fd); |
682 | |
683 | num_msgs = num_sends = 0; |
684 | tnow = gettimeofday_ms(); |
685 | tstart = tnow; |
686 | tend = tnow; |
687 | tstop = tnow + cfg_runtime_ms; |
688 | treport = tnow + 1000; |
689 | |
690 | i = 0; |
691 | do { |
692 | if (cfg_tcp) |
693 | num_sends += send_tcp(fd, data: buf[i]); |
694 | else if (cfg_segment) |
695 | num_sends += send_udp_segment(fd, data: buf[i]); |
696 | else if (cfg_sendmmsg) |
697 | num_sends += send_udp_sendmmsg(fd, data: buf[i]); |
698 | else |
699 | num_sends += send_udp(fd, data: buf[i]); |
700 | num_msgs++; |
701 | if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp) |
702 | flush_errqueue(fd, do_poll: cfg_poll, poll_timeout: 500, poll_err: true); |
703 | |
704 | if (cfg_msg_nr && num_msgs >= cfg_msg_nr) |
705 | break; |
706 | |
707 | tnow = gettimeofday_ms(); |
708 | if (tnow >= treport) { |
709 | print_report(num_msgs, num_sends); |
710 | num_msgs = num_sends = 0; |
711 | treport = tnow + 1000; |
712 | } |
713 | |
714 | /* cold cache when writing buffer */ |
715 | if (cfg_cache_trash) |
716 | i = ++i < NUM_PKT ? i : 0; |
717 | |
718 | } while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop)); |
719 | |
720 | if (cfg_zerocopy || cfg_tx_tstamp) |
721 | flush_errqueue_retry(fd, num_sends); |
722 | |
723 | if (close(fd)) |
724 | error(1, errno, "close" ); |
725 | |
726 | if (cfg_audit) { |
727 | tend = tnow; |
728 | total_num_msgs += num_msgs; |
729 | total_num_sends += num_sends; |
730 | print_audit_report(num_msgs: total_num_msgs, num_sends: total_num_sends); |
731 | } |
732 | |
733 | return 0; |
734 | } |
735 | |