1// SPDX-License-Identifier: GPL-2.0
2/*
3 * delaytop.c - system-wide delay monitoring tool.
4 *
5 * This tool provides real-time monitoring and statistics of
6 * system, container, and task-level delays, including CPU,
7 * memory, IO, and IRQ. It supports both interactive (top-like),
8 * and can output delay information for the whole system, specific
9 * containers (cgroups), or individual tasks (PIDs).
10 *
11 * Key features:
12 * - Collects per-task delay accounting statistics via taskstats.
13 * - Collects system-wide PSI information.
14 * - Supports sorting, filtering.
15 * - Supports both interactive (screen refresh).
16 *
17 * Copyright (C) Fan Yu, ZTE Corp. 2025
18 * Copyright (C) Wang Yaxin, ZTE Corp. 2025
19 *
20 * Compile with
21 * gcc -I/usr/src/linux/include delaytop.c -o delaytop
22 */
23
24#include <stdio.h>
25#include <stdlib.h>
26#include <string.h>
27#include <errno.h>
28#include <unistd.h>
29#include <fcntl.h>
30#include <getopt.h>
31#include <signal.h>
32#include <time.h>
33#include <dirent.h>
34#include <ctype.h>
35#include <stdbool.h>
36#include <sys/types.h>
37#include <sys/stat.h>
38#include <sys/socket.h>
39#include <sys/select.h>
40#include <termios.h>
41#include <limits.h>
42#include <linux/genetlink.h>
43#include <linux/taskstats.h>
44#include <linux/cgroupstats.h>
45#include <stddef.h>
46
47#define PSI_PATH "/proc/pressure"
48#define PSI_CPU_PATH "/proc/pressure/cpu"
49#define PSI_MEMORY_PATH "/proc/pressure/memory"
50#define PSI_IO_PATH "/proc/pressure/io"
51#define PSI_IRQ_PATH "/proc/pressure/irq"
52
53#define NLA_NEXT(na) ((struct nlattr *)((char *)(na) + NLA_ALIGN((na)->nla_len)))
54#define NLA_DATA(na) ((void *)((char *)(na) + NLA_HDRLEN))
55#define NLA_PAYLOAD(len) (len - NLA_HDRLEN)
56
57#define GENLMSG_DATA(glh) ((void *)(NLMSG_DATA(glh) + GENL_HDRLEN))
58#define GENLMSG_PAYLOAD(glh) (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN)
59
60#define TASK_COMM_LEN 16
61#define MAX_MSG_SIZE 1024
62#define MAX_TASKS 1000
63#define MAX_BUF_LEN 256
64#define SET_TASK_STAT(task_count, field) tasks[task_count].field = stats.field
65#define BOOL_FPRINT(stream, fmt, ...) \
66({ \
67 int ret = fprintf(stream, fmt, ##__VA_ARGS__); \
68 ret >= 0; \
69})
70#define TASK_AVG(task, field) average_ms((task).field##_delay_total, (task).field##_count)
71#define PSI_LINE_FORMAT "%-12s %6.1f%%/%6.1f%%/%6.1f%%/%8llu(ms)\n"
72#define DELAY_FMT_DEFAULT "%8.2f %8.2f %8.2f %8.2f\n"
73#define DELAY_FMT_MEMVERBOSE "%8.2f %8.2f %8.2f %8.2f %8.2f %8.2f\n"
74#define SORT_FIELD(name, cmd, modes) \
75 {#name, #cmd, \
76 offsetof(struct task_info, name##_delay_total), \
77 offsetof(struct task_info, name##_count), \
78 modes}
79#define END_FIELD {NULL, 0, 0}
80
81/* Display mode types */
82#define MODE_TYPE_ALL (0xFFFFFFFF)
83#define MODE_DEFAULT (1 << 0)
84#define MODE_MEMVERBOSE (1 << 1)
85
86/* PSI statistics structure */
87struct psi_stats {
88 double cpu_some_avg10, cpu_some_avg60, cpu_some_avg300;
89 unsigned long long cpu_some_total;
90 double cpu_full_avg10, cpu_full_avg60, cpu_full_avg300;
91 unsigned long long cpu_full_total;
92 double memory_some_avg10, memory_some_avg60, memory_some_avg300;
93 unsigned long long memory_some_total;
94 double memory_full_avg10, memory_full_avg60, memory_full_avg300;
95 unsigned long long memory_full_total;
96 double io_some_avg10, io_some_avg60, io_some_avg300;
97 unsigned long long io_some_total;
98 double io_full_avg10, io_full_avg60, io_full_avg300;
99 unsigned long long io_full_total;
100 double irq_full_avg10, irq_full_avg60, irq_full_avg300;
101 unsigned long long irq_full_total;
102};
103
104/* Task delay information structure */
105struct task_info {
106 int pid;
107 int tgid;
108 char command[TASK_COMM_LEN];
109 unsigned long long cpu_count;
110 unsigned long long cpu_delay_total;
111 unsigned long long blkio_count;
112 unsigned long long blkio_delay_total;
113 unsigned long long swapin_count;
114 unsigned long long swapin_delay_total;
115 unsigned long long freepages_count;
116 unsigned long long freepages_delay_total;
117 unsigned long long thrashing_count;
118 unsigned long long thrashing_delay_total;
119 unsigned long long compact_count;
120 unsigned long long compact_delay_total;
121 unsigned long long wpcopy_count;
122 unsigned long long wpcopy_delay_total;
123 unsigned long long irq_count;
124 unsigned long long irq_delay_total;
125 unsigned long long mem_count;
126 unsigned long long mem_delay_total;
127};
128
129/* Container statistics structure */
130struct container_stats {
131 int nr_sleeping; /* Number of sleeping processes */
132 int nr_running; /* Number of running processes */
133 int nr_stopped; /* Number of stopped processes */
134 int nr_uninterruptible; /* Number of uninterruptible processes */
135 int nr_io_wait; /* Number of processes in IO wait */
136};
137
138/* Delay field structure */
139struct field_desc {
140 const char *name; /* Field name for cmdline argument */
141 const char *cmd_char; /* Interactive command */
142 unsigned long total_offset; /* Offset of total delay in task_info */
143 unsigned long count_offset; /* Offset of count in task_info */
144 size_t supported_modes; /* Supported display modes */
145};
146
147/* Program settings structure */
148struct config {
149 int delay; /* Update interval in seconds */
150 int iterations; /* Number of iterations, 0 == infinite */
151 int max_processes; /* Maximum number of processes to show */
152 int output_one_time; /* Output once and exit */
153 int monitor_pid; /* Monitor specific PID */
154 char *container_path; /* Path to container cgroup */
155 const struct field_desc *sort_field; /* Current sort field */
156 size_t display_mode; /* Current display mode */
157};
158
159/* Global variables */
160static struct config cfg;
161static struct psi_stats psi;
162static struct task_info tasks[MAX_TASKS];
163static int task_count;
164static int running = 1;
165static struct container_stats container_stats;
166static const struct field_desc sort_fields[] = {
167 SORT_FIELD(cpu, c, MODE_DEFAULT),
168 SORT_FIELD(blkio, i, MODE_DEFAULT),
169 SORT_FIELD(irq, q, MODE_DEFAULT),
170 SORT_FIELD(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE),
171 SORT_FIELD(swapin, s, MODE_MEMVERBOSE),
172 SORT_FIELD(freepages, r, MODE_MEMVERBOSE),
173 SORT_FIELD(thrashing, t, MODE_MEMVERBOSE),
174 SORT_FIELD(compact, p, MODE_MEMVERBOSE),
175 SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE),
176 END_FIELD
177};
178static int sort_selected;
179
180/* Netlink socket variables */
181static int nl_sd = -1;
182static int family_id;
183
184/* Set terminal to non-canonical mode for q-to-quit */
185static struct termios orig_termios;
186static void enable_raw_mode(void)
187{
188 struct termios raw;
189
190 tcgetattr(STDIN_FILENO, &orig_termios);
191 raw = orig_termios;
192 raw.c_lflag &= ~(ICANON | ECHO);
193 tcsetattr(STDIN_FILENO, TCSAFLUSH, &raw);
194}
195static void disable_raw_mode(void)
196{
197 tcsetattr(STDIN_FILENO, TCSAFLUSH, &orig_termios);
198}
199
200/* Find field descriptor by command line */
201static const struct field_desc *get_field_by_cmd_char(char ch)
202{
203 const struct field_desc *field;
204
205 for (field = sort_fields; field->name != NULL; field++) {
206 if (field->cmd_char[0] == ch)
207 return field;
208 }
209
210 return NULL;
211}
212
213/* Find field descriptor by name with string comparison */
214static const struct field_desc *get_field_by_name(const char *name)
215{
216 const struct field_desc *field;
217 size_t field_len;
218
219 for (field = sort_fields; field->name != NULL; field++) {
220 field_len = strlen(field->name);
221 if (field_len != strlen(name))
222 continue;
223 if (strncmp(field->name, name, field_len) == 0)
224 return field;
225 }
226
227 return NULL;
228}
229
230/* Find display name for a field descriptor */
231static const char *get_name_by_field(const struct field_desc *field)
232{
233 return field ? field->name : "UNKNOWN";
234}
235
236/* Generate string of available field names */
237static void display_available_fields(size_t mode)
238{
239 const struct field_desc *field;
240 char buf[MAX_BUF_LEN];
241
242 buf[0] = '\0';
243
244 for (field = sort_fields; field->name != NULL; field++) {
245 if (!(field->supported_modes & mode))
246 continue;
247 strncat(p: buf, q: "|", MAX_BUF_LEN - strlen(buf) - 1);
248 strncat(p: buf, q: field->name, MAX_BUF_LEN - strlen(buf) - 1);
249 buf[MAX_BUF_LEN - 1] = '\0';
250 }
251
252 fprintf(stderr, "Available fields: %s\n", buf);
253}
254
255/* Display usage information and command line options */
256static void usage(void)
257{
258 printf("Usage: delaytop [Options]\n"
259 "Options:\n"
260 " -h, --help Show this help message and exit\n"
261 " -d, --delay=SECONDS Set refresh interval (default: 2 seconds, min: 1)\n"
262 " -n, --iterations=COUNT Set number of updates (default: 0 = infinite)\n"
263 " -P, --processes=NUMBER Set maximum number of processes to show (default: 20, max: 1000)\n"
264 " -o, --once Display once and exit\n"
265 " -p, --pid=PID Monitor only the specified PID\n"
266 " -C, --container=PATH Monitor the container at specified cgroup path\n"
267 " -s, --sort=FIELD Sort by delay field (default: cpu)\n"
268 " -M, --memverbose Display memory detailed information\n");
269 exit(0);
270}
271
272/* Parse command line arguments and set configuration */
273static void parse_args(int argc, char **argv)
274{
275 int c;
276 const struct field_desc *field;
277 struct option long_options[] = {
278 {"help", no_argument, 0, 'h'},
279 {"delay", required_argument, 0, 'd'},
280 {"iterations", required_argument, 0, 'n'},
281 {"pid", required_argument, 0, 'p'},
282 {"once", no_argument, 0, 'o'},
283 {"processes", required_argument, 0, 'P'},
284 {"sort", required_argument, 0, 's'},
285 {"container", required_argument, 0, 'C'},
286 {"memverbose", no_argument, 0, 'M'},
287 {0, 0, 0, 0}
288 };
289
290 /* Set defaults */
291 cfg.delay = 2;
292 cfg.iterations = 0;
293 cfg.max_processes = 20;
294 cfg.sort_field = &sort_fields[0]; /* Default sorted by CPU delay */
295 cfg.output_one_time = 0;
296 cfg.monitor_pid = 0; /* 0 means monitor all PIDs */
297 cfg.container_path = NULL;
298 cfg.display_mode = MODE_DEFAULT;
299
300 while (1) {
301 int option_index = 0;
302
303 c = getopt_long(argc, argv, "hd:n:p:oP:C:s:M", long_options, &option_index);
304 if (c == -1)
305 break;
306
307 switch (c) {
308 case 'h':
309 usage();
310 break;
311 case 'd':
312 cfg.delay = atoi(optarg);
313 if (cfg.delay < 1) {
314 fprintf(stderr, "Error: delay must be >= 1.\n");
315 exit(1);
316 }
317 break;
318 case 'n':
319 cfg.iterations = atoi(optarg);
320 if (cfg.iterations < 0) {
321 fprintf(stderr, "Error: iterations must be >= 0.\n");
322 exit(1);
323 }
324 break;
325 case 'p':
326 cfg.monitor_pid = atoi(optarg);
327 if (cfg.monitor_pid < 1) {
328 fprintf(stderr, "Error: pid must be >= 1.\n");
329 exit(1);
330 }
331 break;
332 case 'o':
333 cfg.output_one_time = 1;
334 break;
335 case 'P':
336 cfg.max_processes = atoi(optarg);
337 if (cfg.max_processes < 1) {
338 fprintf(stderr, "Error: processes must be >= 1.\n");
339 exit(1);
340 }
341 if (cfg.max_processes > MAX_TASKS) {
342 fprintf(stderr, "Warning: processes capped to %d.\n",
343 MAX_TASKS);
344 cfg.max_processes = MAX_TASKS;
345 }
346 break;
347 case 'C':
348 cfg.container_path = strdup(optarg);
349 break;
350 case 's':
351 if (strlen(optarg) == 0) {
352 fprintf(stderr, "Error: empty sort field\n");
353 exit(1);
354 }
355
356 field = get_field_by_name(name: optarg);
357 /* Show available fields if invalid option provided */
358 if (!field) {
359 fprintf(stderr, "Error: invalid sort field '%s'\n", optarg);
360 display_available_fields(MODE_TYPE_ALL);
361 exit(1);
362 }
363
364 cfg.sort_field = field;
365 break;
366 case 'M':
367 cfg.display_mode = MODE_MEMVERBOSE;
368 cfg.sort_field = get_field_by_name(name: "mem");
369 break;
370 default:
371 fprintf(stderr, "Try 'delaytop --help' for more information.\n");
372 exit(1);
373 }
374 }
375}
376
377/* Calculate average delay in milliseconds for overall memory */
378static void set_mem_delay_total(struct task_info *t)
379{
380 t->mem_delay_total = t->swapin_delay_total +
381 t->freepages_delay_total +
382 t->thrashing_delay_total +
383 t->compact_delay_total +
384 t->wpcopy_delay_total;
385}
386
387static void set_mem_count(struct task_info *t)
388{
389 t->mem_count = t->swapin_count +
390 t->freepages_count +
391 t->thrashing_count +
392 t->compact_count +
393 t->wpcopy_count;
394}
395
396/* Create a raw netlink socket and bind */
397static int create_nl_socket(void)
398{
399 int fd;
400 struct sockaddr_nl local;
401
402 fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
403 if (fd < 0)
404 return -1;
405
406 memset(&local, 0, sizeof(local));
407 local.nl_family = AF_NETLINK;
408
409 if (bind(fd, (struct sockaddr *) &local, sizeof(local)) < 0) {
410 fprintf(stderr, "Failed to bind socket when create nl_socket\n");
411 close(fd);
412 return -1;
413 }
414
415 return fd;
416}
417
418/* Send a command via netlink */
419static int send_cmd(int sd, __u16 nlmsg_type, __u32 nlmsg_pid,
420 __u8 genl_cmd, __u16 nla_type,
421 void *nla_data, int nla_len)
422{
423 struct sockaddr_nl nladdr;
424 struct nlattr *na;
425 int r, buflen;
426 char *buf;
427
428 struct {
429 struct nlmsghdr n;
430 struct genlmsghdr g;
431 char buf[MAX_MSG_SIZE];
432 } msg;
433
434 msg.n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
435 msg.n.nlmsg_type = nlmsg_type;
436 msg.n.nlmsg_flags = NLM_F_REQUEST;
437 msg.n.nlmsg_seq = 0;
438 msg.n.nlmsg_pid = nlmsg_pid;
439 msg.g.cmd = genl_cmd;
440 msg.g.version = 0x1;
441 na = (struct nlattr *) GENLMSG_DATA(&msg);
442 na->nla_type = nla_type;
443 na->nla_len = nla_len + NLA_HDRLEN;
444 memcpy(NLA_DATA(na), nla_data, nla_len);
445 msg.n.nlmsg_len += NLMSG_ALIGN(na->nla_len);
446
447 buf = (char *) &msg;
448 buflen = msg.n.nlmsg_len;
449 memset(&nladdr, 0, sizeof(nladdr));
450 nladdr.nl_family = AF_NETLINK;
451 while ((r = sendto(sd, buf, buflen, 0, (struct sockaddr *) &nladdr,
452 sizeof(nladdr))) < buflen) {
453 if (r > 0) {
454 buf += r;
455 buflen -= r;
456 } else if (errno != EAGAIN)
457 return -1;
458 }
459 return 0;
460}
461
462/* Get family ID for taskstats via netlink */
463static int get_family_id(int sd)
464{
465 struct {
466 struct nlmsghdr n;
467 struct genlmsghdr g;
468 char buf[256];
469 } ans;
470
471 int id = 0, rc;
472 struct nlattr *na;
473 int rep_len;
474 char name[100];
475
476 strncpy(p: name, TASKSTATS_GENL_NAME, size: sizeof(name) - 1);
477 name[sizeof(name) - 1] = '\0';
478 rc = send_cmd(sd, GENL_ID_CTRL, nlmsg_pid: getpid(), genl_cmd: CTRL_CMD_GETFAMILY,
479 nla_type: CTRL_ATTR_FAMILY_NAME, nla_data: (void *)name,
480 strlen(TASKSTATS_GENL_NAME)+1);
481 if (rc < 0) {
482 fprintf(stderr, "Failed to send cmd for family id\n");
483 return 0;
484 }
485
486 rep_len = recv(sd, &ans, sizeof(ans), 0);
487 if (ans.n.nlmsg_type == NLMSG_ERROR ||
488 (rep_len < 0) || !NLMSG_OK((&ans.n), rep_len)) {
489 fprintf(stderr, "Failed to receive response for family id\n");
490 return 0;
491 }
492
493 na = (struct nlattr *) GENLMSG_DATA(&ans);
494 na = (struct nlattr *) ((char *) na + NLA_ALIGN(na->nla_len));
495 if (na->nla_type == CTRL_ATTR_FAMILY_ID)
496 id = *(__u16 *) NLA_DATA(na);
497 return id;
498}
499
500static int read_psi_stats(void)
501{
502 FILE *fp;
503 char line[256];
504 int ret = 0;
505 int error_count = 0;
506
507 /* Check if PSI path exists */
508 if (access(PSI_PATH, F_OK) != 0) {
509 fprintf(stderr, "Error: PSI interface not found at %s\n", PSI_PATH);
510 fprintf(stderr, "Please ensure your kernel supports PSI (Pressure Stall Information)\n");
511 return -1;
512 }
513
514 /* Zero all fields */
515 memset(&psi, 0, sizeof(psi));
516
517 /* CPU pressure */
518 fp = fopen(PSI_CPU_PATH, "r");
519 if (fp) {
520 while (fgets(line, sizeof(line), fp)) {
521 if (strncmp(line, "some", 4) == 0) {
522 ret = sscanf(line, "some avg10=%lf avg60=%lf avg300=%lf total=%llu",
523 &psi.cpu_some_avg10, &psi.cpu_some_avg60,
524 &psi.cpu_some_avg300, &psi.cpu_some_total);
525 if (ret != 4) {
526 fprintf(stderr, "Failed to parse CPU some PSI data\n");
527 error_count++;
528 }
529 } else if (strncmp(line, "full", 4) == 0) {
530 ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu",
531 &psi.cpu_full_avg10, &psi.cpu_full_avg60,
532 &psi.cpu_full_avg300, &psi.cpu_full_total);
533 if (ret != 4) {
534 fprintf(stderr, "Failed to parse CPU full PSI data\n");
535 error_count++;
536 }
537 }
538 }
539 fclose(fp);
540 } else {
541 fprintf(stderr, "Warning: Failed to open %s\n", PSI_CPU_PATH);
542 error_count++;
543 }
544
545 /* Memory pressure */
546 fp = fopen(PSI_MEMORY_PATH, "r");
547 if (fp) {
548 while (fgets(line, sizeof(line), fp)) {
549 if (strncmp(line, "some", 4) == 0) {
550 ret = sscanf(line, "some avg10=%lf avg60=%lf avg300=%lf total=%llu",
551 &psi.memory_some_avg10, &psi.memory_some_avg60,
552 &psi.memory_some_avg300, &psi.memory_some_total);
553 if (ret != 4) {
554 fprintf(stderr, "Failed to parse Memory some PSI data\n");
555 error_count++;
556 }
557 } else if (strncmp(line, "full", 4) == 0) {
558 ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu",
559 &psi.memory_full_avg10, &psi.memory_full_avg60,
560 &psi.memory_full_avg300, &psi.memory_full_total);
561 if (ret != 4) {
562 fprintf(stderr, "Failed to parse Memory full PSI data\n");
563 error_count++;
564 }
565 }
566 }
567 fclose(fp);
568 } else {
569 fprintf(stderr, "Warning: Failed to open %s\n", PSI_MEMORY_PATH);
570 error_count++;
571 }
572
573 /* IO pressure */
574 fp = fopen(PSI_IO_PATH, "r");
575 if (fp) {
576 while (fgets(line, sizeof(line), fp)) {
577 if (strncmp(line, "some", 4) == 0) {
578 ret = sscanf(line, "some avg10=%lf avg60=%lf avg300=%lf total=%llu",
579 &psi.io_some_avg10, &psi.io_some_avg60,
580 &psi.io_some_avg300, &psi.io_some_total);
581 if (ret != 4) {
582 fprintf(stderr, "Failed to parse IO some PSI data\n");
583 error_count++;
584 }
585 } else if (strncmp(line, "full", 4) == 0) {
586 ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu",
587 &psi.io_full_avg10, &psi.io_full_avg60,
588 &psi.io_full_avg300, &psi.io_full_total);
589 if (ret != 4) {
590 fprintf(stderr, "Failed to parse IO full PSI data\n");
591 error_count++;
592 }
593 }
594 }
595 fclose(fp);
596 } else {
597 fprintf(stderr, "Warning: Failed to open %s\n", PSI_IO_PATH);
598 error_count++;
599 }
600
601 /* IRQ pressure (only full) */
602 fp = fopen(PSI_IRQ_PATH, "r");
603 if (fp) {
604 while (fgets(line, sizeof(line), fp)) {
605 if (strncmp(line, "full", 4) == 0) {
606 ret = sscanf(line, "full avg10=%lf avg60=%lf avg300=%lf total=%llu",
607 &psi.irq_full_avg10, &psi.irq_full_avg60,
608 &psi.irq_full_avg300, &psi.irq_full_total);
609 if (ret != 4) {
610 fprintf(stderr, "Failed to parse IRQ full PSI data\n");
611 error_count++;
612 }
613 }
614 }
615 fclose(fp);
616 } else {
617 fprintf(stderr, "Warning: Failed to open %s\n", PSI_IRQ_PATH);
618 error_count++;
619 }
620
621 /* Return error count: 0 means success, >0 means warnings, -1 means fatal error */
622 if (error_count > 0) {
623 fprintf(stderr, "PSI stats reading completed with %d warnings\n", error_count);
624 return error_count;
625 }
626
627 return 0;
628}
629
630static int read_comm(int pid, char *comm_buf, size_t buf_size)
631{
632 char path[64];
633 int ret = -1;
634 size_t len;
635 FILE *fp;
636
637 snprintf(buf: path, size: sizeof(path), fmt: "/proc/%d/comm", pid);
638 fp = fopen(path, "r");
639 if (!fp) {
640 fprintf(stderr, "Failed to open comm file /proc/%d/comm\n", pid);
641 return ret;
642 }
643
644 if (fgets(comm_buf, buf_size, fp)) {
645 len = strlen(comm_buf);
646 if (len > 0 && comm_buf[len - 1] == '\n')
647 comm_buf[len - 1] = '\0';
648 ret = 0;
649 }
650
651 fclose(fp);
652
653 return ret;
654}
655
656static void fetch_and_fill_task_info(int pid, const char *comm)
657{
658 struct {
659 struct nlmsghdr n;
660 struct genlmsghdr g;
661 char buf[MAX_MSG_SIZE];
662 } resp;
663 struct taskstats stats;
664 struct nlattr *nested;
665 struct nlattr *na;
666 int nested_len;
667 int nl_len;
668 int rc;
669
670 /* Send request for task stats */
671 if (send_cmd(sd: nl_sd, nlmsg_type: family_id, nlmsg_pid: getpid(), genl_cmd: TASKSTATS_CMD_GET,
672 nla_type: TASKSTATS_CMD_ATTR_PID, nla_data: &pid, nla_len: sizeof(pid)) < 0) {
673 fprintf(stderr, "Failed to send request for task stats\n");
674 return;
675 }
676
677 /* Receive response */
678 rc = recv(nl_sd, &resp, sizeof(resp), 0);
679 if (rc < 0 || resp.n.nlmsg_type == NLMSG_ERROR) {
680 fprintf(stderr, "Failed to receive response for task stats\n");
681 return;
682 }
683
684 /* Parse response */
685 nl_len = GENLMSG_PAYLOAD(&resp.n);
686 na = (struct nlattr *) GENLMSG_DATA(&resp);
687 while (nl_len > 0) {
688 if (na->nla_type == TASKSTATS_TYPE_AGGR_PID) {
689 nested = (struct nlattr *) NLA_DATA(na);
690 nested_len = NLA_PAYLOAD(na->nla_len);
691 while (nested_len > 0) {
692 if (nested->nla_type == TASKSTATS_TYPE_STATS) {
693 memcpy(&stats, NLA_DATA(nested), sizeof(stats));
694 if (task_count < MAX_TASKS) {
695 tasks[task_count].pid = pid;
696 tasks[task_count].tgid = pid;
697 strncpy(p: tasks[task_count].command, q: comm,
698 TASK_COMM_LEN - 1);
699 tasks[task_count].command[TASK_COMM_LEN - 1] = '\0';
700 SET_TASK_STAT(task_count, cpu_count);
701 SET_TASK_STAT(task_count, cpu_delay_total);
702 SET_TASK_STAT(task_count, blkio_count);
703 SET_TASK_STAT(task_count, blkio_delay_total);
704 SET_TASK_STAT(task_count, swapin_count);
705 SET_TASK_STAT(task_count, swapin_delay_total);
706 SET_TASK_STAT(task_count, freepages_count);
707 SET_TASK_STAT(task_count, freepages_delay_total);
708 SET_TASK_STAT(task_count, thrashing_count);
709 SET_TASK_STAT(task_count, thrashing_delay_total);
710 SET_TASK_STAT(task_count, compact_count);
711 SET_TASK_STAT(task_count, compact_delay_total);
712 SET_TASK_STAT(task_count, wpcopy_count);
713 SET_TASK_STAT(task_count, wpcopy_delay_total);
714 SET_TASK_STAT(task_count, irq_count);
715 SET_TASK_STAT(task_count, irq_delay_total);
716 set_mem_count(&tasks[task_count]);
717 set_mem_delay_total(&tasks[task_count]);
718 task_count++;
719 }
720 break;
721 }
722 nested_len -= NLA_ALIGN(nested->nla_len);
723 nested = NLA_NEXT(nested);
724 }
725 }
726 nl_len -= NLA_ALIGN(na->nla_len);
727 na = NLA_NEXT(na);
728 }
729 return;
730}
731
732static void get_task_delays(void)
733{
734 char comm[TASK_COMM_LEN];
735 struct dirent *entry;
736 DIR *dir;
737 int pid;
738
739 task_count = 0;
740 if (cfg.monitor_pid > 0) {
741 if (read_comm(pid: cfg.monitor_pid, comm_buf: comm, buf_size: sizeof(comm)) == 0)
742 fetch_and_fill_task_info(pid: cfg.monitor_pid, comm);
743 return;
744 }
745
746 dir = opendir("/proc");
747 if (!dir) {
748 fprintf(stderr, "Error opening /proc directory\n");
749 return;
750 }
751
752 while ((entry = readdir(dir)) != NULL && task_count < MAX_TASKS) {
753 if (!isdigit(c: entry->d_name[0]))
754 continue;
755 pid = atoi(entry->d_name);
756 if (pid == 0)
757 continue;
758 if (read_comm(pid, comm_buf: comm, buf_size: sizeof(comm)) != 0)
759 continue;
760 fetch_and_fill_task_info(pid, comm);
761 }
762 closedir(dir);
763}
764
765/* Calculate average delay in milliseconds */
766static double average_ms(unsigned long long total, unsigned long long count)
767{
768 if (count == 0)
769 return 0;
770 return (double)total / 1000000.0 / count;
771}
772
773/* Comparison function for sorting tasks */
774static int compare_tasks(const void *a, const void *b)
775{
776 const struct task_info *t1 = (const struct task_info *)a;
777 const struct task_info *t2 = (const struct task_info *)b;
778 unsigned long long total1;
779 unsigned long long total2;
780 unsigned long count1;
781 unsigned long count2;
782 double avg1, avg2;
783
784 total1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->total_offset);
785 total2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->total_offset);
786 count1 = *(unsigned long *)((char *)t1 + cfg.sort_field->count_offset);
787 count2 = *(unsigned long *)((char *)t2 + cfg.sort_field->count_offset);
788
789 avg1 = average_ms(total: total1, count: count1);
790 avg2 = average_ms(total: total2, count: count2);
791 if (avg1 != avg2)
792 return avg2 > avg1 ? 1 : -1;
793
794 return 0;
795}
796
797/* Sort tasks by selected field */
798static void sort_tasks(void)
799{
800 if (task_count > 0)
801 qsort(tasks, task_count, sizeof(struct task_info), compare_tasks);
802}
803
804/* Get container statistics via cgroupstats */
805static void get_container_stats(void)
806{
807 int rc, cfd;
808 struct {
809 struct nlmsghdr n;
810 struct genlmsghdr g;
811 char buf[MAX_MSG_SIZE];
812 } req, resp;
813 struct nlattr *na;
814 int nl_len;
815 struct cgroupstats stats;
816
817 /* Check if container path is set */
818 if (!cfg.container_path)
819 return;
820
821 /* Open container cgroup */
822 cfd = open(cfg.container_path, O_RDONLY);
823 if (cfd < 0) {
824 fprintf(stderr, "Error opening container path: %s\n", cfg.container_path);
825 return;
826 }
827
828 /* Send request for container stats */
829 if (send_cmd(sd: nl_sd, nlmsg_type: family_id, nlmsg_pid: getpid(), genl_cmd: CGROUPSTATS_CMD_GET,
830 nla_type: CGROUPSTATS_CMD_ATTR_FD, nla_data: &cfd, nla_len: sizeof(__u32)) < 0) {
831 fprintf(stderr, "Failed to send request for container stats\n");
832 close(cfd);
833 return;
834 }
835
836 /* Receive response */
837 rc = recv(nl_sd, &resp, sizeof(resp), 0);
838 if (rc < 0 || resp.n.nlmsg_type == NLMSG_ERROR) {
839 fprintf(stderr, "Failed to receive response for container stats\n");
840 close(cfd);
841 return;
842 }
843
844 /* Parse response */
845 nl_len = GENLMSG_PAYLOAD(&resp.n);
846 na = (struct nlattr *) GENLMSG_DATA(&resp);
847 while (nl_len > 0) {
848 if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) {
849 /* Get the cgroupstats structure */
850 memcpy(&stats, NLA_DATA(na), sizeof(stats));
851
852 /* Fill container stats */
853 container_stats.nr_sleeping = stats.nr_sleeping;
854 container_stats.nr_running = stats.nr_running;
855 container_stats.nr_stopped = stats.nr_stopped;
856 container_stats.nr_uninterruptible = stats.nr_uninterruptible;
857 container_stats.nr_io_wait = stats.nr_io_wait;
858 break;
859 }
860 nl_len -= NLA_ALIGN(na->nla_len);
861 na = (struct nlattr *) ((char *) na + NLA_ALIGN(na->nla_len));
862 }
863
864 close(cfd);
865}
866
867/* Display results to stdout or log file */
868static void display_results(int psi_ret)
869{
870 time_t now = time(NULL);
871 struct tm *tm_now = localtime(&now);
872 FILE *out = stdout;
873 char timestamp[32];
874 bool suc = true;
875 int i, count;
876
877 /* Clear terminal screen */
878 suc &= BOOL_FPRINT(out, "\033[H\033[J");
879
880 /* PSI output (one-line, no cat style) */
881 suc &= BOOL_FPRINT(out, "System Pressure Information: (avg10/avg60vg300/total)\n");
882 if (psi_ret) {
883 suc &= BOOL_FPRINT(out, " PSI not found: check if psi=1 enabled in cmdline\n");
884 } else {
885 suc &= BOOL_FPRINT(out, PSI_LINE_FORMAT,
886 "CPU some:",
887 psi.cpu_some_avg10,
888 psi.cpu_some_avg60,
889 psi.cpu_some_avg300,
890 psi.cpu_some_total / 1000);
891 suc &= BOOL_FPRINT(out, PSI_LINE_FORMAT,
892 "CPU full:",
893 psi.cpu_full_avg10,
894 psi.cpu_full_avg60,
895 psi.cpu_full_avg300,
896 psi.cpu_full_total / 1000);
897 suc &= BOOL_FPRINT(out, PSI_LINE_FORMAT,
898 "Memory full:",
899 psi.memory_full_avg10,
900 psi.memory_full_avg60,
901 psi.memory_full_avg300,
902 psi.memory_full_total / 1000);
903 suc &= BOOL_FPRINT(out, PSI_LINE_FORMAT,
904 "Memory some:",
905 psi.memory_some_avg10,
906 psi.memory_some_avg60,
907 psi.memory_some_avg300,
908 psi.memory_some_total / 1000);
909 suc &= BOOL_FPRINT(out, PSI_LINE_FORMAT,
910 "IO full:",
911 psi.io_full_avg10,
912 psi.io_full_avg60,
913 psi.io_full_avg300,
914 psi.io_full_total / 1000);
915 suc &= BOOL_FPRINT(out, PSI_LINE_FORMAT,
916 "IO some:",
917 psi.io_some_avg10,
918 psi.io_some_avg60,
919 psi.io_some_avg300,
920 psi.io_some_total / 1000);
921 suc &= BOOL_FPRINT(out, PSI_LINE_FORMAT,
922 "IRQ full:",
923 psi.irq_full_avg10,
924 psi.irq_full_avg60,
925 psi.irq_full_avg300,
926 psi.irq_full_total / 1000);
927 }
928
929 if (cfg.container_path) {
930 suc &= BOOL_FPRINT(out, "Container Information (%s):\n", cfg.container_path);
931 suc &= BOOL_FPRINT(out, "Processes: running=%d, sleeping=%d, ",
932 container_stats.nr_running, container_stats.nr_sleeping);
933 suc &= BOOL_FPRINT(out, "stopped=%d, uninterruptible=%d, io_wait=%d\n\n",
934 container_stats.nr_stopped, container_stats.nr_uninterruptible,
935 container_stats.nr_io_wait);
936 }
937
938 /* Interacive command */
939 suc &= BOOL_FPRINT(out, "[o]sort [M]memverbose [q]quit\n");
940 if (sort_selected) {
941 if (cfg.display_mode == MODE_MEMVERBOSE)
942 suc &= BOOL_FPRINT(out,
943 "sort selection: [m]MEM [r]RCL [t]THR [p]CMP [w]WP\n");
944 else
945 suc &= BOOL_FPRINT(out,
946 "sort selection: [c]CPU [i]IO [m]MEM [q]IRQ\n");
947 }
948
949 /* Task delay output */
950 suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
951 cfg.max_processes, get_name_by_field(cfg.sort_field));
952
953 suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
954 if (cfg.display_mode == MODE_MEMVERBOSE) {
955 suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
956 "MEM(ms)", "SWAP(ms)", "RCL(ms)",
957 "THR(ms)", "CMP(ms)", "WP(ms)");
958 suc &= BOOL_FPRINT(out, "-----------------------");
959 suc &= BOOL_FPRINT(out, "-----------------------");
960 suc &= BOOL_FPRINT(out, "-----------------------");
961 suc &= BOOL_FPRINT(out, "---------------------\n");
962 } else {
963 suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
964 "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
965 suc &= BOOL_FPRINT(out, "-----------------------");
966 suc &= BOOL_FPRINT(out, "-----------------------");
967 suc &= BOOL_FPRINT(out, "--------------------------\n");
968 }
969
970 count = task_count < cfg.max_processes ? task_count : cfg.max_processes;
971
972 for (i = 0; i < count; i++) {
973 suc &= BOOL_FPRINT(out, "%8d %8d %-15s",
974 tasks[i].pid, tasks[i].tgid, tasks[i].command);
975 if (cfg.display_mode == MODE_MEMVERBOSE) {
976 suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE,
977 TASK_AVG(tasks[i], mem),
978 TASK_AVG(tasks[i], swapin),
979 TASK_AVG(tasks[i], freepages),
980 TASK_AVG(tasks[i], thrashing),
981 TASK_AVG(tasks[i], compact),
982 TASK_AVG(tasks[i], wpcopy));
983 } else {
984 suc &= BOOL_FPRINT(out, DELAY_FMT_DEFAULT,
985 TASK_AVG(tasks[i], cpu),
986 TASK_AVG(tasks[i], blkio),
987 TASK_AVG(tasks[i], irq),
988 TASK_AVG(tasks[i], mem));
989 }
990 }
991
992 suc &= BOOL_FPRINT(out, "\n");
993
994 if (!suc)
995 perror("Error writing to output");
996}
997
998/* Check for keyboard input with timeout based on cfg.delay */
999static char check_for_keypress(void)
1000{
1001 struct timeval tv = {cfg.delay, 0};
1002 fd_set readfds;
1003 char ch = 0;
1004
1005 FD_ZERO(&readfds);
1006 FD_SET(STDIN_FILENO, &readfds);
1007 int r = select(STDIN_FILENO + 1, &readfds, NULL, NULL, &tv);
1008
1009 if (r > 0 && FD_ISSET(STDIN_FILENO, &readfds)) {
1010 read(STDIN_FILENO, &ch, 1);
1011 return ch;
1012 }
1013
1014 return 0;
1015}
1016
1017#define MAX_MODE_SIZE 2
1018static void toggle_display_mode(void)
1019{
1020 static const size_t modes[MAX_MODE_SIZE] = {MODE_DEFAULT, MODE_MEMVERBOSE};
1021 static size_t cur_index;
1022
1023 cur_index = (cur_index + 1) % MAX_MODE_SIZE;
1024 cfg.display_mode = modes[cur_index];
1025}
1026
1027/* Handle keyboard input: sorting selection, mode toggle, or quit */
1028static void handle_keypress(char ch, int *running)
1029{
1030 const struct field_desc *field;
1031
1032 /* Change sort field */
1033 if (sort_selected) {
1034 field = get_field_by_cmd_char(ch);
1035 if (field && (field->supported_modes & cfg.display_mode))
1036 cfg.sort_field = field;
1037
1038 sort_selected = 0;
1039 /* Handle mode changes or quit */
1040 } else {
1041 switch (ch) {
1042 case 'o':
1043 sort_selected = 1;
1044 break;
1045 case 'M':
1046 toggle_display_mode();
1047 for (field = sort_fields; field->name != NULL; field++) {
1048 if (field->supported_modes & cfg.display_mode) {
1049 cfg.sort_field = field;
1050 break;
1051 }
1052 }
1053 break;
1054 case 'q':
1055 case 'Q':
1056 *running = 0;
1057 break;
1058 default:
1059 break;
1060 }
1061 }
1062}
1063
1064/* Main function */
1065int main(int argc, char **argv)
1066{
1067 const struct field_desc *field;
1068 int iterations = 0;
1069 int psi_ret = 0;
1070 char keypress;
1071
1072 /* Parse command line arguments */
1073 parse_args(argc, argv);
1074
1075 /* Setup netlink socket */
1076 nl_sd = create_nl_socket();
1077 if (nl_sd < 0) {
1078 fprintf(stderr, "Error creating netlink socket\n");
1079 exit(1);
1080 }
1081
1082 /* Get family ID for taskstats via netlink */
1083 family_id = get_family_id(sd: nl_sd);
1084 if (!family_id) {
1085 fprintf(stderr, "Error getting taskstats family ID\n");
1086 close(nl_sd);
1087 exit(1);
1088 }
1089
1090 /* Set terminal to non-canonical mode for interaction */
1091 enable_raw_mode();
1092
1093 /* Main loop */
1094 while (running) {
1095 /* Auto-switch sort field when not matching display mode */
1096 if (!(cfg.sort_field->supported_modes & cfg.display_mode)) {
1097 for (field = sort_fields; field->name != NULL; field++) {
1098 if (field->supported_modes & cfg.display_mode) {
1099 cfg.sort_field = field;
1100 printf("Auto-switched sort field to: %s\n", field->name);
1101 break;
1102 }
1103 }
1104 }
1105
1106 /* Read PSI statistics */
1107 psi_ret = read_psi_stats();
1108
1109 /* Get container stats if container path provided */
1110 if (cfg.container_path)
1111 get_container_stats();
1112
1113 /* Get task delays */
1114 get_task_delays();
1115
1116 /* Sort tasks */
1117 sort_tasks();
1118
1119 /* Display results to stdout or log file */
1120 display_results(psi_ret);
1121
1122 /* Check for iterations */
1123 if (cfg.iterations > 0 && ++iterations >= cfg.iterations)
1124 break;
1125
1126 /* Exit if output_one_time is set */
1127 if (cfg.output_one_time)
1128 break;
1129
1130 /* Keypress for interactive usage */
1131 keypress = check_for_keypress();
1132 if (keypress)
1133 handle_keypress(ch: keypress, running: &running);
1134 }
1135
1136 /* Restore terminal mode */
1137 disable_raw_mode();
1138
1139 /* Cleanup */
1140 close(nl_sd);
1141 if (cfg.container_path)
1142 free(cfg.container_path);
1143
1144 return 0;
1145}
1146

source code of linux/tools/accounting/delaytop.c