1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2// Copyright (c) 2021 Google
3#include "vmlinux.h"
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6
7// This should be in sync with "util/ftrace.h"
8#define NUM_BUCKET 22
9
10struct {
11 __uint(type, BPF_MAP_TYPE_HASH);
12 __uint(key_size, sizeof(__u64));
13 __uint(value_size, sizeof(__u64));
14 __uint(max_entries, 10000);
15} functime SEC(".maps");
16
17struct {
18 __uint(type, BPF_MAP_TYPE_HASH);
19 __uint(key_size, sizeof(__u32));
20 __uint(value_size, sizeof(__u8));
21 __uint(max_entries, 1);
22} cpu_filter SEC(".maps");
23
24struct {
25 __uint(type, BPF_MAP_TYPE_HASH);
26 __uint(key_size, sizeof(__u32));
27 __uint(value_size, sizeof(__u8));
28 __uint(max_entries, 1);
29} task_filter SEC(".maps");
30
31struct {
32 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
33 __uint(key_size, sizeof(__u32));
34 __uint(value_size, sizeof(__u64));
35 __uint(max_entries, NUM_BUCKET);
36} latency SEC(".maps");
37
38
39int enabled = 0;
40
41// stats
42__s64 total;
43__s64 count;
44__s64 max;
45__s64 min;
46
47const volatile int has_cpu = 0;
48const volatile int has_task = 0;
49const volatile int use_nsec = 0;
50const volatile unsigned int bucket_range;
51const volatile unsigned int min_latency;
52const volatile unsigned int max_latency;
53const volatile unsigned int bucket_num = NUM_BUCKET;
54
55static bool can_record(void)
56{
57 if (has_cpu) {
58 __u32 cpu = bpf_get_smp_processor_id();
59 __u8 *ok;
60
61 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
62 if (!ok)
63 return false;
64 }
65
66 if (has_task) {
67 __u32 pid = bpf_get_current_pid_tgid();
68 __u8 *ok;
69
70 ok = bpf_map_lookup_elem(&task_filter, &pid);
71 if (!ok)
72 return false;
73 }
74 return true;
75}
76
77static void update_latency(__s64 delta)
78{
79 __u64 val = delta;
80 __u32 key = 0;
81 __u64 *hist;
82 __u64 cmp_base = use_nsec ? 1 : 1000;
83
84 if (delta < 0)
85 return;
86
87 if (bucket_range != 0) {
88 val = delta / cmp_base;
89
90 if (min_latency > 0) {
91 if (val > min_latency)
92 val -= min_latency;
93 else
94 goto do_lookup;
95 }
96
97 // Less than 1 unit (ms or ns), or, in the future,
98 // than the min latency desired.
99 if (val > 0) { // 1st entry: [ 1 unit .. bucket_range units )
100 key = val / bucket_range + 1;
101 if (key >= bucket_num)
102 key = bucket_num - 1;
103 }
104
105 goto do_lookup;
106 }
107 // calculate index using delta
108 for (key = 0; key < (bucket_num - 1); key++) {
109 if (delta < (cmp_base << key))
110 break;
111 }
112
113do_lookup:
114 hist = bpf_map_lookup_elem(&latency, &key);
115 if (!hist)
116 return;
117
118 __sync_fetch_and_add(hist, 1);
119
120 __sync_fetch_and_add(&total, delta); // always in nsec
121 __sync_fetch_and_add(&count, 1);
122
123 if (delta > max)
124 max = delta;
125 if (delta < min)
126 min = delta;
127}
128
129SEC("kprobe/func")
130int BPF_PROG(func_begin)
131{
132 __u64 key, now;
133
134 if (!enabled || !can_record())
135 return 0;
136
137 key = bpf_get_current_pid_tgid();
138 now = bpf_ktime_get_ns();
139
140 // overwrite timestamp for nested functions
141 bpf_map_update_elem(&functime, &key, &now, BPF_ANY);
142 return 0;
143}
144
145SEC("kretprobe/func")
146int BPF_PROG(func_end)
147{
148 __u64 tid;
149 __u64 *start;
150
151 if (!enabled)
152 return 0;
153
154 tid = bpf_get_current_pid_tgid();
155
156 start = bpf_map_lookup_elem(&functime, &tid);
157 if (start) {
158 update_latency(bpf_ktime_get_ns() - *start);
159 bpf_map_delete_elem(&functime, &tid);
160 }
161
162 return 0;
163}
164
165SEC("raw_tp")
166int BPF_PROG(event_begin)
167{
168 __u64 key, now;
169
170 if (!enabled || !can_record())
171 return 0;
172
173 key = bpf_get_current_pid_tgid();
174 now = bpf_ktime_get_ns();
175
176 // overwrite timestamp for nested events
177 bpf_map_update_elem(&functime, &key, &now, BPF_ANY);
178 return 0;
179}
180
181SEC("raw_tp")
182int BPF_PROG(event_end)
183{
184 __u64 tid;
185 __u64 *start;
186
187 if (!enabled)
188 return 0;
189
190 tid = bpf_get_current_pid_tgid();
191
192 start = bpf_map_lookup_elem(&functime, &tid);
193 if (start) {
194 update_latency(bpf_ktime_get_ns() - *start);
195 bpf_map_delete_elem(&functime, &tid);
196 }
197
198 return 0;
199}
200

source code of linux/tools/perf/util/bpf_skel/func_latency.bpf.c