1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | |
3 | #define pr_fmt(fmt) "ref_tracker: " fmt |
4 | |
5 | #include <linux/export.h> |
6 | #include <linux/list_sort.h> |
7 | #include <linux/ref_tracker.h> |
8 | #include <linux/slab.h> |
9 | #include <linux/stacktrace.h> |
10 | #include <linux/stackdepot.h> |
11 | |
12 | #define REF_TRACKER_STACK_ENTRIES 16 |
13 | #define STACK_BUF_SIZE 1024 |
14 | |
15 | struct ref_tracker { |
16 | struct list_head head; /* anchor into dir->list or dir->quarantine */ |
17 | bool dead; |
18 | depot_stack_handle_t alloc_stack_handle; |
19 | depot_stack_handle_t free_stack_handle; |
20 | }; |
21 | |
22 | struct ref_tracker_dir_stats { |
23 | int total; |
24 | int count; |
25 | struct { |
26 | depot_stack_handle_t stack_handle; |
27 | unsigned int count; |
28 | } stacks[]; |
29 | }; |
30 | |
31 | static struct ref_tracker_dir_stats * |
32 | ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit) |
33 | { |
34 | struct ref_tracker_dir_stats *stats; |
35 | struct ref_tracker *tracker; |
36 | |
37 | stats = kmalloc(struct_size(stats, stacks, limit), |
38 | GFP_NOWAIT | __GFP_NOWARN); |
39 | if (!stats) |
40 | return ERR_PTR(error: -ENOMEM); |
41 | stats->total = 0; |
42 | stats->count = 0; |
43 | |
44 | list_for_each_entry(tracker, &dir->list, head) { |
45 | depot_stack_handle_t stack = tracker->alloc_stack_handle; |
46 | int i; |
47 | |
48 | ++stats->total; |
49 | for (i = 0; i < stats->count; ++i) |
50 | if (stats->stacks[i].stack_handle == stack) |
51 | break; |
52 | if (i >= limit) |
53 | continue; |
54 | if (i >= stats->count) { |
55 | stats->stacks[i].stack_handle = stack; |
56 | stats->stacks[i].count = 0; |
57 | ++stats->count; |
58 | } |
59 | ++stats->stacks[i].count; |
60 | } |
61 | |
62 | return stats; |
63 | } |
64 | |
65 | struct ostream { |
66 | char *buf; |
67 | int size, used; |
68 | }; |
69 | |
70 | #define pr_ostream(stream, fmt, args...) \ |
71 | ({ \ |
72 | struct ostream *_s = (stream); \ |
73 | \ |
74 | if (!_s->buf) { \ |
75 | pr_err(fmt, ##args); \ |
76 | } else { \ |
77 | int ret, len = _s->size - _s->used; \ |
78 | ret = snprintf(_s->buf + _s->used, len, pr_fmt(fmt), ##args); \ |
79 | _s->used += min(ret, len); \ |
80 | } \ |
81 | }) |
82 | |
83 | static void |
84 | __ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir, |
85 | unsigned int display_limit, struct ostream *s) |
86 | { |
87 | struct ref_tracker_dir_stats *stats; |
88 | unsigned int i = 0, skipped; |
89 | depot_stack_handle_t stack; |
90 | char *sbuf; |
91 | |
92 | lockdep_assert_held(&dir->lock); |
93 | |
94 | if (list_empty(head: &dir->list)) |
95 | return; |
96 | |
97 | stats = ref_tracker_get_stats(dir, limit: display_limit); |
98 | if (IS_ERR(ptr: stats)) { |
99 | pr_ostream(s, "%s@%pK: couldn't get stats, error %pe\n" , |
100 | dir->name, dir, stats); |
101 | return; |
102 | } |
103 | |
104 | sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT | __GFP_NOWARN); |
105 | |
106 | for (i = 0, skipped = stats->total; i < stats->count; ++i) { |
107 | stack = stats->stacks[i].stack_handle; |
108 | if (sbuf && !stack_depot_snprint(handle: stack, buf: sbuf, STACK_BUF_SIZE, spaces: 4)) |
109 | sbuf[0] = 0; |
110 | pr_ostream(s, "%s@%pK has %d/%d users at\n%s\n" , dir->name, dir, |
111 | stats->stacks[i].count, stats->total, sbuf); |
112 | skipped -= stats->stacks[i].count; |
113 | } |
114 | |
115 | if (skipped) |
116 | pr_ostream(s, "%s@%pK skipped reports about %d/%d users.\n" , |
117 | dir->name, dir, skipped, stats->total); |
118 | |
119 | kfree(objp: sbuf); |
120 | |
121 | kfree(objp: stats); |
122 | } |
123 | |
124 | void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir, |
125 | unsigned int display_limit) |
126 | { |
127 | struct ostream os = {}; |
128 | |
129 | __ref_tracker_dir_pr_ostream(dir, display_limit, s: &os); |
130 | } |
131 | EXPORT_SYMBOL(ref_tracker_dir_print_locked); |
132 | |
133 | void ref_tracker_dir_print(struct ref_tracker_dir *dir, |
134 | unsigned int display_limit) |
135 | { |
136 | unsigned long flags; |
137 | |
138 | spin_lock_irqsave(&dir->lock, flags); |
139 | ref_tracker_dir_print_locked(dir, display_limit); |
140 | spin_unlock_irqrestore(lock: &dir->lock, flags); |
141 | } |
142 | EXPORT_SYMBOL(ref_tracker_dir_print); |
143 | |
144 | int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size) |
145 | { |
146 | struct ostream os = { .buf = buf, .size = size }; |
147 | unsigned long flags; |
148 | |
149 | spin_lock_irqsave(&dir->lock, flags); |
150 | __ref_tracker_dir_pr_ostream(dir, display_limit: 16, s: &os); |
151 | spin_unlock_irqrestore(lock: &dir->lock, flags); |
152 | |
153 | return os.used; |
154 | } |
155 | EXPORT_SYMBOL(ref_tracker_dir_snprint); |
156 | |
157 | void ref_tracker_dir_exit(struct ref_tracker_dir *dir) |
158 | { |
159 | struct ref_tracker *tracker, *n; |
160 | unsigned long flags; |
161 | bool leak = false; |
162 | |
163 | dir->dead = true; |
164 | spin_lock_irqsave(&dir->lock, flags); |
165 | list_for_each_entry_safe(tracker, n, &dir->quarantine, head) { |
166 | list_del(entry: &tracker->head); |
167 | kfree(objp: tracker); |
168 | dir->quarantine_avail++; |
169 | } |
170 | if (!list_empty(head: &dir->list)) { |
171 | ref_tracker_dir_print_locked(dir, 16); |
172 | leak = true; |
173 | list_for_each_entry_safe(tracker, n, &dir->list, head) { |
174 | list_del(entry: &tracker->head); |
175 | kfree(objp: tracker); |
176 | } |
177 | } |
178 | spin_unlock_irqrestore(lock: &dir->lock, flags); |
179 | WARN_ON_ONCE(leak); |
180 | WARN_ON_ONCE(refcount_read(&dir->untracked) != 1); |
181 | WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1); |
182 | } |
183 | EXPORT_SYMBOL(ref_tracker_dir_exit); |
184 | |
185 | int ref_tracker_alloc(struct ref_tracker_dir *dir, |
186 | struct ref_tracker **trackerp, |
187 | gfp_t gfp) |
188 | { |
189 | unsigned long entries[REF_TRACKER_STACK_ENTRIES]; |
190 | struct ref_tracker *tracker; |
191 | unsigned int nr_entries; |
192 | gfp_t gfp_mask = gfp | __GFP_NOWARN; |
193 | unsigned long flags; |
194 | |
195 | WARN_ON_ONCE(dir->dead); |
196 | |
197 | if (!trackerp) { |
198 | refcount_inc(r: &dir->no_tracker); |
199 | return 0; |
200 | } |
201 | if (gfp & __GFP_DIRECT_RECLAIM) |
202 | gfp_mask |= __GFP_NOFAIL; |
203 | *trackerp = tracker = kzalloc(size: sizeof(*tracker), flags: gfp_mask); |
204 | if (unlikely(!tracker)) { |
205 | pr_err_once("memory allocation failure, unreliable refcount tracker.\n" ); |
206 | refcount_inc(r: &dir->untracked); |
207 | return -ENOMEM; |
208 | } |
209 | nr_entries = stack_trace_save(store: entries, ARRAY_SIZE(entries), skipnr: 1); |
210 | tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp_flags: gfp); |
211 | |
212 | spin_lock_irqsave(&dir->lock, flags); |
213 | list_add(new: &tracker->head, head: &dir->list); |
214 | spin_unlock_irqrestore(lock: &dir->lock, flags); |
215 | return 0; |
216 | } |
217 | EXPORT_SYMBOL_GPL(ref_tracker_alloc); |
218 | |
219 | int ref_tracker_free(struct ref_tracker_dir *dir, |
220 | struct ref_tracker **trackerp) |
221 | { |
222 | unsigned long entries[REF_TRACKER_STACK_ENTRIES]; |
223 | depot_stack_handle_t stack_handle; |
224 | struct ref_tracker *tracker; |
225 | unsigned int nr_entries; |
226 | unsigned long flags; |
227 | |
228 | WARN_ON_ONCE(dir->dead); |
229 | |
230 | if (!trackerp) { |
231 | refcount_dec(r: &dir->no_tracker); |
232 | return 0; |
233 | } |
234 | tracker = *trackerp; |
235 | if (!tracker) { |
236 | refcount_dec(r: &dir->untracked); |
237 | return -EEXIST; |
238 | } |
239 | nr_entries = stack_trace_save(store: entries, ARRAY_SIZE(entries), skipnr: 1); |
240 | stack_handle = stack_depot_save(entries, nr_entries, |
241 | GFP_NOWAIT | __GFP_NOWARN); |
242 | |
243 | spin_lock_irqsave(&dir->lock, flags); |
244 | if (tracker->dead) { |
245 | pr_err("reference already released.\n" ); |
246 | if (tracker->alloc_stack_handle) { |
247 | pr_err("allocated in:\n" ); |
248 | stack_depot_print(stack: tracker->alloc_stack_handle); |
249 | } |
250 | if (tracker->free_stack_handle) { |
251 | pr_err("freed in:\n" ); |
252 | stack_depot_print(stack: tracker->free_stack_handle); |
253 | } |
254 | spin_unlock_irqrestore(lock: &dir->lock, flags); |
255 | WARN_ON_ONCE(1); |
256 | return -EINVAL; |
257 | } |
258 | tracker->dead = true; |
259 | |
260 | tracker->free_stack_handle = stack_handle; |
261 | |
262 | list_move_tail(list: &tracker->head, head: &dir->quarantine); |
263 | if (!dir->quarantine_avail) { |
264 | tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head); |
265 | list_del(entry: &tracker->head); |
266 | } else { |
267 | dir->quarantine_avail--; |
268 | tracker = NULL; |
269 | } |
270 | spin_unlock_irqrestore(lock: &dir->lock, flags); |
271 | |
272 | kfree(objp: tracker); |
273 | return 0; |
274 | } |
275 | EXPORT_SYMBOL_GPL(ref_tracker_free); |
276 | |