1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* mm/kmemleak.c
4	*
5	* Copyright (C) 2008 ARM Limited
6	* Written by Catalin Marinas <catalin.marinas@arm.com>
7	*
8	* For more information on the algorithm and kmemleak usage, please see
9	* Documentation/dev-tools/kmemleak.rst.
10	*
11	* Notes on locking
12	* ----------------
13	*
14	* The following locks and mutexes are used by kmemleak:
15	*
16	* - kmemleak_lock (raw_spinlock_t): protects the object_list as well as
17	* del_state modifications and accesses to the object_tree_root (or
18	* object_phys_tree_root). The object_list is the main list holding the
19	* metadata (struct kmemleak_object) for the allocated memory blocks.
20	* The object_tree_root and object_phys_tree_root are red
21	* black trees used to look-up metadata based on a pointer to the
22	* corresponding memory block. The object_phys_tree_root is for objects
23	* allocated with physical address. The kmemleak_object structures are
24	* added to the object_list and object_tree_root (or object_phys_tree_root)
25	* in the create_object() function called from the kmemleak_alloc() (or
26	* kmemleak_alloc_phys()) callback and removed in delete_object() called from
27	* the kmemleak_free() callback
28	* - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
29	* Accesses to the metadata (e.g. count) are protected by this lock. Note
30	* that some members of this structure may be protected by other means
31	* (atomic or kmemleak_lock). This lock is also held when scanning the
32	* corresponding memory block to avoid the kernel freeing it via the
33	* kmemleak_free() callback. This is less heavyweight than holding a global
34	* lock like kmemleak_lock during scanning.
35	* - scan_mutex (mutex): ensures that only one thread may scan the memory for
36	* unreferenced objects at a time. The gray_list contains the objects which
37	* are already referenced or marked as false positives and need to be
38	* scanned. This list is only modified during a scanning episode when the
39	* scan_mutex is held. At the end of a scan, the gray_list is always empty.
40	* Note that the kmemleak_object.use_count is incremented when an object is
41	* added to the gray_list and therefore cannot be freed. This mutex also
42	* prevents multiple users of the "kmemleak" debugfs file together with
43	* modifications to the memory scanning parameters including the scan_thread
44	* pointer
45	*
46	* Locks and mutexes are acquired/nested in the following order:
47	*
48	* scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
49	*
50	* No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
51	* regions.
52	*
53	* The kmemleak_object structures have a use_count incremented or decremented
54	* using the get_object()/put_object() functions. When the use_count becomes
55	* 0, this count can no longer be incremented and put_object() schedules the
56	* kmemleak_object freeing via an RCU callback. All calls to the get_object()
57	* function must be protected by rcu_read_lock() to avoid accessing a freed
58	* structure.
59	*/
60
61	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
62
63	#include <linux/init.h>
64	#include <linux/kernel.h>
65	#include <linux/list.h>
66	#include <linux/sched/signal.h>
67	#include <linux/sched/task.h>
68	#include <linux/sched/task_stack.h>
69	#include <linux/jiffies.h>
70	#include <linux/delay.h>
71	#include <linux/export.h>
72	#include <linux/kthread.h>
73	#include <linux/rbtree.h>
74	#include <linux/fs.h>
75	#include <linux/debugfs.h>
76	#include <linux/seq_file.h>
77	#include <linux/cpumask.h>
78	#include <linux/spinlock.h>
79	#include <linux/module.h>
80	#include <linux/mutex.h>
81	#include <linux/rcupdate.h>
82	#include <linux/stacktrace.h>
83	#include <linux/stackdepot.h>
84	#include <linux/cache.h>
85	#include <linux/percpu.h>
86	#include <linux/memblock.h>
87	#include <linux/pfn.h>
88	#include <linux/mmzone.h>
89	#include <linux/slab.h>
90	#include <linux/thread_info.h>
91	#include <linux/err.h>
92	#include <linux/uaccess.h>
93	#include <linux/string.h>
94	#include <linux/nodemask.h>
95	#include <linux/mm.h>
96	#include <linux/workqueue.h>
97	#include <linux/crc32.h>
98
99	#include <asm/sections.h>
100	#include <asm/processor.h>
101	#include <linux/atomic.h>
102
103	#include <linux/kasan.h>
104	#include <linux/kfence.h>
105	#include <linux/kmemleak.h>
106	#include <linux/memory_hotplug.h>
107
108	/*
109	* Kmemleak configuration and common defines.
110	*/
111	#define MAX_TRACE 16 /* stack trace length */
112	#define MSECS_MIN_AGE 5000 /* minimum object age for reporting */
113	#define SECS_FIRST_SCAN 60 /* delay before the first scan */
114	#define SECS_SCAN_WAIT 600 /* subsequent auto scanning delay */
115	#define MAX_SCAN_SIZE 4096 /* maximum size of a scanned block */
116
117	#define BYTES_PER_POINTER sizeof(void *)
118
119	/* GFP bitmask for kmemleak internal allocations */
120	#define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC | \
121	__GFP_NOLOCKDEP)) | \
122	__GFP_NORETRY | __GFP_NOMEMALLOC | \
123	__GFP_NOWARN)
124
125	/* scanning area inside a memory block */
126	struct kmemleak_scan_area {
127	struct hlist_node node;
128	unsigned long start;
129	size_t size;
130	};
131
132	#define KMEMLEAK_GREY 0
133	#define KMEMLEAK_BLACK -1
134
135	/*
136	* Structure holding the metadata for each allocated memory block.
137	* Modifications to such objects should be made while holding the
138	* object->lock. Insertions or deletions from object_list, gray_list or
139	* rb_node are already protected by the corresponding locks or mutex (see
140	* the notes on locking above). These objects are reference-counted
141	* (use_count) and freed using the RCU mechanism.
142	*/
143	struct kmemleak_object {
144	raw_spinlock_t lock;
145	unsigned int flags; /* object status flags */
146	struct list_head object_list;
147	struct list_head gray_list;
148	struct rb_node rb_node;
149	struct rcu_head rcu; /* object_list lockless traversal */
150	/* object usage count; object freed when use_count == 0 */
151	atomic_t use_count;
152	unsigned int del_state; /* deletion state */
153	unsigned long pointer;
154	size_t size;
155	/* pass surplus references to this pointer */
156	unsigned long excess_ref;
157	/* minimum number of a pointers found before it is considered leak */
158	int min_count;
159	/* the total number of pointers found pointing to this object */
160	int count;
161	/* checksum for detecting modified objects */
162	u32 checksum;
163	/* memory ranges to be scanned inside an object (empty for all) */
164	struct hlist_head area_list;
165	depot_stack_handle_t trace_handle;
166	unsigned long jiffies; /* creation timestamp */
167	pid_t pid; /* pid of the current task */
168	char comm[TASK_COMM_LEN]; /* executable name */
169	};
170
171	/* flag representing the memory block allocation status */
172	#define OBJECT_ALLOCATED (1 << 0)
173	/* flag set after the first reporting of an unreference object */
174	#define OBJECT_REPORTED (1 << 1)
175	/* flag set to not scan the object */
176	#define OBJECT_NO_SCAN (1 << 2)
177	/* flag set to fully scan the object when scan_area allocation failed */
178	#define OBJECT_FULL_SCAN (1 << 3)
179	/* flag set for object allocated with physical address */
180	#define OBJECT_PHYS (1 << 4)
181
182	/* set when __remove_object() called */
183	#define DELSTATE_REMOVED (1 << 0)
184	/* set to temporarily prevent deletion from object_list */
185	#define DELSTATE_NO_DELETE (1 << 1)
186
187	#define HEX_PREFIX " "
188	/* number of bytes to print per line; must be 16 or 32 */
189	#define HEX_ROW_SIZE 16
190	/* number of bytes to print at a time (1, 2, 4, 8) */
191	#define HEX_GROUP_SIZE 1
192	/* include ASCII after the hex output */
193	#define HEX_ASCII 1
194	/* max number of lines to be printed */
195	#define HEX_MAX_LINES 2
196
197	/* the list of all allocated objects */
198	static LIST_HEAD(object_list);
199	/* the list of gray-colored objects (see color_gray comment below) */
200	static LIST_HEAD(gray_list);
201	/* memory pool allocation */
202	static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
203	static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
204	static LIST_HEAD(mem_pool_free_list);
205	/* search tree for object boundaries */
206	static struct rb_root object_tree_root = RB_ROOT;
207	/* search tree for object (with OBJECT_PHYS flag) boundaries */
208	static struct rb_root object_phys_tree_root = RB_ROOT;
209	/* protecting the access to object_list, object_tree_root (or object_phys_tree_root) */
210	static DEFINE_RAW_SPINLOCK(kmemleak_lock);
211
212	/* allocation caches for kmemleak internal data */
213	static struct kmem_cache *object_cache;
214	static struct kmem_cache *scan_area_cache;
215
216	/* set if tracing memory operations is enabled */
217	static int kmemleak_enabled = 1;
218	/* same as above but only for the kmemleak_free() callback */
219	static int kmemleak_free_enabled = 1;
220	/* set in the late_initcall if there were no errors */
221	static int kmemleak_late_initialized;
222	/* set if a kmemleak warning was issued */
223	static int kmemleak_warning;
224	/* set if a fatal kmemleak error has occurred */
225	static int kmemleak_error;
226
227	/* minimum and maximum address that may be valid pointers */
228	static unsigned long min_addr = ULONG_MAX;
229	static unsigned long max_addr;
230
231	static struct task_struct *scan_thread;
232	/* used to avoid reporting of recently allocated objects */
233	static unsigned long jiffies_min_age;
234	static unsigned long jiffies_last_scan;
235	/* delay between automatic memory scannings */
236	static unsigned long jiffies_scan_wait;
237	/* enables or disables the task stacks scanning */
238	static int kmemleak_stack_scan = 1;
239	/* protects the memory scanning, parameters and debug/kmemleak file access */
240	static DEFINE_MUTEX(scan_mutex);
241	/* setting kmemleak=on, will set this var, skipping the disable */
242	static int kmemleak_skip_disable;
243	/* If there are leaks that can be reported */
244	static bool kmemleak_found_leaks;
245
246	static bool kmemleak_verbose;
247	module_param_named(verbose, kmemleak_verbose, bool, 0600);
248
249	static void kmemleak_disable(void);
250
251	/*
252	* Print a warning and dump the stack trace.
253	*/
254	#define kmemleak_warn(x...) do { \
255	pr_warn(x); \
256	dump_stack(); \
257	kmemleak_warning = 1; \
258	} while (0)
259
260	/*
261	* Macro invoked when a serious kmemleak condition occurred and cannot be
262	* recovered from. Kmemleak will be disabled and further allocation/freeing
263	* tracing no longer available.
264	*/
265	#define kmemleak_stop(x...) do { \
266	kmemleak_warn(x); \
267	kmemleak_disable(); \
268	} while (0)
269
270	#define warn_or_seq_printf(seq, fmt, ...) do { \
271	if (seq) \
272	seq_printf(seq, fmt, ##__VA_ARGS__); \
273	else \
274	pr_warn(fmt, ##__VA_ARGS__); \
275	} while (0)
276
277	static void warn_or_seq_hex_dump(struct seq_file *seq, int prefix_type,
278	int rowsize, int groupsize, const void *buf,
279	size_t len, bool ascii)
280	{
281	if (seq)
282	seq_hex_dump(m: seq, HEX_PREFIX, prefix_type, rowsize, groupsize,
283	buf, len, ascii);
284	else
285	print_hex_dump(KERN_WARNING, pr_fmt(HEX_PREFIX), prefix_type,
286	rowsize, groupsize, buf, len, ascii);
287	}
288
289	/*
290	* Printing of the objects hex dump to the seq file. The number of lines to be
291	* printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
292	* actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
293	* with the object->lock held.
294	*/
295	static void hex_dump_object(struct seq_file *seq,
296	struct kmemleak_object *object)
297	{
298	const u8 *ptr = (const u8 *)object->pointer;
299	size_t len;
300
301	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
302	return;
303
304	/* limit the number of lines to HEX_MAX_LINES */
305	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
306
307	warn_or_seq_printf(seq, " hex dump (first %zu bytes):\n", len);
308	kasan_disable_current();
309	warn_or_seq_hex_dump(seq, prefix_type: DUMP_PREFIX_NONE, HEX_ROW_SIZE,
310	HEX_GROUP_SIZE, buf: kasan_reset_tag(addr: (void *)ptr), len, HEX_ASCII);
311	kasan_enable_current();
312	}
313
314	/*
315	* Object colors, encoded with count and min_count:
316	* - white - orphan object, not enough references to it (count < min_count)
317	* - gray - not orphan, not marked as false positive (min_count == 0) or
318	* sufficient references to it (count >= min_count)
319	* - black - ignore, it doesn't contain references (e.g. text section)
320	* (min_count == -1). No function defined for this color.
321	* Newly created objects don't have any color assigned (object->count == -1)
322	* before the next memory scan when they become white.
323	*/
324	static bool color_white(const struct kmemleak_object *object)
325	{
326	return object->count != KMEMLEAK_BLACK &&
327	object->count < object->min_count;
328	}
329
330	static bool color_gray(const struct kmemleak_object *object)
331	{
332	return object->min_count != KMEMLEAK_BLACK &&
333	object->count >= object->min_count;
334	}
335
336	/*
337	* Objects are considered unreferenced only if their color is white, they have
338	* not be deleted and have a minimum age to avoid false positives caused by
339	* pointers temporarily stored in CPU registers.
340	*/
341	static bool unreferenced_object(struct kmemleak_object *object)
342	{
343	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
344	time_before_eq(object->jiffies + jiffies_min_age,
345	jiffies_last_scan);
346	}
347
348	/*
349	* Printing of the unreferenced objects information to the seq file. The
350	* print_unreferenced function must be called with the object->lock held.
351	*/
352	static void print_unreferenced(struct seq_file *seq,
353	struct kmemleak_object *object)
354	{
355	int i;
356	unsigned long *entries;
357	unsigned int nr_entries;
358	unsigned int msecs_age = jiffies_to_msecs(j: jiffies - object->jiffies);
359
360	nr_entries = stack_depot_fetch(handle: object->trace_handle, entries: &entries);
361	warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
362	object->pointer, object->size);
363	warn_or_seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
364	object->comm, object->pid, object->jiffies,
365	msecs_age / 1000, msecs_age % 1000);
366	hex_dump_object(seq, object);
367	warn_or_seq_printf(seq, " backtrace:\n");
368
369	for (i = 0; i < nr_entries; i++) {
370	void *ptr = (void *)entries[i];
371	warn_or_seq_printf(seq, " [<%pK>] %pS\n", ptr, ptr);
372	}
373	}
374
375	/*
376	* Print the kmemleak_object information. This function is used mainly for
377	* debugging special cases when kmemleak operations. It must be called with
378	* the object->lock held.
379	*/
380	static void dump_object_info(struct kmemleak_object *object)
381	{
382	pr_notice("Object 0x%08lx (size %zu):\n",
383	object->pointer, object->size);
384	pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
385	object->comm, object->pid, object->jiffies);
386	pr_notice(" min_count = %d\n", object->min_count);
387	pr_notice(" count = %d\n", object->count);
388	pr_notice(" flags = 0x%x\n", object->flags);
389	pr_notice(" checksum = %u\n", object->checksum);
390	pr_notice(" backtrace:\n");
391	if (object->trace_handle)
392	stack_depot_print(stack: object->trace_handle);
393	}
394
395	/*
396	* Look-up a memory block metadata (kmemleak_object) in the object search
397	* tree based on a pointer value. If alias is 0, only values pointing to the
398	* beginning of the memory block are allowed. The kmemleak_lock must be held
399	* when calling this function.
400	*/
401	static struct kmemleak_object *__lookup_object(unsigned long ptr, int alias,
402	bool is_phys)
403	{
404	struct rb_node *rb = is_phys ? object_phys_tree_root.rb_node :
405	object_tree_root.rb_node;
406	unsigned long untagged_ptr = (unsigned long)kasan_reset_tag(addr: (void *)ptr);
407
408	while (rb) {
409	struct kmemleak_object *object;
410	unsigned long untagged_objp;
411
412	object = rb_entry(rb, struct kmemleak_object, rb_node);
413	untagged_objp = (unsigned long)kasan_reset_tag(addr: (void *)object->pointer);
414
415	if (untagged_ptr < untagged_objp)
416	rb = object->rb_node.rb_left;
417	else if (untagged_objp + object->size <= untagged_ptr)
418	rb = object->rb_node.rb_right;
419	else if (untagged_objp == untagged_ptr || alias)
420	return object;
421	else {
422	kmemleak_warn("Found object by alias at 0x%08lx\n",
423	ptr);
424	dump_object_info(object);
425	break;
426	}
427	}
428	return NULL;
429	}
430
431	/* Look-up a kmemleak object which allocated with virtual address. */
432	static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
433	{
434	return __lookup_object(ptr, alias, is_phys: false);
435	}
436
437	/*
438	* Increment the object use_count. Return 1 if successful or 0 otherwise. Note
439	* that once an object's use_count reached 0, the RCU freeing was already
440	* registered and the object should no longer be used. This function must be
441	* called under the protection of rcu_read_lock().
442	*/
443	static int get_object(struct kmemleak_object *object)
444	{
445	return atomic_inc_not_zero(v: &object->use_count);
446	}
447
448	/*
449	* Memory pool allocation and freeing. kmemleak_lock must not be held.
450	*/
451	static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
452	{
453	unsigned long flags;
454	struct kmemleak_object *object;
455
456	/* try the slab allocator first */
457	if (object_cache) {
458	object = kmem_cache_alloc(cachep: object_cache, gfp_kmemleak_mask(gfp));
459	if (object)
460	return object;
461	}
462
463	/* slab allocation failed, try the memory pool */
464	raw_spin_lock_irqsave(&kmemleak_lock, flags);
465	object = list_first_entry_or_null(&mem_pool_free_list,
466	typeof(*object), object_list);
467	if (object)
468	list_del(entry: &object->object_list);
469	else if (mem_pool_free_count)
470	object = &mem_pool[--mem_pool_free_count];
471	else
472	pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
473	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
474
475	return object;
476	}
477
478	/*
479	* Return the object to either the slab allocator or the memory pool.
480	*/
481	static void mem_pool_free(struct kmemleak_object *object)
482	{
483	unsigned long flags;
484
485	if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {
486	kmem_cache_free(s: object_cache, objp: object);
487	return;
488	}
489
490	/* add the object to the memory pool free list */
491	raw_spin_lock_irqsave(&kmemleak_lock, flags);
492	list_add(new: &object->object_list, head: &mem_pool_free_list);
493	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
494	}
495
496	/*
497	* RCU callback to free a kmemleak_object.
498	*/
499	static void free_object_rcu(struct rcu_head *rcu)
500	{
501	struct hlist_node *tmp;
502	struct kmemleak_scan_area *area;
503	struct kmemleak_object *object =
504	container_of(rcu, struct kmemleak_object, rcu);
505
506	/*
507	* Once use_count is 0 (guaranteed by put_object), there is no other
508	* code accessing this object, hence no need for locking.
509	*/
510	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
511	hlist_del(n: &area->node);
512	kmem_cache_free(s: scan_area_cache, objp: area);
513	}
514	mem_pool_free(object);
515	}
516
517	/*
518	* Decrement the object use_count. Once the count is 0, free the object using
519	* an RCU callback. Since put_object() may be called via the kmemleak_free() ->
520	* delete_object() path, the delayed RCU freeing ensures that there is no
521	* recursive call to the kernel allocator. Lock-less RCU object_list traversal
522	* is also possible.
523	*/
524	static void put_object(struct kmemleak_object *object)
525	{
526	if (!atomic_dec_and_test(v: &object->use_count))
527	return;
528
529	/* should only get here after delete_object was called */
530	WARN_ON(object->flags & OBJECT_ALLOCATED);
531
532	/*
533	* It may be too early for the RCU callbacks, however, there is no
534	* concurrent object_list traversal when !object_cache and all objects
535	* came from the memory pool. Free the object directly.
536	*/
537	if (object_cache)
538	call_rcu(head: &object->rcu, func: free_object_rcu);
539	else
540	free_object_rcu(rcu: &object->rcu);
541	}
542
543	/*
544	* Look up an object in the object search tree and increase its use_count.
545	*/
546	static struct kmemleak_object *__find_and_get_object(unsigned long ptr, int alias,
547	bool is_phys)
548	{
549	unsigned long flags;
550	struct kmemleak_object *object;
551
552	rcu_read_lock();
553	raw_spin_lock_irqsave(&kmemleak_lock, flags);
554	object = __lookup_object(ptr, alias, is_phys);
555	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
556
557	/* check whether the object is still available */
558	if (object && !get_object(object))
559	object = NULL;
560	rcu_read_unlock();
561
562	return object;
563	}
564
565	/* Look up and get an object which allocated with virtual address. */
566	static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
567	{
568	return __find_and_get_object(ptr, alias, is_phys: false);
569	}
570
571	/*
572	* Remove an object from the object_tree_root (or object_phys_tree_root)
573	* and object_list. Must be called with the kmemleak_lock held _if_ kmemleak
574	* is still enabled.
575	*/
576	static void __remove_object(struct kmemleak_object *object)
577	{
578	rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?
579	&object_phys_tree_root :
580	&object_tree_root);
581	if (!(object->del_state & DELSTATE_NO_DELETE))
582	list_del_rcu(entry: &object->object_list);
583	object->del_state |= DELSTATE_REMOVED;
584	}
585
586	static struct kmemleak_object *__find_and_remove_object(unsigned long ptr,
587	int alias,
588	bool is_phys)
589	{
590	struct kmemleak_object *object;
591
592	object = __lookup_object(ptr, alias, is_phys);
593	if (object)
594	__remove_object(object);
595
596	return object;
597	}
598
599	/*
600	* Look up an object in the object search tree and remove it from both
601	* object_tree_root (or object_phys_tree_root) and object_list. The
602	* returned object's use_count should be at least 1, as initially set
603	* by create_object().
604	*/
605	static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias,
606	bool is_phys)
607	{
608	unsigned long flags;
609	struct kmemleak_object *object;
610
611	raw_spin_lock_irqsave(&kmemleak_lock, flags);
612	object = __find_and_remove_object(ptr, alias, is_phys);
613	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
614
615	return object;
616	}
617
618	static noinline depot_stack_handle_t set_track_prepare(void)
619	{
620	depot_stack_handle_t trace_handle;
621	unsigned long entries[MAX_TRACE];
622	unsigned int nr_entries;
623
624	/*
625	* Use object_cache to determine whether kmemleak_init() has
626	* been invoked. stack_depot_early_init() is called before
627	* kmemleak_init() in mm_core_init().
628	*/
629	if (!object_cache)
630	return 0;
631	nr_entries = stack_trace_save(store: entries, ARRAY_SIZE(entries), skipnr: 3);
632	trace_handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);
633
634	return trace_handle;
635	}
636
637	static struct kmemleak_object *__alloc_object(gfp_t gfp)
638	{
639	struct kmemleak_object *object;
640
641	object = mem_pool_alloc(gfp);
642	if (!object) {
643	pr_warn("Cannot allocate a kmemleak_object structure\n");
644	kmemleak_disable();
645	}
646
647	return object;
648	}
649
650	static int __link_object(struct kmemleak_object *object, unsigned long ptr,
651	size_t size, int min_count, bool is_phys)
652	{
653
654	struct kmemleak_object *parent;
655	struct rb_node **link, *rb_parent;
656	unsigned long untagged_ptr;
657	unsigned long untagged_objp;
658
659	INIT_LIST_HEAD(list: &object->object_list);
660	INIT_LIST_HEAD(list: &object->gray_list);
661	INIT_HLIST_HEAD(&object->area_list);
662	raw_spin_lock_init(&object->lock);
663	atomic_set(v: &object->use_count, i: 1);
664	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);
665	object->pointer = ptr;
666	object->size = kfence_ksize(addr: (void *)ptr) ?: size;
667	object->excess_ref = 0;
668	object->min_count = min_count;
669	object->count = 0; /* white color initially */
670	object->jiffies = jiffies;
671	object->checksum = 0;
672	object->del_state = 0;
673
674	/* task information */
675	if (in_hardirq()) {
676	object->pid = 0;
677	strncpy(p: object->comm, q: "hardirq", size: sizeof(object->comm));
678	} else if (in_serving_softirq()) {
679	object->pid = 0;
680	strncpy(p: object->comm, q: "softirq", size: sizeof(object->comm));
681	} else {
682	object->pid = current->pid;
683	/*
684	* There is a small chance of a race with set_task_comm(),
685	* however using get_task_comm() here may cause locking
686	* dependency issues with current->alloc_lock. In the worst
687	* case, the command line is not correct.
688	*/
689	strncpy(p: object->comm, current->comm, size: sizeof(object->comm));
690	}
691
692	/* kernel backtrace */
693	object->trace_handle = set_track_prepare();
694
695	untagged_ptr = (unsigned long)kasan_reset_tag(addr: (void *)ptr);
696	/*
697	* Only update min_addr and max_addr with object
698	* storing virtual address.
699	*/
700	if (!is_phys) {
701	min_addr = min(min_addr, untagged_ptr);
702	max_addr = max(max_addr, untagged_ptr + size);
703	}
704	link = is_phys ? &object_phys_tree_root.rb_node :
705	&object_tree_root.rb_node;
706	rb_parent = NULL;
707	while (*link) {
708	rb_parent = *link;
709	parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
710	untagged_objp = (unsigned long)kasan_reset_tag(addr: (void *)parent->pointer);
711	if (untagged_ptr + size <= untagged_objp)
712	link = &parent->rb_node.rb_left;
713	else if (untagged_objp + parent->size <= untagged_ptr)
714	link = &parent->rb_node.rb_right;
715	else {
716	kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
717	ptr);
718	/*
719	* No need for parent->lock here since "parent" cannot
720	* be freed while the kmemleak_lock is held.
721	*/
722	dump_object_info(object: parent);
723	return -EEXIST;
724	}
725	}
726	rb_link_node(node: &object->rb_node, parent: rb_parent, rb_link: link);
727	rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :
728	&object_tree_root);
729	list_add_tail_rcu(new: &object->object_list, head: &object_list);
730
731	return 0;
732	}
733
734	/*
735	* Create the metadata (struct kmemleak_object) corresponding to an allocated
736	* memory block and add it to the object_list and object_tree_root (or
737	* object_phys_tree_root).
738	*/
739	static void __create_object(unsigned long ptr, size_t size,
740	int min_count, gfp_t gfp, bool is_phys)
741	{
742	struct kmemleak_object *object;
743	unsigned long flags;
744	int ret;
745
746	object = __alloc_object(gfp);
747	if (!object)
748	return;
749
750	raw_spin_lock_irqsave(&kmemleak_lock, flags);
751	ret = __link_object(object, ptr, size, min_count, is_phys);
752	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
753	if (ret)
754	mem_pool_free(object);
755	}
756
757	/* Create kmemleak object which allocated with virtual address. */
758	static void create_object(unsigned long ptr, size_t size,
759	int min_count, gfp_t gfp)
760	{
761	__create_object(ptr, size, min_count, gfp, is_phys: false);
762	}
763
764	/* Create kmemleak object which allocated with physical address. */
765	static void create_object_phys(unsigned long ptr, size_t size,
766	int min_count, gfp_t gfp)
767	{
768	__create_object(ptr, size, min_count, gfp, is_phys: true);
769	}
770
771	/*
772	* Mark the object as not allocated and schedule RCU freeing via put_object().
773	*/
774	static void __delete_object(struct kmemleak_object *object)
775	{
776	unsigned long flags;
777
778	WARN_ON(!(object->flags & OBJECT_ALLOCATED));
779	WARN_ON(atomic_read(&object->use_count) < 1);
780
781	/*
782	* Locking here also ensures that the corresponding memory block
783	* cannot be freed when it is being scanned.
784	*/
785	raw_spin_lock_irqsave(&object->lock, flags);
786	object->flags &= ~OBJECT_ALLOCATED;
787	raw_spin_unlock_irqrestore(&object->lock, flags);
788	put_object(object);
789	}
790
791	/*
792	* Look up the metadata (struct kmemleak_object) corresponding to ptr and
793	* delete it.
794	*/
795	static void delete_object_full(unsigned long ptr)
796	{
797	struct kmemleak_object *object;
798
799	object = find_and_remove_object(ptr, alias: 0, is_phys: false);
800	if (!object) {
801	#ifdef DEBUG
802	kmemleak_warn("Freeing unknown object at 0x%08lx\n",
803	ptr);
804	#endif
805	return;
806	}
807	__delete_object(object);
808	}
809
810	/*
811	* Look up the metadata (struct kmemleak_object) corresponding to ptr and
812	* delete it. If the memory block is partially freed, the function may create
813	* additional metadata for the remaining parts of the block.
814	*/
815	static void delete_object_part(unsigned long ptr, size_t size, bool is_phys)
816	{
817	struct kmemleak_object *object, *object_l, *object_r;
818	unsigned long start, end, flags;
819
820	object_l = __alloc_object(GFP_KERNEL);
821	if (!object_l)
822	return;
823
824	object_r = __alloc_object(GFP_KERNEL);
825	if (!object_r)
826	goto out;
827
828	raw_spin_lock_irqsave(&kmemleak_lock, flags);
829	object = __find_and_remove_object(ptr, alias: 1, is_phys);
830	if (!object) {
831	#ifdef DEBUG
832	kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
833	ptr, size);
834	#endif
835	goto unlock;
836	}
837
838	/*
839	* Create one or two objects that may result from the memory block
840	* split. Note that partial freeing is only done by free_bootmem() and
841	* this happens before kmemleak_init() is called.
842	*/
843	start = object->pointer;
844	end = object->pointer + object->size;
845	if ((ptr > start) &&
846	!__link_object(object: object_l, ptr: start, size: ptr - start,
847	min_count: object->min_count, is_phys))
848	object_l = NULL;
849	if ((ptr + size < end) &&
850	!__link_object(object: object_r, ptr: ptr + size, size: end - ptr - size,
851	min_count: object->min_count, is_phys))
852	object_r = NULL;
853
854	unlock:
855	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
856	if (object)
857	__delete_object(object);
858
859	out:
860	if (object_l)
861	mem_pool_free(object: object_l);
862	if (object_r)
863	mem_pool_free(object: object_r);
864	}
865
866	static void __paint_it(struct kmemleak_object *object, int color)
867	{
868	object->min_count = color;
869	if (color == KMEMLEAK_BLACK)
870	object->flags |= OBJECT_NO_SCAN;
871	}
872
873	static void paint_it(struct kmemleak_object *object, int color)
874	{
875	unsigned long flags;
876
877	raw_spin_lock_irqsave(&object->lock, flags);
878	__paint_it(object, color);
879	raw_spin_unlock_irqrestore(&object->lock, flags);
880	}
881
882	static void paint_ptr(unsigned long ptr, int color, bool is_phys)
883	{
884	struct kmemleak_object *object;
885
886	object = __find_and_get_object(ptr, alias: 0, is_phys);
887	if (!object) {
888	kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
889	ptr,
890	(color == KMEMLEAK_GREY) ? "Grey" :
891	(color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
892	return;
893	}
894	paint_it(object, color);
895	put_object(object);
896	}
897
898	/*
899	* Mark an object permanently as gray-colored so that it can no longer be
900	* reported as a leak. This is used in general to mark a false positive.
901	*/
902	static void make_gray_object(unsigned long ptr)
903	{
904	paint_ptr(ptr, KMEMLEAK_GREY, is_phys: false);
905	}
906
907	/*
908	* Mark the object as black-colored so that it is ignored from scans and
909	* reporting.
910	*/
911	static void make_black_object(unsigned long ptr, bool is_phys)
912	{
913	paint_ptr(ptr, KMEMLEAK_BLACK, is_phys);
914	}
915
916	/*
917	* Add a scanning area to the object. If at least one such area is added,
918	* kmemleak will only scan these ranges rather than the whole memory block.
919	*/
920	static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
921	{
922	unsigned long flags;
923	struct kmemleak_object *object;
924	struct kmemleak_scan_area *area = NULL;
925	unsigned long untagged_ptr;
926	unsigned long untagged_objp;
927
928	object = find_and_get_object(ptr, alias: 1);
929	if (!object) {
930	kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
931	ptr);
932	return;
933	}
934
935	untagged_ptr = (unsigned long)kasan_reset_tag(addr: (void *)ptr);
936	untagged_objp = (unsigned long)kasan_reset_tag(addr: (void *)object->pointer);
937
938	if (scan_area_cache)
939	area = kmem_cache_alloc(cachep: scan_area_cache, gfp_kmemleak_mask(gfp));
940
941	raw_spin_lock_irqsave(&object->lock, flags);
942	if (!area) {
943	pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
944	/* mark the object for full scan to avoid false positives */
945	object->flags |= OBJECT_FULL_SCAN;
946	goto out_unlock;
947	}
948	if (size == SIZE_MAX) {
949	size = untagged_objp + object->size - untagged_ptr;
950	} else if (untagged_ptr + size > untagged_objp + object->size) {
951	kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
952	dump_object_info(object);
953	kmem_cache_free(s: scan_area_cache, objp: area);
954	goto out_unlock;
955	}
956
957	INIT_HLIST_NODE(h: &area->node);
958	area->start = ptr;
959	area->size = size;
960
961	hlist_add_head(n: &area->node, h: &object->area_list);
962	out_unlock:
963	raw_spin_unlock_irqrestore(&object->lock, flags);
964	put_object(object);
965	}
966
967	/*
968	* Any surplus references (object already gray) to 'ptr' are passed to
969	* 'excess_ref'. This is used in the vmalloc() case where a pointer to
970	* vm_struct may be used as an alternative reference to the vmalloc'ed object
971	* (see free_thread_stack()).
972	*/
973	static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
974	{
975	unsigned long flags;
976	struct kmemleak_object *object;
977
978	object = find_and_get_object(ptr, alias: 0);
979	if (!object) {
980	kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
981	ptr);
982	return;
983	}
984
985	raw_spin_lock_irqsave(&object->lock, flags);
986	object->excess_ref = excess_ref;
987	raw_spin_unlock_irqrestore(&object->lock, flags);
988	put_object(object);
989	}
990
991	/*
992	* Set the OBJECT_NO_SCAN flag for the object corresponding to the give
993	* pointer. Such object will not be scanned by kmemleak but references to it
994	* are searched.
995	*/
996	static void object_no_scan(unsigned long ptr)
997	{
998	unsigned long flags;
999	struct kmemleak_object *object;
1000
1001	object = find_and_get_object(ptr, alias: 0);
1002	if (!object) {
1003	kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
1004	return;
1005	}
1006
1007	raw_spin_lock_irqsave(&object->lock, flags);
1008	object->flags |= OBJECT_NO_SCAN;
1009	raw_spin_unlock_irqrestore(&object->lock, flags);
1010	put_object(object);
1011	}
1012
1013	/**
1014	* kmemleak_alloc - register a newly allocated object
1015	* @ptr: pointer to beginning of the object
1016	* @size: size of the object
1017	* @min_count: minimum number of references to this object. If during memory
1018	* scanning a number of references less than @min_count is found,
1019	* the object is reported as a memory leak. If @min_count is 0,
1020	* the object is never reported as a leak. If @min_count is -1,
1021	* the object is ignored (not scanned and not reported as a leak)
1022	* @gfp: kmalloc() flags used for kmemleak internal memory allocations
1023	*
1024	* This function is called from the kernel allocators when a new object
1025	* (memory block) is allocated (kmem_cache_alloc, kmalloc etc.).
1026	*/
1027	void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
1028	gfp_t gfp)
1029	{
1030	pr_debug("%s(0x%px, %zu, %d)\n", __func__, ptr, size, min_count);
1031
1032	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1033	create_object(ptr: (unsigned long)ptr, size, min_count, gfp);
1034	}
1035	EXPORT_SYMBOL_GPL(kmemleak_alloc);
1036
1037	/**
1038	* kmemleak_alloc_percpu - register a newly allocated __percpu object
1039	* @ptr: __percpu pointer to beginning of the object
1040	* @size: size of the object
1041	* @gfp: flags used for kmemleak internal memory allocations
1042	*
1043	* This function is called from the kernel percpu allocator when a new object
1044	* (memory block) is allocated (alloc_percpu).
1045	*/
1046	void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
1047	gfp_t gfp)
1048	{
1049	unsigned int cpu;
1050
1051	pr_debug("%s(0x%px, %zu)\n", __func__, ptr, size);
1052
1053	/*
1054	* Percpu allocations are only scanned and not reported as leaks
1055	* (min_count is set to 0).
1056	*/
1057	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1058	for_each_possible_cpu(cpu)
1059	create_object(ptr: (unsigned long)per_cpu_ptr(ptr, cpu),
1060	size, min_count: 0, gfp);
1061	}
1062	EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
1063
1064	/**
1065	* kmemleak_vmalloc - register a newly vmalloc'ed object
1066	* @area: pointer to vm_struct
1067	* @size: size of the object
1068	* @gfp: __vmalloc() flags used for kmemleak internal memory allocations
1069	*
1070	* This function is called from the vmalloc() kernel allocator when a new
1071	* object (memory block) is allocated.
1072	*/
1073	void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
1074	{
1075	pr_debug("%s(0x%px, %zu)\n", __func__, area, size);
1076
1077	/*
1078	* A min_count = 2 is needed because vm_struct contains a reference to
1079	* the virtual address of the vmalloc'ed block.
1080	*/
1081	if (kmemleak_enabled) {
1082	create_object(ptr: (unsigned long)area->addr, size, min_count: 2, gfp);
1083	object_set_excess_ref(ptr: (unsigned long)area,
1084	excess_ref: (unsigned long)area->addr);
1085	}
1086	}
1087	EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
1088
1089	/**
1090	* kmemleak_free - unregister a previously registered object
1091	* @ptr: pointer to beginning of the object
1092	*
1093	* This function is called from the kernel allocators when an object (memory
1094	* block) is freed (kmem_cache_free, kfree, vfree etc.).
1095	*/
1096	void __ref kmemleak_free(const void *ptr)
1097	{
1098	pr_debug("%s(0x%px)\n", __func__, ptr);
1099
1100	if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1101	delete_object_full(ptr: (unsigned long)ptr);
1102	}
1103	EXPORT_SYMBOL_GPL(kmemleak_free);
1104
1105	/**
1106	* kmemleak_free_part - partially unregister a previously registered object
1107	* @ptr: pointer to the beginning or inside the object. This also
1108	* represents the start of the range to be freed
1109	* @size: size to be unregistered
1110	*
1111	* This function is called when only a part of a memory block is freed
1112	* (usually from the bootmem allocator).
1113	*/
1114	void __ref kmemleak_free_part(const void *ptr, size_t size)
1115	{
1116	pr_debug("%s(0x%px)\n", __func__, ptr);
1117
1118	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1119	delete_object_part(ptr: (unsigned long)ptr, size, is_phys: false);
1120	}
1121	EXPORT_SYMBOL_GPL(kmemleak_free_part);
1122
1123	/**
1124	* kmemleak_free_percpu - unregister a previously registered __percpu object
1125	* @ptr: __percpu pointer to beginning of the object
1126	*
1127	* This function is called from the kernel percpu allocator when an object
1128	* (memory block) is freed (free_percpu).
1129	*/
1130	void __ref kmemleak_free_percpu(const void __percpu *ptr)
1131	{
1132	unsigned int cpu;
1133
1134	pr_debug("%s(0x%px)\n", __func__, ptr);
1135
1136	if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1137	for_each_possible_cpu(cpu)
1138	delete_object_full(ptr: (unsigned long)per_cpu_ptr(ptr,
1139	cpu));
1140	}
1141	EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1142
1143	/**
1144	* kmemleak_update_trace - update object allocation stack trace
1145	* @ptr: pointer to beginning of the object
1146	*
1147	* Override the object allocation stack trace for cases where the actual
1148	* allocation place is not always useful.
1149	*/
1150	void __ref kmemleak_update_trace(const void *ptr)
1151	{
1152	struct kmemleak_object *object;
1153	unsigned long flags;
1154
1155	pr_debug("%s(0x%px)\n", __func__, ptr);
1156
1157	if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1158	return;
1159
1160	object = find_and_get_object(ptr: (unsigned long)ptr, alias: 1);
1161	if (!object) {
1162	#ifdef DEBUG
1163	kmemleak_warn("Updating stack trace for unknown object at %p\n",
1164	ptr);
1165	#endif
1166	return;
1167	}
1168
1169	raw_spin_lock_irqsave(&object->lock, flags);
1170	object->trace_handle = set_track_prepare();
1171	raw_spin_unlock_irqrestore(&object->lock, flags);
1172
1173	put_object(object);
1174	}
1175	EXPORT_SYMBOL(kmemleak_update_trace);
1176
1177	/**
1178	* kmemleak_not_leak - mark an allocated object as false positive
1179	* @ptr: pointer to beginning of the object
1180	*
1181	* Calling this function on an object will cause the memory block to no longer
1182	* be reported as leak and always be scanned.
1183	*/
1184	void __ref kmemleak_not_leak(const void *ptr)
1185	{
1186	pr_debug("%s(0x%px)\n", __func__, ptr);
1187
1188	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1189	make_gray_object(ptr: (unsigned long)ptr);
1190	}
1191	EXPORT_SYMBOL(kmemleak_not_leak);
1192
1193	/**
1194	* kmemleak_ignore - ignore an allocated object
1195	* @ptr: pointer to beginning of the object
1196	*
1197	* Calling this function on an object will cause the memory block to be
1198	* ignored (not scanned and not reported as a leak). This is usually done when
1199	* it is known that the corresponding block is not a leak and does not contain
1200	* any references to other allocated memory blocks.
1201	*/
1202	void __ref kmemleak_ignore(const void *ptr)
1203	{
1204	pr_debug("%s(0x%px)\n", __func__, ptr);
1205
1206	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1207	make_black_object(ptr: (unsigned long)ptr, is_phys: false);
1208	}
1209	EXPORT_SYMBOL(kmemleak_ignore);
1210
1211	/**
1212	* kmemleak_scan_area - limit the range to be scanned in an allocated object
1213	* @ptr: pointer to beginning or inside the object. This also
1214	* represents the start of the scan area
1215	* @size: size of the scan area
1216	* @gfp: kmalloc() flags used for kmemleak internal memory allocations
1217	*
1218	* This function is used when it is known that only certain parts of an object
1219	* contain references to other objects. Kmemleak will only scan these areas
1220	* reducing the number false negatives.
1221	*/
1222	void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1223	{
1224	pr_debug("%s(0x%px)\n", __func__, ptr);
1225
1226	if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1227	add_scan_area(ptr: (unsigned long)ptr, size, gfp);
1228	}
1229	EXPORT_SYMBOL(kmemleak_scan_area);
1230
1231	/**
1232	* kmemleak_no_scan - do not scan an allocated object
1233	* @ptr: pointer to beginning of the object
1234	*
1235	* This function notifies kmemleak not to scan the given memory block. Useful
1236	* in situations where it is known that the given object does not contain any
1237	* references to other objects. Kmemleak will not scan such objects reducing
1238	* the number of false negatives.
1239	*/
1240	void __ref kmemleak_no_scan(const void *ptr)
1241	{
1242	pr_debug("%s(0x%px)\n", __func__, ptr);
1243
1244	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1245	object_no_scan(ptr: (unsigned long)ptr);
1246	}
1247	EXPORT_SYMBOL(kmemleak_no_scan);
1248
1249	/**
1250	* kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1251	* address argument
1252	* @phys: physical address of the object
1253	* @size: size of the object
1254	* @gfp: kmalloc() flags used for kmemleak internal memory allocations
1255	*/
1256	void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, gfp_t gfp)
1257	{
1258	pr_debug("%s(0x%px, %zu)\n", __func__, &phys, size);
1259
1260	if (kmemleak_enabled)
1261	/*
1262	* Create object with OBJECT_PHYS flag and
1263	* assume min_count 0.
1264	*/
1265	create_object_phys(ptr: (unsigned long)phys, size, min_count: 0, gfp);
1266	}
1267	EXPORT_SYMBOL(kmemleak_alloc_phys);
1268
1269	/**
1270	* kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1271	* physical address argument
1272	* @phys: physical address if the beginning or inside an object. This
1273	* also represents the start of the range to be freed
1274	* @size: size to be unregistered
1275	*/
1276	void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1277	{
1278	pr_debug("%s(0x%px)\n", __func__, &phys);
1279
1280	if (kmemleak_enabled)
1281	delete_object_part(ptr: (unsigned long)phys, size, is_phys: true);
1282	}
1283	EXPORT_SYMBOL(kmemleak_free_part_phys);
1284
1285	/**
1286	* kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1287	* address argument
1288	* @phys: physical address of the object
1289	*/
1290	void __ref kmemleak_ignore_phys(phys_addr_t phys)
1291	{
1292	pr_debug("%s(0x%px)\n", __func__, &phys);
1293
1294	if (kmemleak_enabled)
1295	make_black_object(ptr: (unsigned long)phys, is_phys: true);
1296	}
1297	EXPORT_SYMBOL(kmemleak_ignore_phys);
1298
1299	/*
1300	* Update an object's checksum and return true if it was modified.
1301	*/
1302	static bool update_checksum(struct kmemleak_object *object)
1303	{
1304	u32 old_csum = object->checksum;
1305
1306	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
1307	return false;
1308
1309	kasan_disable_current();
1310	kcsan_disable_current();
1311	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);
1312	kasan_enable_current();
1313	kcsan_enable_current();
1314
1315	return object->checksum != old_csum;
1316	}
1317
1318	/*
1319	* Update an object's references. object->lock must be held by the caller.
1320	*/
1321	static void update_refs(struct kmemleak_object *object)
1322	{
1323	if (!color_white(object)) {
1324	/* non-orphan, ignored or new */
1325	return;
1326	}
1327
1328	/*
1329	* Increase the object's reference count (number of pointers to the
1330	* memory block). If this count reaches the required minimum, the
1331	* object's color will become gray and it will be added to the
1332	* gray_list.
1333	*/
1334	object->count++;
1335	if (color_gray(object)) {
1336	/* put_object() called when removing from gray_list */
1337	WARN_ON(!get_object(object));
1338	list_add_tail(new: &object->gray_list, head: &gray_list);
1339	}
1340	}
1341
1342	/*
1343	* Memory scanning is a long process and it needs to be interruptible. This
1344	* function checks whether such interrupt condition occurred.
1345	*/
1346	static int scan_should_stop(void)
1347	{
1348	if (!kmemleak_enabled)
1349	return 1;
1350
1351	/*
1352	* This function may be called from either process or kthread context,
1353	* hence the need to check for both stop conditions.
1354	*/
1355	if (current->mm)
1356	return signal_pending(current);
1357	else
1358	return kthread_should_stop();
1359
1360	return 0;
1361	}
1362
1363	/*
1364	* Scan a memory block (exclusive range) for valid pointers and add those
1365	* found to the gray list.
1366	*/
1367	static void scan_block(void *_start, void *_end,
1368	struct kmemleak_object *scanned)
1369	{
1370	unsigned long *ptr;
1371	unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1372	unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1373	unsigned long flags;
1374	unsigned long untagged_ptr;
1375
1376	raw_spin_lock_irqsave(&kmemleak_lock, flags);
1377	for (ptr = start; ptr < end; ptr++) {
1378	struct kmemleak_object *object;
1379	unsigned long pointer;
1380	unsigned long excess_ref;
1381
1382	if (scan_should_stop())
1383	break;
1384
1385	kasan_disable_current();
1386	pointer = *(unsigned long *)kasan_reset_tag(addr: (void *)ptr);
1387	kasan_enable_current();
1388
1389	untagged_ptr = (unsigned long)kasan_reset_tag(addr: (void *)pointer);
1390	if (untagged_ptr < min_addr || untagged_ptr >= max_addr)
1391	continue;
1392
1393	/*
1394	* No need for get_object() here since we hold kmemleak_lock.
1395	* object->use_count cannot be dropped to 0 while the object
1396	* is still present in object_tree_root and object_list
1397	* (with updates protected by kmemleak_lock).
1398	*/
1399	object = lookup_object(ptr: pointer, alias: 1);
1400	if (!object)
1401	continue;
1402	if (object == scanned)
1403	/* self referenced, ignore */
1404	continue;
1405
1406	/*
1407	* Avoid the lockdep recursive warning on object->lock being
1408	* previously acquired in scan_object(). These locks are
1409	* enclosed by scan_mutex.
1410	*/
1411	raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1412	/* only pass surplus references (object already gray) */
1413	if (color_gray(object)) {
1414	excess_ref = object->excess_ref;
1415	/* no need for update_refs() if object already gray */
1416	} else {
1417	excess_ref = 0;
1418	update_refs(object);
1419	}
1420	raw_spin_unlock(&object->lock);
1421
1422	if (excess_ref) {
1423	object = lookup_object(ptr: excess_ref, alias: 0);
1424	if (!object)
1425	continue;
1426	if (object == scanned)
1427	/* circular reference, ignore */
1428	continue;
1429	raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1430	update_refs(object);
1431	raw_spin_unlock(&object->lock);
1432	}
1433	}
1434	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
1435	}
1436
1437	/*
1438	* Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1439	*/
1440	#ifdef CONFIG_SMP
1441	static void scan_large_block(void *start, void *end)
1442	{
1443	void *next;
1444
1445	while (start < end) {
1446	next = min(start + MAX_SCAN_SIZE, end);
1447	scan_block(start: start, end: next, NULL);
1448	start = next;
1449	cond_resched();
1450	}
1451	}
1452	#endif
1453
1454	/*
1455	* Scan a memory block corresponding to a kmemleak_object. A condition is
1456	* that object->use_count >= 1.
1457	*/
1458	static void scan_object(struct kmemleak_object *object)
1459	{
1460	struct kmemleak_scan_area *area;
1461	unsigned long flags;
1462	void *obj_ptr;
1463
1464	/*
1465	* Once the object->lock is acquired, the corresponding memory block
1466	* cannot be freed (the same lock is acquired in delete_object).
1467	*/
1468	raw_spin_lock_irqsave(&object->lock, flags);
1469	if (object->flags & OBJECT_NO_SCAN)
1470	goto out;
1471	if (!(object->flags & OBJECT_ALLOCATED))
1472	/* already freed object */
1473	goto out;
1474
1475	obj_ptr = object->flags & OBJECT_PHYS ?
1476	__va((phys_addr_t)object->pointer) :
1477	(void *)object->pointer;
1478
1479	if (hlist_empty(h: &object->area_list) ||
1480	object->flags & OBJECT_FULL_SCAN) {
1481	void *start = obj_ptr;
1482	void *end = obj_ptr + object->size;
1483	void *next;
1484
1485	do {
1486	next = min(start + MAX_SCAN_SIZE, end);
1487	scan_block(start: start, end: next, scanned: object);
1488
1489	start = next;
1490	if (start >= end)
1491	break;
1492
1493	raw_spin_unlock_irqrestore(&object->lock, flags);
1494	cond_resched();
1495	raw_spin_lock_irqsave(&object->lock, flags);
1496	} while (object->flags & OBJECT_ALLOCATED);
1497	} else
1498	hlist_for_each_entry(area, &object->area_list, node)
1499	scan_block(start: (void *)area->start,
1500	end: (void *)(area->start + area->size),
1501	scanned: object);
1502	out:
1503	raw_spin_unlock_irqrestore(&object->lock, flags);
1504	}
1505
1506	/*
1507	* Scan the objects already referenced (gray objects). More objects will be
1508	* referenced and, if there are no memory leaks, all the objects are scanned.
1509	*/
1510	static void scan_gray_list(void)
1511	{
1512	struct kmemleak_object *object, *tmp;
1513
1514	/*
1515	* The list traversal is safe for both tail additions and removals
1516	* from inside the loop. The kmemleak objects cannot be freed from
1517	* outside the loop because their use_count was incremented.
1518	*/
1519	object = list_entry(gray_list.next, typeof(*object), gray_list);
1520	while (&object->gray_list != &gray_list) {
1521	cond_resched();
1522
1523	/* may add new objects to the list */
1524	if (!scan_should_stop())
1525	scan_object(object);
1526
1527	tmp = list_entry(object->gray_list.next, typeof(*object),
1528	gray_list);
1529
1530	/* remove the object from the list and release it */
1531	list_del(entry: &object->gray_list);
1532	put_object(object);
1533
1534	object = tmp;
1535	}
1536	WARN_ON(!list_empty(&gray_list));
1537	}
1538
1539	/*
1540	* Conditionally call resched() in an object iteration loop while making sure
1541	* that the given object won't go away without RCU read lock by performing a
1542	* get_object() if necessaary.
1543	*/
1544	static void kmemleak_cond_resched(struct kmemleak_object *object)
1545	{
1546	if (!get_object(object))
1547	return; /* Try next object */
1548
1549	raw_spin_lock_irq(&kmemleak_lock);
1550	if (object->del_state & DELSTATE_REMOVED)
1551	goto unlock_put; /* Object removed */
1552	object->del_state |= DELSTATE_NO_DELETE;
1553	raw_spin_unlock_irq(&kmemleak_lock);
1554
1555	rcu_read_unlock();
1556	cond_resched();
1557	rcu_read_lock();
1558
1559	raw_spin_lock_irq(&kmemleak_lock);
1560	if (object->del_state & DELSTATE_REMOVED)
1561	list_del_rcu(entry: &object->object_list);
1562	object->del_state &= ~DELSTATE_NO_DELETE;
1563	unlock_put:
1564	raw_spin_unlock_irq(&kmemleak_lock);
1565	put_object(object);
1566	}
1567
1568	/*
1569	* Scan data sections and all the referenced memory blocks allocated via the
1570	* kernel's standard allocators. This function must be called with the
1571	* scan_mutex held.
1572	*/
1573	static void kmemleak_scan(void)
1574	{
1575	struct kmemleak_object *object;
1576	struct zone *zone;
1577	int __maybe_unused i;
1578	int new_leaks = 0;
1579
1580	jiffies_last_scan = jiffies;
1581
1582	/* prepare the kmemleak_object's */
1583	rcu_read_lock();
1584	list_for_each_entry_rcu(object, &object_list, object_list) {
1585	raw_spin_lock_irq(&object->lock);
1586	#ifdef DEBUG
1587	/*
1588	* With a few exceptions there should be a maximum of
1589	* 1 reference to any object at this point.
1590	*/
1591	if (atomic_read(&object->use_count) > 1) {
1592	pr_debug("object->use_count = %d\n",
1593	atomic_read(&object->use_count));
1594	dump_object_info(object);
1595	}
1596	#endif
1597
1598	/* ignore objects outside lowmem (paint them black) */
1599	if ((object->flags & OBJECT_PHYS) &&
1600	!(object->flags & OBJECT_NO_SCAN)) {
1601	unsigned long phys = object->pointer;
1602
1603	if (PHYS_PFN(phys) < min_low_pfn ||
1604	PHYS_PFN(phys + object->size) >= max_low_pfn)
1605	__paint_it(object, KMEMLEAK_BLACK);
1606	}
1607
1608	/* reset the reference count (whiten the object) */
1609	object->count = 0;
1610	if (color_gray(object) && get_object(object))
1611	list_add_tail(new: &object->gray_list, head: &gray_list);
1612
1613	raw_spin_unlock_irq(&object->lock);
1614
1615	if (need_resched())
1616	kmemleak_cond_resched(object);
1617	}
1618	rcu_read_unlock();
1619
1620	#ifdef CONFIG_SMP
1621	/* per-cpu sections scanning */
1622	for_each_possible_cpu(i)
1623	scan_large_block(start: __per_cpu_start + per_cpu_offset(i),
1624	end: __per_cpu_end + per_cpu_offset(i));
1625	#endif
1626
1627	/*
1628	* Struct page scanning for each node.
1629	*/
1630	get_online_mems();
1631	for_each_populated_zone(zone) {
1632	unsigned long start_pfn = zone->zone_start_pfn;
1633	unsigned long end_pfn = zone_end_pfn(zone);
1634	unsigned long pfn;
1635
1636	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1637	struct page *page = pfn_to_online_page(pfn);
1638
1639	if (!(pfn & 63))
1640	cond_resched();
1641
1642	if (!page)
1643	continue;
1644
1645	/* only scan pages belonging to this zone */
1646	if (page_zone(page) != zone)
1647	continue;
1648	/* only scan if page is in use */
1649	if (page_count(page) == 0)
1650	continue;
1651	scan_block(start: page, end: page + 1, NULL);
1652	}
1653	}
1654	put_online_mems();
1655
1656	/*
1657	* Scanning the task stacks (may introduce false negatives).
1658	*/
1659	if (kmemleak_stack_scan) {
1660	struct task_struct *p, *g;
1661
1662	rcu_read_lock();
1663	for_each_process_thread(g, p) {
1664	void *stack = try_get_task_stack(tsk: p);
1665	if (stack) {
1666	scan_block(start: stack, end: stack + THREAD_SIZE, NULL);
1667	put_task_stack(tsk: p);
1668	}
1669	}
1670	rcu_read_unlock();
1671	}
1672
1673	/*
1674	* Scan the objects already referenced from the sections scanned
1675	* above.
1676	*/
1677	scan_gray_list();
1678
1679	/*
1680	* Check for new or unreferenced objects modified since the previous
1681	* scan and color them gray until the next scan.
1682	*/
1683	rcu_read_lock();
1684	list_for_each_entry_rcu(object, &object_list, object_list) {
1685	if (need_resched())
1686	kmemleak_cond_resched(object);
1687
1688	/*
1689	* This is racy but we can save the overhead of lock/unlock
1690	* calls. The missed objects, if any, should be caught in
1691	* the next scan.
1692	*/
1693	if (!color_white(object))
1694	continue;
1695	raw_spin_lock_irq(&object->lock);
1696	if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1697	&& update_checksum(object) && get_object(object)) {
1698	/* color it gray temporarily */
1699	object->count = object->min_count;
1700	list_add_tail(new: &object->gray_list, head: &gray_list);
1701	}
1702	raw_spin_unlock_irq(&object->lock);
1703	}
1704	rcu_read_unlock();
1705
1706	/*
1707	* Re-scan the gray list for modified unreferenced objects.
1708	*/
1709	scan_gray_list();
1710
1711	/*
1712	* If scanning was stopped do not report any new unreferenced objects.
1713	*/
1714	if (scan_should_stop())
1715	return;
1716
1717	/*
1718	* Scanning result reporting.
1719	*/
1720	rcu_read_lock();
1721	list_for_each_entry_rcu(object, &object_list, object_list) {
1722	if (need_resched())
1723	kmemleak_cond_resched(object);
1724
1725	/*
1726	* This is racy but we can save the overhead of lock/unlock
1727	* calls. The missed objects, if any, should be caught in
1728	* the next scan.
1729	*/
1730	if (!color_white(object))
1731	continue;
1732	raw_spin_lock_irq(&object->lock);
1733	if (unreferenced_object(object) &&
1734	!(object->flags & OBJECT_REPORTED)) {
1735	object->flags |= OBJECT_REPORTED;
1736
1737	if (kmemleak_verbose)
1738	print_unreferenced(NULL, object);
1739
1740	new_leaks++;
1741	}
1742	raw_spin_unlock_irq(&object->lock);
1743	}
1744	rcu_read_unlock();
1745
1746	if (new_leaks) {
1747	kmemleak_found_leaks = true;
1748
1749	pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1750	new_leaks);
1751	}
1752
1753	}
1754
1755	/*
1756	* Thread function performing automatic memory scanning. Unreferenced objects
1757	* at the end of a memory scan are reported but only the first time.
1758	*/
1759	static int kmemleak_scan_thread(void *arg)
1760	{
1761	static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN);
1762
1763	pr_info("Automatic memory scanning thread started\n");
1764	set_user_nice(current, nice: 10);
1765
1766	/*
1767	* Wait before the first scan to allow the system to fully initialize.
1768	*/
1769	if (first_run) {
1770	signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
1771	first_run = 0;
1772	while (timeout && !kthread_should_stop())
1773	timeout = schedule_timeout_interruptible(timeout);
1774	}
1775
1776	while (!kthread_should_stop()) {
1777	signed long timeout = READ_ONCE(jiffies_scan_wait);
1778
1779	mutex_lock(&scan_mutex);
1780	kmemleak_scan();
1781	mutex_unlock(lock: &scan_mutex);
1782
1783	/* wait before the next scan */
1784	while (timeout && !kthread_should_stop())
1785	timeout = schedule_timeout_interruptible(timeout);
1786	}
1787
1788	pr_info("Automatic memory scanning thread ended\n");
1789
1790	return 0;
1791	}
1792
1793	/*
1794	* Start the automatic memory scanning thread. This function must be called
1795	* with the scan_mutex held.
1796	*/
1797	static void start_scan_thread(void)
1798	{
1799	if (scan_thread)
1800	return;
1801	scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1802	if (IS_ERR(ptr: scan_thread)) {
1803	pr_warn("Failed to create the scan thread\n");
1804	scan_thread = NULL;
1805	}
1806	}
1807
1808	/*
1809	* Stop the automatic memory scanning thread.
1810	*/
1811	static void stop_scan_thread(void)
1812	{
1813	if (scan_thread) {
1814	kthread_stop(k: scan_thread);
1815	scan_thread = NULL;
1816	}
1817	}
1818
1819	/*
1820	* Iterate over the object_list and return the first valid object at or after
1821	* the required position with its use_count incremented. The function triggers
1822	* a memory scanning when the pos argument points to the first position.
1823	*/
1824	static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1825	{
1826	struct kmemleak_object *object;
1827	loff_t n = *pos;
1828	int err;
1829
1830	err = mutex_lock_interruptible(&scan_mutex);
1831	if (err < 0)
1832	return ERR_PTR(error: err);
1833
1834	rcu_read_lock();
1835	list_for_each_entry_rcu(object, &object_list, object_list) {
1836	if (n-- > 0)
1837	continue;
1838	if (get_object(object))
1839	goto out;
1840	}
1841	object = NULL;
1842	out:
1843	return object;
1844	}
1845
1846	/*
1847	* Return the next object in the object_list. The function decrements the
1848	* use_count of the previous object and increases that of the next one.
1849	*/
1850	static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1851	{
1852	struct kmemleak_object *prev_obj = v;
1853	struct kmemleak_object *next_obj = NULL;
1854	struct kmemleak_object *obj = prev_obj;
1855
1856	++(*pos);
1857
1858	list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1859	if (get_object(object: obj)) {
1860	next_obj = obj;
1861	break;
1862	}
1863	}
1864
1865	put_object(object: prev_obj);
1866	return next_obj;
1867	}
1868
1869	/*
1870	* Decrement the use_count of the last object required, if any.
1871	*/
1872	static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1873	{
1874	if (!IS_ERR(ptr: v)) {
1875	/*
1876	* kmemleak_seq_start may return ERR_PTR if the scan_mutex
1877	* waiting was interrupted, so only release it if !IS_ERR.
1878	*/
1879	rcu_read_unlock();
1880	mutex_unlock(lock: &scan_mutex);
1881	if (v)
1882	put_object(object: v);
1883	}
1884	}
1885
1886	/*
1887	* Print the information for an unreferenced object to the seq file.
1888	*/
1889	static int kmemleak_seq_show(struct seq_file *seq, void *v)
1890	{
1891	struct kmemleak_object *object = v;
1892	unsigned long flags;
1893
1894	raw_spin_lock_irqsave(&object->lock, flags);
1895	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1896	print_unreferenced(seq, object);
1897	raw_spin_unlock_irqrestore(&object->lock, flags);
1898	return 0;
1899	}
1900
1901	static const struct seq_operations kmemleak_seq_ops = {
1902	.start = kmemleak_seq_start,
1903	.next = kmemleak_seq_next,
1904	.stop = kmemleak_seq_stop,
1905	.show = kmemleak_seq_show,
1906	};
1907
1908	static int kmemleak_open(struct inode *inode, struct file *file)
1909	{
1910	return seq_open(file, &kmemleak_seq_ops);
1911	}
1912
1913	static int dump_str_object_info(const char *str)
1914	{
1915	unsigned long flags;
1916	struct kmemleak_object *object;
1917	unsigned long addr;
1918
1919	if (kstrtoul(s: str, base: 0, res: &addr))
1920	return -EINVAL;
1921	object = find_and_get_object(ptr: addr, alias: 0);
1922	if (!object) {
1923	pr_info("Unknown object at 0x%08lx\n", addr);
1924	return -EINVAL;
1925	}
1926
1927	raw_spin_lock_irqsave(&object->lock, flags);
1928	dump_object_info(object);
1929	raw_spin_unlock_irqrestore(&object->lock, flags);
1930
1931	put_object(object);
1932	return 0;
1933	}
1934
1935	/*
1936	* We use grey instead of black to ensure we can do future scans on the same
1937	* objects. If we did not do future scans these black objects could
1938	* potentially contain references to newly allocated objects in the future and
1939	* we'd end up with false positives.
1940	*/
1941	static void kmemleak_clear(void)
1942	{
1943	struct kmemleak_object *object;
1944
1945	rcu_read_lock();
1946	list_for_each_entry_rcu(object, &object_list, object_list) {
1947	raw_spin_lock_irq(&object->lock);
1948	if ((object->flags & OBJECT_REPORTED) &&
1949	unreferenced_object(object))
1950	__paint_it(object, KMEMLEAK_GREY);
1951	raw_spin_unlock_irq(&object->lock);
1952	}
1953	rcu_read_unlock();
1954
1955	kmemleak_found_leaks = false;
1956	}
1957
1958	static void __kmemleak_do_cleanup(void);
1959
1960	/*
1961	* File write operation to configure kmemleak at run-time. The following
1962	* commands can be written to the /sys/kernel/debug/kmemleak file:
1963	* off - disable kmemleak (irreversible)
1964	* stack=on - enable the task stacks scanning
1965	* stack=off - disable the tasks stacks scanning
1966	* scan=on - start the automatic memory scanning thread
1967	* scan=off - stop the automatic memory scanning thread
1968	* scan=... - set the automatic memory scanning period in seconds (0 to
1969	* disable it)
1970	* scan - trigger a memory scan
1971	* clear - mark all current reported unreferenced kmemleak objects as
1972	* grey to ignore printing them, or free all kmemleak objects
1973	* if kmemleak has been disabled.
1974	* dump=... - dump information about the object found at the given address
1975	*/
1976	static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1977	size_t size, loff_t *ppos)
1978	{
1979	char buf[64];
1980	int buf_size;
1981	int ret;
1982
1983	buf_size = min(size, (sizeof(buf) - 1));
1984	if (strncpy_from_user(dst: buf, src: user_buf, count: buf_size) < 0)
1985	return -EFAULT;
1986	buf[buf_size] = 0;
1987
1988	ret = mutex_lock_interruptible(&scan_mutex);
1989	if (ret < 0)
1990	return ret;
1991
1992	if (strncmp(buf, "clear", 5) == 0) {
1993	if (kmemleak_enabled)
1994	kmemleak_clear();
1995	else
1996	__kmemleak_do_cleanup();
1997	goto out;
1998	}
1999
2000	if (!kmemleak_enabled) {
2001	ret = -EPERM;
2002	goto out;
2003	}
2004
2005	if (strncmp(buf, "off", 3) == 0)
2006	kmemleak_disable();
2007	else if (strncmp(buf, "stack=on", 8) == 0)
2008	kmemleak_stack_scan = 1;
2009	else if (strncmp(buf, "stack=off", 9) == 0)
2010	kmemleak_stack_scan = 0;
2011	else if (strncmp(buf, "scan=on", 7) == 0)
2012	start_scan_thread();
2013	else if (strncmp(buf, "scan=off", 8) == 0)
2014	stop_scan_thread();
2015	else if (strncmp(buf, "scan=", 5) == 0) {
2016	unsigned secs;
2017	unsigned long msecs;
2018
2019	ret = kstrtouint(s: buf + 5, base: 0, res: &secs);
2020	if (ret < 0)
2021	goto out;
2022
2023	msecs = secs * MSEC_PER_SEC;
2024	if (msecs > UINT_MAX)
2025	msecs = UINT_MAX;
2026
2027	stop_scan_thread();
2028	if (msecs) {
2029	WRITE_ONCE(jiffies_scan_wait, msecs_to_jiffies(msecs));
2030	start_scan_thread();
2031	}
2032	} else if (strncmp(buf, "scan", 4) == 0)
2033	kmemleak_scan();
2034	else if (strncmp(buf, "dump=", 5) == 0)
2035	ret = dump_str_object_info(str: buf + 5);
2036	else
2037	ret = -EINVAL;
2038
2039	out:
2040	mutex_unlock(lock: &scan_mutex);
2041	if (ret < 0)
2042	return ret;
2043
2044	/* ignore the rest of the buffer, only one command at a time */
2045	*ppos += size;
2046	return size;
2047	}
2048
2049	static const struct file_operations kmemleak_fops = {
2050	.owner = THIS_MODULE,
2051	.open = kmemleak_open,
2052	.read = seq_read,
2053	.write = kmemleak_write,
2054	.llseek = seq_lseek,
2055	.release = seq_release,
2056	};
2057
2058	static void __kmemleak_do_cleanup(void)
2059	{
2060	struct kmemleak_object *object, *tmp;
2061
2062	/*
2063	* Kmemleak has already been disabled, no need for RCU list traversal
2064	* or kmemleak_lock held.
2065	*/
2066	list_for_each_entry_safe(object, tmp, &object_list, object_list) {
2067	__remove_object(object);
2068	__delete_object(object);
2069	}
2070	}
2071
2072	/*
2073	* Stop the memory scanning thread and free the kmemleak internal objects if
2074	* no previous scan thread (otherwise, kmemleak may still have some useful
2075	* information on memory leaks).
2076	*/
2077	static void kmemleak_do_cleanup(struct work_struct *work)
2078	{
2079	stop_scan_thread();
2080
2081	mutex_lock(&scan_mutex);
2082	/*
2083	* Once it is made sure that kmemleak_scan has stopped, it is safe to no
2084	* longer track object freeing. Ordering of the scan thread stopping and
2085	* the memory accesses below is guaranteed by the kthread_stop()
2086	* function.
2087	*/
2088	kmemleak_free_enabled = 0;
2089	mutex_unlock(lock: &scan_mutex);
2090
2091	if (!kmemleak_found_leaks)
2092	__kmemleak_do_cleanup();
2093	else
2094	pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
2095	}
2096
2097	static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
2098
2099	/*
2100	* Disable kmemleak. No memory allocation/freeing will be traced once this
2101	* function is called. Disabling kmemleak is an irreversible operation.
2102	*/
2103	static void kmemleak_disable(void)
2104	{
2105	/* atomically check whether it was already invoked */
2106	if (cmpxchg(&kmemleak_error, 0, 1))
2107	return;
2108
2109	/* stop any memory operation tracing */
2110	kmemleak_enabled = 0;
2111
2112	/* check whether it is too early for a kernel thread */
2113	if (kmemleak_late_initialized)
2114	schedule_work(work: &cleanup_work);
2115	else
2116	kmemleak_free_enabled = 0;
2117
2118	pr_info("Kernel memory leak detector disabled\n");
2119	}
2120
2121	/*
2122	* Allow boot-time kmemleak disabling (enabled by default).
2123	*/
2124	static int __init kmemleak_boot_config(char *str)
2125	{
2126	if (!str)
2127	return -EINVAL;
2128	if (strcmp(str, "off") == 0)
2129	kmemleak_disable();
2130	else if (strcmp(str, "on") == 0) {
2131	kmemleak_skip_disable = 1;
2132	stack_depot_request_early_init();
2133	}
2134	else
2135	return -EINVAL;
2136	return 0;
2137	}
2138	early_param("kmemleak", kmemleak_boot_config);
2139
2140	/*
2141	* Kmemleak initialization.
2142	*/
2143	void __init kmemleak_init(void)
2144	{
2145	#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
2146	if (!kmemleak_skip_disable) {
2147	kmemleak_disable();
2148	return;
2149	}
2150	#endif
2151
2152	if (kmemleak_error)
2153	return;
2154
2155	jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
2156	jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
2157
2158	object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
2159	scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
2160
2161	/* register the data/bss sections */
2162	create_object(ptr: (unsigned long)_sdata, size: _edata - _sdata,
2163	KMEMLEAK_GREY, GFP_ATOMIC);
2164	create_object(ptr: (unsigned long)__bss_start, size: __bss_stop - __bss_start,
2165	KMEMLEAK_GREY, GFP_ATOMIC);
2166	/* only register .data..ro_after_init if not within .data */
2167	if (&__start_ro_after_init < &_sdata || &__end_ro_after_init > &_edata)
2168	create_object(ptr: (unsigned long)__start_ro_after_init,
2169	size: __end_ro_after_init - __start_ro_after_init,
2170	KMEMLEAK_GREY, GFP_ATOMIC);
2171	}
2172
2173	/*
2174	* Late initialization function.
2175	*/
2176	static int __init kmemleak_late_init(void)
2177	{
2178	kmemleak_late_initialized = 1;
2179
2180	debugfs_create_file(name: "kmemleak", mode: 0644, NULL, NULL, fops: &kmemleak_fops);
2181
2182	if (kmemleak_error) {
2183	/*
2184	* Some error occurred and kmemleak was disabled. There is a
2185	* small chance that kmemleak_disable() was called immediately
2186	* after setting kmemleak_late_initialized and we may end up with
2187	* two clean-up threads but serialized by scan_mutex.
2188	*/
2189	schedule_work(work: &cleanup_work);
2190	return -ENOMEM;
2191	}
2192
2193	if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) {
2194	mutex_lock(&scan_mutex);
2195	start_scan_thread();
2196	mutex_unlock(lock: &scan_mutex);
2197	}
2198
2199	pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",
2200	mem_pool_free_count);
2201
2202	return 0;
2203	}
2204	late_initcall(kmemleak_late_init);
2205