1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* kernel/lockdep.c
4	*
5	* Runtime locking correctness validator
6	*
7	* Started by Ingo Molnar:
8	*
9	* Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
10	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
11	*
12	* this code maps all the lock dependencies as they occur in a live kernel
13	* and will warn about the following classes of locking bugs:
14	*
15	* - lock inversion scenarios
16	* - circular lock dependencies
17	* - hardirq/softirq safe/unsafe locking bugs
18	*
19	* Bugs are reported even if the current locking scenario does not cause
20	* any deadlock at this point.
21	*
22	* I.e. if anytime in the past two locks were taken in a different order,
23	* even if it happened for another task, even if those were different
24	* locks (but of the same class as this lock), this code will detect it.
25	*
26	* Thanks to Arjan van de Ven for coming up with the initial idea of
27	* mapping lock dependencies runtime.
28	*/
29	#define DISABLE_BRANCH_PROFILING
30	#include <linux/mutex.h>
31	#include <linux/sched.h>
32	#include <linux/sched/clock.h>
33	#include <linux/sched/task.h>
34	#include <linux/sched/mm.h>
35	#include <linux/delay.h>
36	#include <linux/module.h>
37	#include <linux/proc_fs.h>
38	#include <linux/seq_file.h>
39	#include <linux/spinlock.h>
40	#include <linux/kallsyms.h>
41	#include <linux/interrupt.h>
42	#include <linux/stacktrace.h>
43	#include <linux/debug_locks.h>
44	#include <linux/irqflags.h>
45	#include <linux/utsname.h>
46	#include <linux/hash.h>
47	#include <linux/ftrace.h>
48	#include <linux/stringify.h>
49	#include <linux/bitmap.h>
50	#include <linux/bitops.h>
51	#include <linux/gfp.h>
52	#include <linux/random.h>
53	#include <linux/jhash.h>
54	#include <linux/nmi.h>
55	#include <linux/rcupdate.h>
56	#include <linux/kprobes.h>
57	#include <linux/lockdep.h>
58	#include <linux/context_tracking.h>
59
60	#include <asm/sections.h>
61
62	#include "lockdep_internals.h"
63
64	#include <trace/events/lock.h>
65
66	#ifdef CONFIG_PROVE_LOCKING
67	static int prove_locking = 1;
68	module_param(prove_locking, int, 0644);
69	#else
70	#define prove_locking 0
71	#endif
72
73	#ifdef CONFIG_LOCK_STAT
74	static int lock_stat = 1;
75	module_param(lock_stat, int, 0644);
76	#else
77	#define lock_stat 0
78	#endif
79
80	#ifdef CONFIG_SYSCTL
81	static struct ctl_table kern_lockdep_table[] = {
82	#ifdef CONFIG_PROVE_LOCKING
83	{
84	.procname = "prove_locking",
85	.data = &prove_locking,
86	.maxlen = sizeof(int),
87	.mode = 0644,
88	.proc_handler = proc_dointvec,
89	},
90	#endif /* CONFIG_PROVE_LOCKING */
91	#ifdef CONFIG_LOCK_STAT
92	{
93	.procname = "lock_stat",
94	.data = &lock_stat,
95	.maxlen = sizeof(int),
96	.mode = 0644,
97	.proc_handler = proc_dointvec,
98	},
99	#endif /* CONFIG_LOCK_STAT */
100	{ }
101	};
102
103	static __init int kernel_lockdep_sysctls_init(void)
104	{
105	register_sysctl_init("kernel", kern_lockdep_table);
106	return 0;
107	}
108	late_initcall(kernel_lockdep_sysctls_init);
109	#endif /* CONFIG_SYSCTL */
110
111	DEFINE_PER_CPU(unsigned int, lockdep_recursion);
112	EXPORT_PER_CPU_SYMBOL_GPL(lockdep_recursion);
113
114	static __always_inline bool lockdep_enabled(void)
115	{
116	if (!debug_locks)
117	return false;
118
119	if (this_cpu_read(lockdep_recursion))
120	return false;
121
122	if (current->lockdep_recursion)
123	return false;
124
125	return true;
126	}
127
128	/*
129	* lockdep_lock: protects the lockdep graph, the hashes and the
130	* class/list/hash allocators.
131	*
132	* This is one of the rare exceptions where it's justified
133	* to use a raw spinlock - we really dont want the spinlock
134	* code to recurse back into the lockdep code...
135	*/
136	static arch_spinlock_t __lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
137	static struct task_struct *__owner;
138
139	static inline void lockdep_lock(void)
140	{
141	DEBUG_LOCKS_WARN_ON(!irqs_disabled());
142
143	__this_cpu_inc(lockdep_recursion);
144	arch_spin_lock(&__lock);
145	__owner = current;
146	}
147
148	static inline void lockdep_unlock(void)
149	{
150	DEBUG_LOCKS_WARN_ON(!irqs_disabled());
151
152	if (debug_locks && DEBUG_LOCKS_WARN_ON(__owner != current))
153	return;
154
155	__owner = NULL;
156	arch_spin_unlock(&__lock);
157	__this_cpu_dec(lockdep_recursion);
158	}
159
160	static inline bool lockdep_assert_locked(void)
161	{
162	return DEBUG_LOCKS_WARN_ON(__owner != current);
163	}
164
165	static struct task_struct *lockdep_selftest_task_struct;
166
167
168	static int graph_lock(void)
169	{
170	lockdep_lock();
171	/*
172	* Make sure that if another CPU detected a bug while
173	* walking the graph we dont change it (while the other
174	* CPU is busy printing out stuff with the graph lock
175	* dropped already)
176	*/
177	if (!debug_locks) {
178	lockdep_unlock();
179	return 0;
180	}
181	return 1;
182	}
183
184	static inline void graph_unlock(void)
185	{
186	lockdep_unlock();
187	}
188
189	/*
190	* Turn lock debugging off and return with 0 if it was off already,
191	* and also release the graph lock:
192	*/
193	static inline int debug_locks_off_graph_unlock(void)
194	{
195	int ret = debug_locks_off();
196
197	lockdep_unlock();
198
199	return ret;
200	}
201
202	unsigned long nr_list_entries;
203	static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];
204	static DECLARE_BITMAP(list_entries_in_use, MAX_LOCKDEP_ENTRIES);
205
206	/*
207	* All data structures here are protected by the global debug_lock.
208	*
209	* nr_lock_classes is the number of elements of lock_classes[] that is
210	* in use.
211	*/
212	#define KEYHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1)
213	#define KEYHASH_SIZE (1UL << KEYHASH_BITS)
214	static struct hlist_head lock_keys_hash[KEYHASH_SIZE];
215	unsigned long nr_lock_classes;
216	unsigned long nr_zapped_classes;
217	unsigned long max_lock_class_idx;
218	struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
219	DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS);
220
221	static inline struct lock_class *hlock_class(struct held_lock *hlock)
222	{
223	unsigned int class_idx = hlock->class_idx;
224
225	/* Don't re-read hlock->class_idx, can't use READ_ONCE() on bitfield */
226	barrier();
227
228	if (!test_bit(class_idx, lock_classes_in_use)) {
229	/*
230	* Someone passed in garbage, we give up.
231	*/
232	DEBUG_LOCKS_WARN_ON(1);
233	return NULL;
234	}
235
236	/*
237	* At this point, if the passed hlock->class_idx is still garbage,
238	* we just have to live with it
239	*/
240	return lock_classes + class_idx;
241	}
242
243	#ifdef CONFIG_LOCK_STAT
244	static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats);
245
246	static inline u64 lockstat_clock(void)
247	{
248	return local_clock();
249	}
250
251	static int lock_point(unsigned long points[], unsigned long ip)
252	{
253	int i;
254
255	for (i = 0; i < LOCKSTAT_POINTS; i++) {
256	if (points[i] == 0) {
257	points[i] = ip;
258	break;
259	}
260	if (points[i] == ip)
261	break;
262	}
263
264	return i;
265	}
266
267	static void lock_time_inc(struct lock_time *lt, u64 time)
268	{
269	if (time > lt->max)
270	lt->max = time;
271
272	if (time < lt->min || !lt->nr)
273	lt->min = time;
274
275	lt->total += time;
276	lt->nr++;
277	}
278
279	static inline void lock_time_add(struct lock_time *src, struct lock_time *dst)
280	{
281	if (!src->nr)
282	return;
283
284	if (src->max > dst->max)
285	dst->max = src->max;
286
287	if (src->min < dst->min || !dst->nr)
288	dst->min = src->min;
289
290	dst->total += src->total;
291	dst->nr += src->nr;
292	}
293
294	struct lock_class_stats lock_stats(struct lock_class *class)
295	{
296	struct lock_class_stats stats;
297	int cpu, i;
298
299	memset(&stats, 0, sizeof(struct lock_class_stats));
300	for_each_possible_cpu(cpu) {
301	struct lock_class_stats *pcs =
302	&per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
303
304	for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)
305	stats.contention_point[i] += pcs->contention_point[i];
306
307	for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)
308	stats.contending_point[i] += pcs->contending_point[i];
309
310	lock_time_add(src: &pcs->read_waittime, dst: &stats.read_waittime);
311	lock_time_add(src: &pcs->write_waittime, dst: &stats.write_waittime);
312
313	lock_time_add(src: &pcs->read_holdtime, dst: &stats.read_holdtime);
314	lock_time_add(src: &pcs->write_holdtime, dst: &stats.write_holdtime);
315
316	for (i = 0; i < ARRAY_SIZE(stats.bounces); i++)
317	stats.bounces[i] += pcs->bounces[i];
318	}
319
320	return stats;
321	}
322
323	void clear_lock_stats(struct lock_class *class)
324	{
325	int cpu;
326
327	for_each_possible_cpu(cpu) {
328	struct lock_class_stats *cpu_stats =
329	&per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
330
331	memset(cpu_stats, 0, sizeof(struct lock_class_stats));
332	}
333	memset(class->contention_point, 0, sizeof(class->contention_point));
334	memset(class->contending_point, 0, sizeof(class->contending_point));
335	}
336
337	static struct lock_class_stats *get_lock_stats(struct lock_class *class)
338	{
339	return &this_cpu_ptr(cpu_lock_stats)[class - lock_classes];
340	}
341
342	static void lock_release_holdtime(struct held_lock *hlock)
343	{
344	struct lock_class_stats *stats;
345	u64 holdtime;
346
347	if (!lock_stat)
348	return;
349
350	holdtime = lockstat_clock() - hlock->holdtime_stamp;
351
352	stats = get_lock_stats(class: hlock_class(hlock));
353	if (hlock->read)
354	lock_time_inc(lt: &stats->read_holdtime, time: holdtime);
355	else
356	lock_time_inc(lt: &stats->write_holdtime, time: holdtime);
357	}
358	#else
359	static inline void lock_release_holdtime(struct held_lock *hlock)
360	{
361	}
362	#endif
363
364	/*
365	* We keep a global list of all lock classes. The list is only accessed with
366	* the lockdep spinlock lock held. free_lock_classes is a list with free
367	* elements. These elements are linked together by the lock_entry member in
368	* struct lock_class.
369	*/
370	static LIST_HEAD(all_lock_classes);
371	static LIST_HEAD(free_lock_classes);
372
373	/**
374	* struct pending_free - information about data structures about to be freed
375	* @zapped: Head of a list with struct lock_class elements.
376	* @lock_chains_being_freed: Bitmap that indicates which lock_chains[] elements
377	* are about to be freed.
378	*/
379	struct pending_free {
380	struct list_head zapped;
381	DECLARE_BITMAP(lock_chains_being_freed, MAX_LOCKDEP_CHAINS);
382	};
383
384	/**
385	* struct delayed_free - data structures used for delayed freeing
386	*
387	* A data structure for delayed freeing of data structures that may be
388	* accessed by RCU readers at the time these were freed.
389	*
390	* @rcu_head: Used to schedule an RCU callback for freeing data structures.
391	* @index: Index of @pf to which freed data structures are added.
392	* @scheduled: Whether or not an RCU callback has been scheduled.
393	* @pf: Array with information about data structures about to be freed.
394	*/
395	static struct delayed_free {
396	struct rcu_head rcu_head;
397	int index;
398	int scheduled;
399	struct pending_free pf[2];
400	} delayed_free;
401
402	/*
403	* The lockdep classes are in a hash-table as well, for fast lookup:
404	*/
405	#define CLASSHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1)
406	#define CLASSHASH_SIZE (1UL << CLASSHASH_BITS)
407	#define __classhashfn(key) hash_long((unsigned long)key, CLASSHASH_BITS)
408	#define classhashentry(key) (classhash_table + __classhashfn((key)))
409
410	static struct hlist_head classhash_table[CLASSHASH_SIZE];
411
412	/*
413	* We put the lock dependency chains into a hash-table as well, to cache
414	* their existence:
415	*/
416	#define CHAINHASH_BITS (MAX_LOCKDEP_CHAINS_BITS-1)
417	#define CHAINHASH_SIZE (1UL << CHAINHASH_BITS)
418	#define __chainhashfn(chain) hash_long(chain, CHAINHASH_BITS)
419	#define chainhashentry(chain) (chainhash_table + __chainhashfn((chain)))
420
421	static struct hlist_head chainhash_table[CHAINHASH_SIZE];
422
423	/*
424	* the id of held_lock
425	*/
426	static inline u16 hlock_id(struct held_lock *hlock)
427	{
428	BUILD_BUG_ON(MAX_LOCKDEP_KEYS_BITS + 2 > 16);
429
430	return (hlock->class_idx | (hlock->read << MAX_LOCKDEP_KEYS_BITS));
431	}
432
433	static inline unsigned int chain_hlock_class_idx(u16 hlock_id)
434	{
435	return hlock_id & (MAX_LOCKDEP_KEYS - 1);
436	}
437
438	/*
439	* The hash key of the lock dependency chains is a hash itself too:
440	* it's a hash of all locks taken up to that lock, including that lock.
441	* It's a 64-bit hash, because it's important for the keys to be
442	* unique.
443	*/
444	static inline u64 iterate_chain_key(u64 key, u32 idx)
445	{
446	u32 k0 = key, k1 = key >> 32;
447
448	__jhash_mix(idx, k0, k1); /* Macro that modifies arguments! */
449
450	return k0 | (u64)k1 << 32;
451	}
452
453	void lockdep_init_task(struct task_struct *task)
454	{
455	task->lockdep_depth = 0; /* no locks held yet */
456	task->curr_chain_key = INITIAL_CHAIN_KEY;
457	task->lockdep_recursion = 0;
458	}
459
460	static __always_inline void lockdep_recursion_inc(void)
461	{
462	__this_cpu_inc(lockdep_recursion);
463	}
464
465	static __always_inline void lockdep_recursion_finish(void)
466	{
467	if (WARN_ON_ONCE(__this_cpu_dec_return(lockdep_recursion)))
468	__this_cpu_write(lockdep_recursion, 0);
469	}
470
471	void lockdep_set_selftest_task(struct task_struct *task)
472	{
473	lockdep_selftest_task_struct = task;
474	}
475
476	/*
477	* Debugging switches:
478	*/
479
480	#define VERBOSE 0
481	#define VERY_VERBOSE 0
482
483	#if VERBOSE
484	# define HARDIRQ_VERBOSE 1
485	# define SOFTIRQ_VERBOSE 1
486	#else
487	# define HARDIRQ_VERBOSE 0
488	# define SOFTIRQ_VERBOSE 0
489	#endif
490
491	#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE
492	/*
493	* Quick filtering for interesting events:
494	*/
495	static int class_filter(struct lock_class *class)
496	{
497	#if 0
498	/* Example */
499	if (class->name_version == 1 &&
500	!strcmp(class->name, "lockname"))
501	return 1;
502	if (class->name_version == 1 &&
503	!strcmp(class->name, "&struct->lockfield"))
504	return 1;
505	#endif
506	/* Filter everything else. 1 would be to allow everything else */
507	return 0;
508	}
509	#endif
510
511	static int verbose(struct lock_class *class)
512	{
513	#if VERBOSE
514	return class_filter(class);
515	#endif
516	return 0;
517	}
518
519	static void print_lockdep_off(const char *bug_msg)
520	{
521	printk(KERN_DEBUG "%s\n", bug_msg);
522	printk(KERN_DEBUG "turning off the locking correctness validator.\n");
523	#ifdef CONFIG_LOCK_STAT
524	printk(KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n");
525	#endif
526	}
527
528	unsigned long nr_stack_trace_entries;
529
530	#ifdef CONFIG_PROVE_LOCKING
531	/**
532	* struct lock_trace - single stack backtrace
533	* @hash_entry: Entry in a stack_trace_hash[] list.
534	* @hash: jhash() of @entries.
535	* @nr_entries: Number of entries in @entries.
536	* @entries: Actual stack backtrace.
537	*/
538	struct lock_trace {
539	struct hlist_node hash_entry;
540	u32 hash;
541	u32 nr_entries;
542	unsigned long entries[] __aligned(sizeof(unsigned long));
543	};
544	#define LOCK_TRACE_SIZE_IN_LONGS \
545	(sizeof(struct lock_trace) / sizeof(unsigned long))
546	/*
547	* Stack-trace: sequence of lock_trace structures. Protected by the graph_lock.
548	*/
549	static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];
550	static struct hlist_head stack_trace_hash[STACK_TRACE_HASH_SIZE];
551
552	static bool traces_identical(struct lock_trace *t1, struct lock_trace *t2)
553	{
554	return t1->hash == t2->hash && t1->nr_entries == t2->nr_entries &&
555	memcmp(p: t1->entries, q: t2->entries,
556	size: t1->nr_entries * sizeof(t1->entries[0])) == 0;
557	}
558
559	static struct lock_trace *save_trace(void)
560	{
561	struct lock_trace *trace, *t2;
562	struct hlist_head *hash_head;
563	u32 hash;
564	int max_entries;
565
566	BUILD_BUG_ON_NOT_POWER_OF_2(STACK_TRACE_HASH_SIZE);
567	BUILD_BUG_ON(LOCK_TRACE_SIZE_IN_LONGS >= MAX_STACK_TRACE_ENTRIES);
568
569	trace = (struct lock_trace *)(stack_trace + nr_stack_trace_entries);
570	max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries -
571	LOCK_TRACE_SIZE_IN_LONGS;
572
573	if (max_entries <= 0) {
574	if (!debug_locks_off_graph_unlock())
575	return NULL;
576
577	print_lockdep_off(bug_msg: "BUG: MAX_STACK_TRACE_ENTRIES too low!");
578	dump_stack();
579
580	return NULL;
581	}
582	trace->nr_entries = stack_trace_save(store: trace->entries, size: max_entries, skipnr: 3);
583
584	hash = jhash(key: trace->entries, length: trace->nr_entries *
585	sizeof(trace->entries[0]), initval: 0);
586	trace->hash = hash;
587	hash_head = stack_trace_hash + (hash & (STACK_TRACE_HASH_SIZE - 1));
588	hlist_for_each_entry(t2, hash_head, hash_entry) {
589	if (traces_identical(t1: trace, t2))
590	return t2;
591	}
592	nr_stack_trace_entries += LOCK_TRACE_SIZE_IN_LONGS + trace->nr_entries;
593	hlist_add_head(n: &trace->hash_entry, h: hash_head);
594
595	return trace;
596	}
597
598	/* Return the number of stack traces in the stack_trace[] array. */
599	u64 lockdep_stack_trace_count(void)
600	{
601	struct lock_trace *trace;
602	u64 c = 0;
603	int i;
604
605	for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++) {
606	hlist_for_each_entry(trace, &stack_trace_hash[i], hash_entry) {
607	c++;
608	}
609	}
610
611	return c;
612	}
613
614	/* Return the number of stack hash chains that have at least one stack trace. */
615	u64 lockdep_stack_hash_count(void)
616	{
617	u64 c = 0;
618	int i;
619
620	for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++)
621	if (!hlist_empty(h: &stack_trace_hash[i]))
622	c++;
623
624	return c;
625	}
626	#endif
627
628	unsigned int nr_hardirq_chains;
629	unsigned int nr_softirq_chains;
630	unsigned int nr_process_chains;
631	unsigned int max_lockdep_depth;
632
633	#ifdef CONFIG_DEBUG_LOCKDEP
634	/*
635	* Various lockdep statistics:
636	*/
637	DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);
638	#endif
639
640	#ifdef CONFIG_PROVE_LOCKING
641	/*
642	* Locking printouts:
643	*/
644
645	#define __USAGE(__STATE) \
646	[LOCK_USED_IN_##__STATE] = "IN-"__stringify(__STATE)"-W", \
647	[LOCK_ENABLED_##__STATE] = __stringify(__STATE)"-ON-W", \
648	[LOCK_USED_IN_##__STATE##_READ] = "IN-"__stringify(__STATE)"-R",\
649	[LOCK_ENABLED_##__STATE##_READ] = __stringify(__STATE)"-ON-R",
650
651	static const char *usage_str[] =
652	{
653	#define LOCKDEP_STATE(__STATE) __USAGE(__STATE)
654	#include "lockdep_states.h"
655	#undef LOCKDEP_STATE
656	[LOCK_USED] = "INITIAL USE",
657	[LOCK_USED_READ] = "INITIAL READ USE",
658	/* abused as string storage for verify_lock_unused() */
659	[LOCK_USAGE_STATES] = "IN-NMI",
660	};
661	#endif
662
663	const char *__get_key_name(const struct lockdep_subclass_key *key, char *str)
664	{
665	return kallsyms_lookup(addr: (unsigned long)key, NULL, NULL, NULL, namebuf: str);
666	}
667
668	static inline unsigned long lock_flag(enum lock_usage_bit bit)
669	{
670	return 1UL << bit;
671	}
672
673	static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit)
674	{
675	/*
676	* The usage character defaults to '.' (i.e., irqs disabled and not in
677	* irq context), which is the safest usage category.
678	*/
679	char c = '.';
680
681	/*
682	* The order of the following usage checks matters, which will
683	* result in the outcome character as follows:
684	*
685	* - '+': irq is enabled and not in irq context
686	* - '-': in irq context and irq is disabled
687	* - '?': in irq context and irq is enabled
688	*/
689	if (class->usage_mask & lock_flag(bit: bit + LOCK_USAGE_DIR_MASK)) {
690	c = '+';
691	if (class->usage_mask & lock_flag(bit))
692	c = '?';
693	} else if (class->usage_mask & lock_flag(bit))
694	c = '-';
695
696	return c;
697	}
698
699	void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS])
700	{
701	int i = 0;
702
703	#define LOCKDEP_STATE(__STATE) \
704	usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE); \
705	usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);
706	#include "lockdep_states.h"
707	#undef LOCKDEP_STATE
708
709	usage[i] = '\0';
710	}
711
712	static void __print_lock_name(struct held_lock *hlock, struct lock_class *class)
713	{
714	char str[KSYM_NAME_LEN];
715	const char *name;
716
717	name = class->name;
718	if (!name) {
719	name = __get_key_name(key: class->key, str);
720	printk(KERN_CONT "%s", name);
721	} else {
722	printk(KERN_CONT "%s", name);
723	if (class->name_version > 1)
724	printk(KERN_CONT "#%d", class->name_version);
725	if (class->subclass)
726	printk(KERN_CONT "/%d", class->subclass);
727	if (hlock && class->print_fn)
728	class->print_fn(hlock->instance);
729	}
730	}
731
732	static void print_lock_name(struct held_lock *hlock, struct lock_class *class)
733	{
734	char usage[LOCK_USAGE_CHARS];
735
736	get_usage_chars(class, usage);
737
738	printk(KERN_CONT " (");
739	__print_lock_name(hlock, class);
740	printk(KERN_CONT "){%s}-{%d:%d}", usage,
741	class->wait_type_outer ?: class->wait_type_inner,
742	class->wait_type_inner);
743	}
744
745	static void print_lockdep_cache(struct lockdep_map *lock)
746	{
747	const char *name;
748	char str[KSYM_NAME_LEN];
749
750	name = lock->name;
751	if (!name)
752	name = __get_key_name(key: lock->key->subkeys, str);
753
754	printk(KERN_CONT "%s", name);
755	}
756
757	static void print_lock(struct held_lock *hlock)
758	{
759	/*
760	* We can be called locklessly through debug_show_all_locks() so be
761	* extra careful, the hlock might have been released and cleared.
762	*
763	* If this indeed happens, lets pretend it does not hurt to continue
764	* to print the lock unless the hlock class_idx does not point to a
765	* registered class. The rationale here is: since we don't attempt
766	* to distinguish whether we are in this situation, if it just
767	* happened we can't count on class_idx to tell either.
768	*/
769	struct lock_class *lock = hlock_class(hlock);
770
771	if (!lock) {
772	printk(KERN_CONT "<RELEASED>\n");
773	return;
774	}
775
776	printk(KERN_CONT "%px", hlock->instance);
777	print_lock_name(hlock, class: lock);
778	printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);
779	}
780
781	static void lockdep_print_held_locks(struct task_struct *p)
782	{
783	int i, depth = READ_ONCE(p->lockdep_depth);
784
785	if (!depth)
786	printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p));
787	else
788	printk("%d lock%s held by %s/%d:\n", depth,
789	depth > 1 ? "s" : "", p->comm, task_pid_nr(p));
790	/*
791	* It's not reliable to print a task's held locks if it's not sleeping
792	* and it's not the current task.
793	*/
794	if (p != current && task_is_running(p))
795	return;
796	for (i = 0; i < depth; i++) {
797	printk(" #%d: ", i);
798	print_lock(hlock: p->held_locks + i);
799	}
800	}
801
802	static void print_kernel_ident(void)
803	{
804	printk("%s %.*s %s\n", init_utsname()->release,
805	(int)strcspn(init_utsname()->version, " "),
806	init_utsname()->version,
807	print_tainted());
808	}
809
810	static int very_verbose(struct lock_class *class)
811	{
812	#if VERY_VERBOSE
813	return class_filter(class);
814	#endif
815	return 0;
816	}
817
818	/*
819	* Is this the address of a static object:
820	*/
821	#ifdef __KERNEL__
822	static int static_obj(const void *obj)
823	{
824	unsigned long addr = (unsigned long) obj;
825
826	if (is_kernel_core_data(addr))
827	return 1;
828
829	/*
830	* keys are allowed in the __ro_after_init section.
831	*/
832	if (is_kernel_rodata(addr))
833	return 1;
834
835	/*
836	* in initdata section and used during bootup only?
837	* NOTE: On some platforms the initdata section is
838	* outside of the _stext ... _end range.
839	*/
840	if (system_state < SYSTEM_FREEING_INITMEM &&
841	init_section_contains(virt: (void *)addr, size: 1))
842	return 1;
843
844	/*
845	* in-kernel percpu var?
846	*/
847	if (is_kernel_percpu_address(addr))
848	return 1;
849
850	/*
851	* module static or percpu var?
852	*/
853	return is_module_address(addr) || is_module_percpu_address(addr);
854	}
855	#endif
856
857	/*
858	* To make lock name printouts unique, we calculate a unique
859	* class->name_version generation counter. The caller must hold the graph
860	* lock.
861	*/
862	static int count_matching_names(struct lock_class *new_class)
863	{
864	struct lock_class *class;
865	int count = 0;
866
867	if (!new_class->name)
868	return 0;
869
870	list_for_each_entry(class, &all_lock_classes, lock_entry) {
871	if (new_class->key - new_class->subclass == class->key)
872	return class->name_version;
873	if (class->name && !strcmp(class->name, new_class->name))
874	count = max(count, class->name_version);
875	}
876
877	return count + 1;
878	}
879
880	/* used from NMI context -- must be lockless */
881	static noinstr struct lock_class *
882	look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
883	{
884	struct lockdep_subclass_key *key;
885	struct hlist_head *hash_head;
886	struct lock_class *class;
887
888	if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
889	instrumentation_begin();
890	debug_locks_off();
891	printk(KERN_ERR
892	"BUG: looking up invalid subclass: %u\n", subclass);
893	printk(KERN_ERR
894	"turning off the locking correctness validator.\n");
895	dump_stack();
896	instrumentation_end();
897	return NULL;
898	}
899
900	/*
901	* If it is not initialised then it has never been locked,
902	* so it won't be present in the hash table.
903	*/
904	if (unlikely(!lock->key))
905	return NULL;
906
907	/*
908	* NOTE: the class-key must be unique. For dynamic locks, a static
909	* lock_class_key variable is passed in through the mutex_init()
910	* (or spin_lock_init()) call - which acts as the key. For static
911	* locks we use the lock object itself as the key.
912	*/
913	BUILD_BUG_ON(sizeof(struct lock_class_key) >
914	sizeof(struct lockdep_map));
915
916	key = lock->key->subkeys + subclass;
917
918	hash_head = classhashentry(key);
919
920	/*
921	* We do an RCU walk of the hash, see lockdep_free_key_range().
922	*/
923	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
924	return NULL;
925
926	hlist_for_each_entry_rcu_notrace(class, hash_head, hash_entry) {
927	if (class->key == key) {
928	/*
929	* Huh! same key, different name? Did someone trample
930	* on some memory? We're most confused.
931	*/
932	WARN_ONCE(class->name != lock->name &&
933	lock->key != &__lockdep_no_validate__,
934	"Looking for class \"%s\" with key %ps, but found a different class \"%s\" with the same key\n",
935	lock->name, lock->key, class->name);
936	return class;
937	}
938	}
939
940	return NULL;
941	}
942
943	/*
944	* Static locks do not have their class-keys yet - for them the key is
945	* the lock object itself. If the lock is in the per cpu area, the
946	* canonical address of the lock (per cpu offset removed) is used.
947	*/
948	static bool assign_lock_key(struct lockdep_map *lock)
949	{
950	unsigned long can_addr, addr = (unsigned long)lock;
951
952	#ifdef __KERNEL__
953	/*
954	* lockdep_free_key_range() assumes that struct lock_class_key
955	* objects do not overlap. Since we use the address of lock
956	* objects as class key for static objects, check whether the
957	* size of lock_class_key objects does not exceed the size of
958	* the smallest lock object.
959	*/
960	BUILD_BUG_ON(sizeof(struct lock_class_key) > sizeof(raw_spinlock_t));
961	#endif
962
963	if (__is_kernel_percpu_address(addr, can_addr: &can_addr))
964	lock->key = (void *)can_addr;
965	else if (__is_module_percpu_address(addr, can_addr: &can_addr))
966	lock->key = (void *)can_addr;
967	else if (static_obj(obj: lock))
968	lock->key = (void *)lock;
969	else {
970	/* Debug-check: all keys must be persistent! */
971	debug_locks_off();
972	pr_err("INFO: trying to register non-static key.\n");
973	pr_err("The code is fine but needs lockdep annotation, or maybe\n");
974	pr_err("you didn't initialize this object before use?\n");
975	pr_err("turning off the locking correctness validator.\n");
976	dump_stack();
977	return false;
978	}
979
980	return true;
981	}
982
983	#ifdef CONFIG_DEBUG_LOCKDEP
984
985	/* Check whether element @e occurs in list @h */
986	static bool in_list(struct list_head *e, struct list_head *h)
987	{
988	struct list_head *f;
989
990	list_for_each(f, h) {
991	if (e == f)
992	return true;
993	}
994
995	return false;
996	}
997
998	/*
999	* Check whether entry @e occurs in any of the locks_after or locks_before
1000	* lists.
1001	*/
1002	static bool in_any_class_list(struct list_head *e)
1003	{
1004	struct lock_class *class;
1005	int i;
1006
1007	for (i = 0; i < ARRAY_SIZE(lock_classes); i++) {
1008	class = &lock_classes[i];
1009	if (in_list(e, h: &class->locks_after) ||
1010	in_list(e, h: &class->locks_before))
1011	return true;
1012	}
1013	return false;
1014	}
1015
1016	static bool class_lock_list_valid(struct lock_class *c, struct list_head *h)
1017	{
1018	struct lock_list *e;
1019
1020	list_for_each_entry(e, h, entry) {
1021	if (e->links_to != c) {
1022	printk(KERN_INFO "class %s: mismatch for lock entry %ld; class %s <> %s",
1023	c->name ? : "(?)",
1024	(unsigned long)(e - list_entries),
1025	e->links_to && e->links_to->name ?
1026	e->links_to->name : "(?)",
1027	e->class && e->class->name ? e->class->name :
1028	"(?)");
1029	return false;
1030	}
1031	}
1032	return true;
1033	}
1034
1035	#ifdef CONFIG_PROVE_LOCKING
1036	static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS];
1037	#endif
1038
1039	static bool check_lock_chain_key(struct lock_chain *chain)
1040	{
1041	#ifdef CONFIG_PROVE_LOCKING
1042	u64 chain_key = INITIAL_CHAIN_KEY;
1043	int i;
1044
1045	for (i = chain->base; i < chain->base + chain->depth; i++)
1046	chain_key = iterate_chain_key(key: chain_key, idx: chain_hlocks[i]);
1047	/*
1048	* The 'unsigned long long' casts avoid that a compiler warning
1049	* is reported when building tools/lib/lockdep.
1050	*/
1051	if (chain->chain_key != chain_key) {
1052	printk(KERN_INFO "chain %lld: key %#llx <> %#llx\n",
1053	(unsigned long long)(chain - lock_chains),
1054	(unsigned long long)chain->chain_key,
1055	(unsigned long long)chain_key);
1056	return false;
1057	}
1058	#endif
1059	return true;
1060	}
1061
1062	static bool in_any_zapped_class_list(struct lock_class *class)
1063	{
1064	struct pending_free *pf;
1065	int i;
1066
1067	for (i = 0, pf = delayed_free.pf; i < ARRAY_SIZE(delayed_free.pf); i++, pf++) {
1068	if (in_list(e: &class->lock_entry, h: &pf->zapped))
1069	return true;
1070	}
1071
1072	return false;
1073	}
1074
1075	static bool __check_data_structures(void)
1076	{
1077	struct lock_class *class;
1078	struct lock_chain *chain;
1079	struct hlist_head *head;
1080	struct lock_list *e;
1081	int i;
1082
1083	/* Check whether all classes occur in a lock list. */
1084	for (i = 0; i < ARRAY_SIZE(lock_classes); i++) {
1085	class = &lock_classes[i];
1086	if (!in_list(e: &class->lock_entry, h: &all_lock_classes) &&
1087	!in_list(e: &class->lock_entry, h: &free_lock_classes) &&
1088	!in_any_zapped_class_list(class)) {
1089	printk(KERN_INFO "class %px/%s is not in any class list\n",
1090	class, class->name ? : "(?)");
1091	return false;
1092	}
1093	}
1094
1095	/* Check whether all classes have valid lock lists. */
1096	for (i = 0; i < ARRAY_SIZE(lock_classes); i++) {
1097	class = &lock_classes[i];
1098	if (!class_lock_list_valid(c: class, h: &class->locks_before))
1099	return false;
1100	if (!class_lock_list_valid(c: class, h: &class->locks_after))
1101	return false;
1102	}
1103
1104	/* Check the chain_key of all lock chains. */
1105	for (i = 0; i < ARRAY_SIZE(chainhash_table); i++) {
1106	head = chainhash_table + i;
1107	hlist_for_each_entry_rcu(chain, head, entry) {
1108	if (!check_lock_chain_key(chain))
1109	return false;
1110	}
1111	}
1112
1113	/*
1114	* Check whether all list entries that are in use occur in a class
1115	* lock list.
1116	*/
1117	for_each_set_bit(i, list_entries_in_use, ARRAY_SIZE(list_entries)) {
1118	e = list_entries + i;
1119	if (!in_any_class_list(e: &e->entry)) {
1120	printk(KERN_INFO "list entry %d is not in any class list; class %s <> %s\n",
1121	(unsigned int)(e - list_entries),
1122	e->class->name ? : "(?)",
1123	e->links_to->name ? : "(?)");
1124	return false;
1125	}
1126	}
1127
1128	/*
1129	* Check whether all list entries that are not in use do not occur in
1130	* a class lock list.
1131	*/
1132	for_each_clear_bit(i, list_entries_in_use, ARRAY_SIZE(list_entries)) {
1133	e = list_entries + i;
1134	if (in_any_class_list(e: &e->entry)) {
1135	printk(KERN_INFO "list entry %d occurs in a class list; class %s <> %s\n",
1136	(unsigned int)(e - list_entries),
1137	e->class && e->class->name ? e->class->name :
1138	"(?)",
1139	e->links_to && e->links_to->name ?
1140	e->links_to->name : "(?)");
1141	return false;
1142	}
1143	}
1144
1145	return true;
1146	}
1147
1148	int check_consistency = 0;
1149	module_param(check_consistency, int, 0644);
1150
1151	static void check_data_structures(void)
1152	{
1153	static bool once = false;
1154
1155	if (check_consistency && !once) {
1156	if (!__check_data_structures()) {
1157	once = true;
1158	WARN_ON(once);
1159	}
1160	}
1161	}
1162
1163	#else /* CONFIG_DEBUG_LOCKDEP */
1164
1165	static inline void check_data_structures(void) { }
1166
1167	#endif /* CONFIG_DEBUG_LOCKDEP */
1168
1169	static void init_chain_block_buckets(void);
1170
1171	/*
1172	* Initialize the lock_classes[] array elements, the free_lock_classes list
1173	* and also the delayed_free structure.
1174	*/
1175	static void init_data_structures_once(void)
1176	{
1177	static bool __read_mostly ds_initialized, rcu_head_initialized;
1178	int i;
1179
1180	if (likely(rcu_head_initialized))
1181	return;
1182
1183	if (system_state >= SYSTEM_SCHEDULING) {
1184	init_rcu_head(head: &delayed_free.rcu_head);
1185	rcu_head_initialized = true;
1186	}
1187
1188	if (ds_initialized)
1189	return;
1190
1191	ds_initialized = true;
1192
1193	INIT_LIST_HEAD(list: &delayed_free.pf[0].zapped);
1194	INIT_LIST_HEAD(list: &delayed_free.pf[1].zapped);
1195
1196	for (i = 0; i < ARRAY_SIZE(lock_classes); i++) {
1197	list_add_tail(new: &lock_classes[i].lock_entry, head: &free_lock_classes);
1198	INIT_LIST_HEAD(list: &lock_classes[i].locks_after);
1199	INIT_LIST_HEAD(list: &lock_classes[i].locks_before);
1200	}
1201	init_chain_block_buckets();
1202	}
1203
1204	static inline struct hlist_head *keyhashentry(const struct lock_class_key *key)
1205	{
1206	unsigned long hash = hash_long((uintptr_t)key, KEYHASH_BITS);
1207
1208	return lock_keys_hash + hash;
1209	}
1210
1211	/* Register a dynamically allocated key. */
1212	void lockdep_register_key(struct lock_class_key *key)
1213	{
1214	struct hlist_head *hash_head;
1215	struct lock_class_key *k;
1216	unsigned long flags;
1217
1218	if (WARN_ON_ONCE(static_obj(key)))
1219	return;
1220	hash_head = keyhashentry(key);
1221
1222	raw_local_irq_save(flags);
1223	if (!graph_lock())
1224	goto restore_irqs;
1225	hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
1226	if (WARN_ON_ONCE(k == key))
1227	goto out_unlock;
1228	}
1229	hlist_add_head_rcu(n: &key->hash_entry, h: hash_head);
1230	out_unlock:
1231	graph_unlock();
1232	restore_irqs:
1233	raw_local_irq_restore(flags);
1234	}
1235	EXPORT_SYMBOL_GPL(lockdep_register_key);
1236
1237	/* Check whether a key has been registered as a dynamic key. */
1238	static bool is_dynamic_key(const struct lock_class_key *key)
1239	{
1240	struct hlist_head *hash_head;
1241	struct lock_class_key *k;
1242	bool found = false;
1243
1244	if (WARN_ON_ONCE(static_obj(key)))
1245	return false;
1246
1247	/*
1248	* If lock debugging is disabled lock_keys_hash[] may contain
1249	* pointers to memory that has already been freed. Avoid triggering
1250	* a use-after-free in that case by returning early.
1251	*/
1252	if (!debug_locks)
1253	return true;
1254
1255	hash_head = keyhashentry(key);
1256
1257	rcu_read_lock();
1258	hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
1259	if (k == key) {
1260	found = true;
1261	break;
1262	}
1263	}
1264	rcu_read_unlock();
1265
1266	return found;
1267	}
1268
1269	/*
1270	* Register a lock's class in the hash-table, if the class is not present
1271	* yet. Otherwise we look it up. We cache the result in the lock object
1272	* itself, so actual lookup of the hash should be once per lock object.
1273	*/
1274	static struct lock_class *
1275	register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
1276	{
1277	struct lockdep_subclass_key *key;
1278	struct hlist_head *hash_head;
1279	struct lock_class *class;
1280	int idx;
1281
1282	DEBUG_LOCKS_WARN_ON(!irqs_disabled());
1283
1284	class = look_up_lock_class(lock, subclass);
1285	if (likely(class))
1286	goto out_set_class_cache;
1287
1288	if (!lock->key) {
1289	if (!assign_lock_key(lock))
1290	return NULL;
1291	} else if (!static_obj(obj: lock->key) && !is_dynamic_key(key: lock->key)) {
1292	return NULL;
1293	}
1294
1295	key = lock->key->subkeys + subclass;
1296	hash_head = classhashentry(key);
1297
1298	if (!graph_lock()) {
1299	return NULL;
1300	}
1301	/*
1302	* We have to do the hash-walk again, to avoid races
1303	* with another CPU:
1304	*/
1305	hlist_for_each_entry_rcu(class, hash_head, hash_entry) {
1306	if (class->key == key)
1307	goto out_unlock_set;
1308	}
1309
1310	init_data_structures_once();
1311
1312	/* Allocate a new lock class and add it to the hash. */
1313	class = list_first_entry_or_null(&free_lock_classes, typeof(*class),
1314	lock_entry);
1315	if (!class) {
1316	if (!debug_locks_off_graph_unlock()) {
1317	return NULL;
1318	}
1319
1320	print_lockdep_off(bug_msg: "BUG: MAX_LOCKDEP_KEYS too low!");
1321	dump_stack();
1322	return NULL;
1323	}
1324	nr_lock_classes++;
1325	__set_bit(class - lock_classes, lock_classes_in_use);
1326	debug_atomic_inc(nr_unused_locks);
1327	class->key = key;
1328	class->name = lock->name;
1329	class->subclass = subclass;
1330	WARN_ON_ONCE(!list_empty(&class->locks_before));
1331	WARN_ON_ONCE(!list_empty(&class->locks_after));
1332	class->name_version = count_matching_names(new_class: class);
1333	class->wait_type_inner = lock->wait_type_inner;
1334	class->wait_type_outer = lock->wait_type_outer;
1335	class->lock_type = lock->lock_type;
1336	/*
1337	* We use RCU's safe list-add method to make
1338	* parallel walking of the hash-list safe:
1339	*/
1340	hlist_add_head_rcu(n: &class->hash_entry, h: hash_head);
1341	/*
1342	* Remove the class from the free list and add it to the global list
1343	* of classes.
1344	*/
1345	list_move_tail(list: &class->lock_entry, head: &all_lock_classes);
1346	idx = class - lock_classes;
1347	if (idx > max_lock_class_idx)
1348	max_lock_class_idx = idx;
1349
1350	if (verbose(class)) {
1351	graph_unlock();
1352
1353	printk("\nnew class %px: %s", class->key, class->name);
1354	if (class->name_version > 1)
1355	printk(KERN_CONT "#%d", class->name_version);
1356	printk(KERN_CONT "\n");
1357	dump_stack();
1358
1359	if (!graph_lock()) {
1360	return NULL;
1361	}
1362	}
1363	out_unlock_set:
1364	graph_unlock();
1365
1366	out_set_class_cache:
1367	if (!subclass || force)
1368	lock->class_cache[0] = class;
1369	else if (subclass < NR_LOCKDEP_CACHING_CLASSES)
1370	lock->class_cache[subclass] = class;
1371
1372	/*
1373	* Hash collision, did we smoke some? We found a class with a matching
1374	* hash but the subclass -- which is hashed in -- didn't match.
1375	*/
1376	if (DEBUG_LOCKS_WARN_ON(class->subclass != subclass))
1377	return NULL;
1378
1379	return class;
1380	}
1381
1382	#ifdef CONFIG_PROVE_LOCKING
1383	/*
1384	* Allocate a lockdep entry. (assumes the graph_lock held, returns
1385	* with NULL on failure)
1386	*/
1387	static struct lock_list *alloc_list_entry(void)
1388	{
1389	int idx = find_first_zero_bit(addr: list_entries_in_use,
1390	ARRAY_SIZE(list_entries));
1391
1392	if (idx >= ARRAY_SIZE(list_entries)) {
1393	if (!debug_locks_off_graph_unlock())
1394	return NULL;
1395
1396	print_lockdep_off(bug_msg: "BUG: MAX_LOCKDEP_ENTRIES too low!");
1397	dump_stack();
1398	return NULL;
1399	}
1400	nr_list_entries++;
1401	__set_bit(idx, list_entries_in_use);
1402	return list_entries + idx;
1403	}
1404
1405	/*
1406	* Add a new dependency to the head of the list:
1407	*/
1408	static int add_lock_to_list(struct lock_class *this,
1409	struct lock_class *links_to, struct list_head *head,
1410	u16 distance, u8 dep,
1411	const struct lock_trace *trace)
1412	{
1413	struct lock_list *entry;
1414	/*
1415	* Lock not present yet - get a new dependency struct and
1416	* add it to the list:
1417	*/
1418	entry = alloc_list_entry();
1419	if (!entry)
1420	return 0;
1421
1422	entry->class = this;
1423	entry->links_to = links_to;
1424	entry->dep = dep;
1425	entry->distance = distance;
1426	entry->trace = trace;
1427	/*
1428	* Both allocation and removal are done under the graph lock; but
1429	* iteration is under RCU-sched; see look_up_lock_class() and
1430	* lockdep_free_key_range().
1431	*/
1432	list_add_tail_rcu(new: &entry->entry, head);
1433
1434	return 1;
1435	}
1436
1437	/*
1438	* For good efficiency of modular, we use power of 2
1439	*/
1440	#define MAX_CIRCULAR_QUEUE_SIZE (1UL << CONFIG_LOCKDEP_CIRCULAR_QUEUE_BITS)
1441	#define CQ_MASK (MAX_CIRCULAR_QUEUE_SIZE-1)
1442
1443	/*
1444	* The circular_queue and helpers are used to implement graph
1445	* breadth-first search (BFS) algorithm, by which we can determine
1446	* whether there is a path from a lock to another. In deadlock checks,
1447	* a path from the next lock to be acquired to a previous held lock
1448	* indicates that adding the <prev> -> <next> lock dependency will
1449	* produce a circle in the graph. Breadth-first search instead of
1450	* depth-first search is used in order to find the shortest (circular)
1451	* path.
1452	*/
1453	struct circular_queue {
1454	struct lock_list *element[MAX_CIRCULAR_QUEUE_SIZE];
1455	unsigned int front, rear;
1456	};
1457
1458	static struct circular_queue lock_cq;
1459
1460	unsigned int max_bfs_queue_depth;
1461
1462	static unsigned int lockdep_dependency_gen_id;
1463
1464	static inline void __cq_init(struct circular_queue *cq)
1465	{
1466	cq->front = cq->rear = 0;
1467	lockdep_dependency_gen_id++;
1468	}
1469
1470	static inline int __cq_empty(struct circular_queue *cq)
1471	{
1472	return (cq->front == cq->rear);
1473	}
1474
1475	static inline int __cq_full(struct circular_queue *cq)
1476	{
1477	return ((cq->rear + 1) & CQ_MASK) == cq->front;
1478	}
1479
1480	static inline int __cq_enqueue(struct circular_queue *cq, struct lock_list *elem)
1481	{
1482	if (__cq_full(cq))
1483	return -1;
1484
1485	cq->element[cq->rear] = elem;
1486	cq->rear = (cq->rear + 1) & CQ_MASK;
1487	return 0;
1488	}
1489
1490	/*
1491	* Dequeue an element from the circular_queue, return a lock_list if
1492	* the queue is not empty, or NULL if otherwise.
1493	*/
1494	static inline struct lock_list * __cq_dequeue(struct circular_queue *cq)
1495	{
1496	struct lock_list * lock;
1497
1498	if (__cq_empty(cq))
1499	return NULL;
1500
1501	lock = cq->element[cq->front];
1502	cq->front = (cq->front + 1) & CQ_MASK;
1503
1504	return lock;
1505	}
1506
1507	static inline unsigned int __cq_get_elem_count(struct circular_queue *cq)
1508	{
1509	return (cq->rear - cq->front) & CQ_MASK;
1510	}
1511
1512	static inline void mark_lock_accessed(struct lock_list *lock)
1513	{
1514	lock->class->dep_gen_id = lockdep_dependency_gen_id;
1515	}
1516
1517	static inline void visit_lock_entry(struct lock_list *lock,
1518	struct lock_list *parent)
1519	{
1520	lock->parent = parent;
1521	}
1522
1523	static inline unsigned long lock_accessed(struct lock_list *lock)
1524	{
1525	return lock->class->dep_gen_id == lockdep_dependency_gen_id;
1526	}
1527
1528	static inline struct lock_list *get_lock_parent(struct lock_list *child)
1529	{
1530	return child->parent;
1531	}
1532
1533	static inline int get_lock_depth(struct lock_list *child)
1534	{
1535	int depth = 0;
1536	struct lock_list *parent;
1537
1538	while ((parent = get_lock_parent(child))) {
1539	child = parent;
1540	depth++;
1541	}
1542	return depth;
1543	}
1544
1545	/*
1546	* Return the forward or backward dependency list.
1547	*
1548	* @lock: the lock_list to get its class's dependency list
1549	* @offset: the offset to struct lock_class to determine whether it is
1550	* locks_after or locks_before
1551	*/
1552	static inline struct list_head *get_dep_list(struct lock_list *lock, int offset)
1553	{
1554	void *lock_class = lock->class;
1555
1556	return lock_class + offset;
1557	}
1558	/*
1559	* Return values of a bfs search:
1560	*
1561	* BFS_E* indicates an error
1562	* BFS_R* indicates a result (match or not)
1563	*
1564	* BFS_EINVALIDNODE: Find a invalid node in the graph.
1565	*
1566	* BFS_EQUEUEFULL: The queue is full while doing the bfs.
1567	*
1568	* BFS_RMATCH: Find the matched node in the graph, and put that node into
1569	* *@target_entry.
1570	*
1571	* BFS_RNOMATCH: Haven't found the matched node and keep *@target_entry
1572	* _unchanged_.
1573	*/
1574	enum bfs_result {
1575	BFS_EINVALIDNODE = -2,
1576	BFS_EQUEUEFULL = -1,
1577	BFS_RMATCH = 0,
1578	BFS_RNOMATCH = 1,
1579	};
1580
1581	/*
1582	* bfs_result < 0 means error
1583	*/
1584	static inline bool bfs_error(enum bfs_result res)
1585	{
1586	return res < 0;
1587	}
1588
1589	/*
1590	* DEP_*_BIT in lock_list::dep
1591	*
1592	* For dependency @prev -> @next:
1593	*
1594	* SR: @prev is shared reader (->read != 0) and @next is recursive reader
1595	* (->read == 2)
1596	* ER: @prev is exclusive locker (->read == 0) and @next is recursive reader
1597	* SN: @prev is shared reader and @next is non-recursive locker (->read != 2)
1598	* EN: @prev is exclusive locker and @next is non-recursive locker
1599	*
1600	* Note that we define the value of DEP_*_BITs so that:
1601	* bit0 is prev->read == 0
1602	* bit1 is next->read != 2
1603	*/
1604	#define DEP_SR_BIT (0 + (0 << 1)) /* 0 */
1605	#define DEP_ER_BIT (1 + (0 << 1)) /* 1 */
1606	#define DEP_SN_BIT (0 + (1 << 1)) /* 2 */
1607	#define DEP_EN_BIT (1 + (1 << 1)) /* 3 */
1608
1609	#define DEP_SR_MASK (1U << (DEP_SR_BIT))
1610	#define DEP_ER_MASK (1U << (DEP_ER_BIT))
1611	#define DEP_SN_MASK (1U << (DEP_SN_BIT))
1612	#define DEP_EN_MASK (1U << (DEP_EN_BIT))
1613
1614	static inline unsigned int
1615	__calc_dep_bit(struct held_lock *prev, struct held_lock *next)
1616	{
1617	return (prev->read == 0) + ((next->read != 2) << 1);
1618	}
1619
1620	static inline u8 calc_dep(struct held_lock *prev, struct held_lock *next)
1621	{
1622	return 1U << __calc_dep_bit(prev, next);
1623	}
1624
1625	/*
1626	* calculate the dep_bit for backwards edges. We care about whether @prev is
1627	* shared and whether @next is recursive.
1628	*/
1629	static inline unsigned int
1630	__calc_dep_bitb(struct held_lock *prev, struct held_lock *next)
1631	{
1632	return (next->read != 2) + ((prev->read == 0) << 1);
1633	}
1634
1635	static inline u8 calc_depb(struct held_lock *prev, struct held_lock *next)
1636	{
1637	return 1U << __calc_dep_bitb(prev, next);
1638	}
1639
1640	/*
1641	* Initialize a lock_list entry @lock belonging to @class as the root for a BFS
1642	* search.
1643	*/
1644	static inline void __bfs_init_root(struct lock_list *lock,
1645	struct lock_class *class)
1646	{
1647	lock->class = class;
1648	lock->parent = NULL;
1649	lock->only_xr = 0;
1650	}
1651
1652	/*
1653	* Initialize a lock_list entry @lock based on a lock acquisition @hlock as the
1654	* root for a BFS search.
1655	*
1656	* ->only_xr of the initial lock node is set to @hlock->read == 2, to make sure
1657	* that <prev> -> @hlock and @hlock -> <whatever __bfs() found> is not -(*R)->
1658	* and -(S*)->.
1659	*/
1660	static inline void bfs_init_root(struct lock_list *lock,
1661	struct held_lock *hlock)
1662	{
1663	__bfs_init_root(lock, class: hlock_class(hlock));
1664	lock->only_xr = (hlock->read == 2);
1665	}
1666
1667	/*
1668	* Similar to bfs_init_root() but initialize the root for backwards BFS.
1669	*
1670	* ->only_xr of the initial lock node is set to @hlock->read != 0, to make sure
1671	* that <next> -> @hlock and @hlock -> <whatever backwards BFS found> is not
1672	* -(*S)-> and -(R*)-> (reverse order of -(*R)-> and -(S*)->).
1673	*/
1674	static inline void bfs_init_rootb(struct lock_list *lock,
1675	struct held_lock *hlock)
1676	{
1677	__bfs_init_root(lock, class: hlock_class(hlock));
1678	lock->only_xr = (hlock->read != 0);
1679	}
1680
1681	static inline struct lock_list *__bfs_next(struct lock_list *lock, int offset)
1682	{
1683	if (!lock || !lock->parent)
1684	return NULL;
1685
1686	return list_next_or_null_rcu(get_dep_list(lock->parent, offset),
1687	&lock->entry, struct lock_list, entry);
1688	}
1689
1690	/*
1691	* Breadth-First Search to find a strong path in the dependency graph.
1692	*
1693	* @source_entry: the source of the path we are searching for.
1694	* @data: data used for the second parameter of @match function
1695	* @match: match function for the search
1696	* @target_entry: pointer to the target of a matched path
1697	* @offset: the offset to struct lock_class to determine whether it is
1698	* locks_after or locks_before
1699	*
1700	* We may have multiple edges (considering different kinds of dependencies,
1701	* e.g. ER and SN) between two nodes in the dependency graph. But
1702	* only the strong dependency path in the graph is relevant to deadlocks. A
1703	* strong dependency path is a dependency path that doesn't have two adjacent
1704	* dependencies as -(*R)-> -(S*)->, please see:
1705	*
1706	* Documentation/locking/lockdep-design.rst
1707	*
1708	* for more explanation of the definition of strong dependency paths
1709	*
1710	* In __bfs(), we only traverse in the strong dependency path:
1711	*
1712	* In lock_list::only_xr, we record whether the previous dependency only
1713	* has -(*R)-> in the search, and if it does (prev only has -(*R)->), we
1714	* filter out any -(S*)-> in the current dependency and after that, the
1715	* ->only_xr is set according to whether we only have -(*R)-> left.
1716	*/
1717	static enum bfs_result __bfs(struct lock_list *source_entry,
1718	void *data,
1719	bool (*match)(struct lock_list *entry, void *data),
1720	bool (*skip)(struct lock_list *entry, void *data),
1721	struct lock_list **target_entry,
1722	int offset)
1723	{
1724	struct circular_queue *cq = &lock_cq;
1725	struct lock_list *lock = NULL;
1726	struct lock_list *entry;
1727	struct list_head *head;
1728	unsigned int cq_depth;
1729	bool first;
1730
1731	lockdep_assert_locked();
1732
1733	__cq_init(cq);
1734	__cq_enqueue(cq, elem: source_entry);
1735
1736	while ((lock = __bfs_next(lock, offset)) || (lock = __cq_dequeue(cq))) {
1737	if (!lock->class)
1738	return BFS_EINVALIDNODE;
1739
1740	/*
1741	* Step 1: check whether we already finish on this one.
1742	*
1743	* If we have visited all the dependencies from this @lock to
1744	* others (iow, if we have visited all lock_list entries in
1745	* @lock->class->locks_{after,before}) we skip, otherwise go
1746	* and visit all the dependencies in the list and mark this
1747	* list accessed.
1748	*/
1749	if (lock_accessed(lock))
1750	continue;
1751	else
1752	mark_lock_accessed(lock);
1753
1754	/*
1755	* Step 2: check whether prev dependency and this form a strong
1756	* dependency path.
1757	*/
1758	if (lock->parent) { /* Parent exists, check prev dependency */
1759	u8 dep = lock->dep;
1760	bool prev_only_xr = lock->parent->only_xr;
1761
1762	/*
1763	* Mask out all -(S*)-> if we only have *R in previous
1764	* step, because -(*R)-> -(S*)-> don't make up a strong
1765	* dependency.
1766	*/
1767	if (prev_only_xr)
1768	dep &= ~(DEP_SR_MASK | DEP_SN_MASK);
1769
1770	/* If nothing left, we skip */
1771	if (!dep)
1772	continue;
1773
1774	/* If there are only -(*R)-> left, set that for the next step */
1775	lock->only_xr = !(dep & (DEP_SN_MASK | DEP_EN_MASK));
1776	}
1777
1778	/*
1779	* Step 3: we haven't visited this and there is a strong
1780	* dependency path to this, so check with @match.
1781	* If @skip is provide and returns true, we skip this
1782	* lock (and any path this lock is in).
1783	*/
1784	if (skip && skip(lock, data))
1785	continue;
1786
1787	if (match(lock, data)) {
1788	*target_entry = lock;
1789	return BFS_RMATCH;
1790	}
1791
1792	/*
1793	* Step 4: if not match, expand the path by adding the
1794	* forward or backwards dependencies in the search
1795	*
1796	*/
1797	first = true;
1798	head = get_dep_list(lock, offset);
1799	list_for_each_entry_rcu(entry, head, entry) {
1800	visit_lock_entry(lock: entry, parent: lock);
1801
1802	/*
1803	* Note we only enqueue the first of the list into the
1804	* queue, because we can always find a sibling
1805	* dependency from one (see __bfs_next()), as a result
1806	* the space of queue is saved.
1807	*/
1808	if (!first)
1809	continue;
1810
1811	first = false;
1812
1813	if (__cq_enqueue(cq, elem: entry))
1814	return BFS_EQUEUEFULL;
1815
1816	cq_depth = __cq_get_elem_count(cq);
1817	if (max_bfs_queue_depth < cq_depth)
1818	max_bfs_queue_depth = cq_depth;
1819	}
1820	}
1821
1822	return BFS_RNOMATCH;
1823	}
1824
1825	static inline enum bfs_result
1826	__bfs_forwards(struct lock_list *src_entry,
1827	void *data,
1828	bool (*match)(struct lock_list *entry, void *data),
1829	bool (*skip)(struct lock_list *entry, void *data),
1830	struct lock_list **target_entry)
1831	{
1832	return __bfs(source_entry: src_entry, data, match, skip, target_entry,
1833	offsetof(struct lock_class, locks_after));
1834
1835	}
1836
1837	static inline enum bfs_result
1838	__bfs_backwards(struct lock_list *src_entry,
1839	void *data,
1840	bool (*match)(struct lock_list *entry, void *data),
1841	bool (*skip)(struct lock_list *entry, void *data),
1842	struct lock_list **target_entry)
1843	{
1844	return __bfs(source_entry: src_entry, data, match, skip, target_entry,
1845	offsetof(struct lock_class, locks_before));
1846
1847	}
1848
1849	static void print_lock_trace(const struct lock_trace *trace,
1850	unsigned int spaces)
1851	{
1852	stack_trace_print(trace: trace->entries, nr_entries: trace->nr_entries, spaces);
1853	}
1854
1855	/*
1856	* Print a dependency chain entry (this is only done when a deadlock
1857	* has been detected):
1858	*/
1859	static noinline void
1860	print_circular_bug_entry(struct lock_list *target, int depth)
1861	{
1862	if (debug_locks_silent)
1863	return;
1864	printk("\n-> #%u", depth);
1865	print_lock_name(NULL, class: target->class);
1866	printk(KERN_CONT ":\n");
1867	print_lock_trace(trace: target->trace, spaces: 6);
1868	}
1869
1870	static void
1871	print_circular_lock_scenario(struct held_lock *src,
1872	struct held_lock *tgt,
1873	struct lock_list *prt)
1874	{
1875	struct lock_class *source = hlock_class(hlock: src);
1876	struct lock_class *target = hlock_class(hlock: tgt);
1877	struct lock_class *parent = prt->class;
1878	int src_read = src->read;
1879	int tgt_read = tgt->read;
1880
1881	/*
1882	* A direct locking problem where unsafe_class lock is taken
1883	* directly by safe_class lock, then all we need to show
1884	* is the deadlock scenario, as it is obvious that the
1885	* unsafe lock is taken under the safe lock.
1886	*
1887	* But if there is a chain instead, where the safe lock takes
1888	* an intermediate lock (middle_class) where this lock is
1889	* not the same as the safe lock, then the lock chain is
1890	* used to describe the problem. Otherwise we would need
1891	* to show a different CPU case for each link in the chain
1892	* from the safe_class lock to the unsafe_class lock.
1893	*/
1894	if (parent != source) {
1895	printk("Chain exists of:\n ");
1896	__print_lock_name(hlock: src, class: source);
1897	printk(KERN_CONT " --> ");
1898	__print_lock_name(NULL, class: parent);
1899	printk(KERN_CONT " --> ");
1900	__print_lock_name(hlock: tgt, class: target);
1901	printk(KERN_CONT "\n\n");
1902	}
1903
1904	printk(" Possible unsafe locking scenario:\n\n");
1905	printk(" CPU0 CPU1\n");
1906	printk(" ---- ----\n");
1907	if (tgt_read != 0)
1908	printk(" rlock(");
1909	else
1910	printk(" lock(");
1911	__print_lock_name(hlock: tgt, class: target);
1912	printk(KERN_CONT ");\n");
1913	printk(" lock(");
1914	__print_lock_name(NULL, class: parent);
1915	printk(KERN_CONT ");\n");
1916	printk(" lock(");
1917	__print_lock_name(hlock: tgt, class: target);
1918	printk(KERN_CONT ");\n");
1919	if (src_read != 0)
1920	printk(" rlock(");
1921	else if (src->sync)
1922	printk(" sync(");
1923	else
1924	printk(" lock(");
1925	__print_lock_name(hlock: src, class: source);
1926	printk(KERN_CONT ");\n");
1927	printk("\n *** DEADLOCK ***\n\n");
1928	}
1929
1930	/*
1931	* When a circular dependency is detected, print the
1932	* header first:
1933	*/
1934	static noinline void
1935	print_circular_bug_header(struct lock_list *entry, unsigned int depth,
1936	struct held_lock *check_src,
1937	struct held_lock *check_tgt)
1938	{
1939	struct task_struct *curr = current;
1940
1941	if (debug_locks_silent)
1942	return;
1943
1944	pr_warn("\n");
1945	pr_warn("======================================================\n");
1946	pr_warn("WARNING: possible circular locking dependency detected\n");
1947	print_kernel_ident();
1948	pr_warn("------------------------------------------------------\n");
1949	pr_warn("%s/%d is trying to acquire lock:\n",
1950	curr->comm, task_pid_nr(curr));
1951	print_lock(hlock: check_src);
1952
1953	pr_warn("\nbut task is already holding lock:\n");
1954
1955	print_lock(hlock: check_tgt);
1956	pr_warn("\nwhich lock already depends on the new lock.\n\n");
1957	pr_warn("\nthe existing dependency chain (in reverse order) is:\n");
1958
1959	print_circular_bug_entry(target: entry, depth);
1960	}
1961
1962	/*
1963	* We are about to add A -> B into the dependency graph, and in __bfs() a
1964	* strong dependency path A -> .. -> B is found: hlock_class equals
1965	* entry->class.
1966	*
1967	* If A -> .. -> B can replace A -> B in any __bfs() search (means the former
1968	* is _stronger_ than or equal to the latter), we consider A -> B as redundant.
1969	* For example if A -> .. -> B is -(EN)-> (i.e. A -(E*)-> .. -(*N)-> B), and A
1970	* -> B is -(ER)-> or -(EN)->, then we don't need to add A -> B into the
1971	* dependency graph, as any strong path ..-> A -> B ->.. we can get with
1972	* having dependency A -> B, we could already get a equivalent path ..-> A ->
1973	* .. -> B -> .. with A -> .. -> B. Therefore A -> B is redundant.
1974	*
1975	* We need to make sure both the start and the end of A -> .. -> B is not
1976	* weaker than A -> B. For the start part, please see the comment in
1977	* check_redundant(). For the end part, we need:
1978	*
1979	* Either
1980	*
1981	* a) A -> B is -(*R)-> (everything is not weaker than that)
1982	*
1983	* or
1984	*
1985	* b) A -> .. -> B is -(*N)-> (nothing is stronger than this)
1986	*
1987	*/
1988	static inline bool hlock_equal(struct lock_list *entry, void *data)
1989	{
1990	struct held_lock *hlock = (struct held_lock *)data;
1991
1992	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */
1993	(hlock->read == 2 || /* A -> B is -(*R)-> */
1994	!entry->only_xr); /* A -> .. -> B is -(*N)-> */
1995	}
1996
1997	/*
1998	* We are about to add B -> A into the dependency graph, and in __bfs() a
1999	* strong dependency path A -> .. -> B is found: hlock_class equals
2000	* entry->class.
2001	*
2002	* We will have a deadlock case (conflict) if A -> .. -> B -> A is a strong
2003	* dependency cycle, that means:
2004	*
2005	* Either
2006	*
2007	* a) B -> A is -(E*)->
2008	*
2009	* or
2010	*
2011	* b) A -> .. -> B is -(*N)-> (i.e. A -> .. -(*N)-> B)
2012	*
2013	* as then we don't have -(*R)-> -(S*)-> in the cycle.
2014	*/
2015	static inline bool hlock_conflict(struct lock_list *entry, void *data)
2016	{
2017	struct held_lock *hlock = (struct held_lock *)data;
2018
2019	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */
2020	(hlock->read == 0 || /* B -> A is -(E*)-> */
2021	!entry->only_xr); /* A -> .. -> B is -(*N)-> */
2022	}
2023
2024	static noinline void print_circular_bug(struct lock_list *this,
2025	struct lock_list *target,
2026	struct held_lock *check_src,
2027	struct held_lock *check_tgt)
2028	{
2029	struct task_struct *curr = current;
2030	struct lock_list *parent;
2031	struct lock_list *first_parent;
2032	int depth;
2033
2034	if (!debug_locks_off_graph_unlock() || debug_locks_silent)
2035	return;
2036
2037	this->trace = save_trace();
2038	if (!this->trace)
2039	return;
2040
2041	depth = get_lock_depth(child: target);
2042
2043	print_circular_bug_header(entry: target, depth, check_src, check_tgt);
2044
2045	parent = get_lock_parent(child: target);
2046	first_parent = parent;
2047
2048	while (parent) {
2049	print_circular_bug_entry(target: parent, depth: --depth);
2050	parent = get_lock_parent(child: parent);
2051	}
2052
2053	printk("\nother info that might help us debug this:\n\n");
2054	print_circular_lock_scenario(src: check_src, tgt: check_tgt,
2055	prt: first_parent);
2056
2057	lockdep_print_held_locks(p: curr);
2058
2059	printk("\nstack backtrace:\n");
2060	dump_stack();
2061	}
2062
2063	static noinline void print_bfs_bug(int ret)
2064	{
2065	if (!debug_locks_off_graph_unlock())
2066	return;
2067
2068	/*
2069	* Breadth-first-search failed, graph got corrupted?
2070	*/
2071	WARN(1, "lockdep bfs error:%d\n", ret);
2072	}
2073
2074	static bool noop_count(struct lock_list *entry, void *data)
2075	{
2076	(*(unsigned long *)data)++;
2077	return false;
2078	}
2079
2080	static unsigned long __lockdep_count_forward_deps(struct lock_list *this)
2081	{
2082	unsigned long count = 0;
2083	struct lock_list *target_entry;
2084
2085	__bfs_forwards(src_entry: this, data: (void *)&count, match: noop_count, NULL, target_entry: &target_entry);
2086
2087	return count;
2088	}
2089	unsigned long lockdep_count_forward_deps(struct lock_class *class)
2090	{
2091	unsigned long ret, flags;
2092	struct lock_list this;
2093
2094	__bfs_init_root(lock: &this, class);
2095
2096	raw_local_irq_save(flags);
2097	lockdep_lock();
2098	ret = __lockdep_count_forward_deps(this: &this);
2099	lockdep_unlock();
2100	raw_local_irq_restore(flags);
2101
2102	return ret;
2103	}
2104
2105	static unsigned long __lockdep_count_backward_deps(struct lock_list *this)
2106	{
2107	unsigned long count = 0;
2108	struct lock_list *target_entry;
2109
2110	__bfs_backwards(src_entry: this, data: (void *)&count, match: noop_count, NULL, target_entry: &target_entry);
2111
2112	return count;
2113	}
2114
2115	unsigned long lockdep_count_backward_deps(struct lock_class *class)
2116	{
2117	unsigned long ret, flags;
2118	struct lock_list this;
2119
2120	__bfs_init_root(lock: &this, class);
2121
2122	raw_local_irq_save(flags);
2123	lockdep_lock();
2124	ret = __lockdep_count_backward_deps(this: &this);
2125	lockdep_unlock();
2126	raw_local_irq_restore(flags);
2127
2128	return ret;
2129	}
2130
2131	/*
2132	* Check that the dependency graph starting at <src> can lead to
2133	* <target> or not.
2134	*/
2135	static noinline enum bfs_result
2136	check_path(struct held_lock *target, struct lock_list *src_entry,
2137	bool (*match)(struct lock_list *entry, void *data),
2138	bool (*skip)(struct lock_list *entry, void *data),
2139	struct lock_list **target_entry)
2140	{
2141	enum bfs_result ret;
2142
2143	ret = __bfs_forwards(src_entry, data: target, match, skip, target_entry);
2144
2145	if (unlikely(bfs_error(ret)))
2146	print_bfs_bug(ret);
2147
2148	return ret;
2149	}
2150
2151	static void print_deadlock_bug(struct task_struct *, struct held_lock *, struct held_lock *);
2152
2153	/*
2154	* Prove that the dependency graph starting at <src> can not
2155	* lead to <target>. If it can, there is a circle when adding
2156	* <target> -> <src> dependency.
2157	*
2158	* Print an error and return BFS_RMATCH if it does.
2159	*/
2160	static noinline enum bfs_result
2161	check_noncircular(struct held_lock *src, struct held_lock *target,
2162	struct lock_trace **const trace)
2163	{
2164	enum bfs_result ret;
2165	struct lock_list *target_entry;
2166	struct lock_list src_entry;
2167
2168	bfs_init_root(lock: &src_entry, hlock: src);
2169
2170	debug_atomic_inc(nr_cyclic_checks);
2171
2172	ret = check_path(target, src_entry: &src_entry, match: hlock_conflict, NULL, target_entry: &target_entry);
2173
2174	if (unlikely(ret == BFS_RMATCH)) {
2175	if (!*trace) {
2176	/*
2177	* If save_trace fails here, the printing might
2178	* trigger a WARN but because of the !nr_entries it
2179	* should not do bad things.
2180	*/
2181	*trace = save_trace();
2182	}
2183
2184	if (src->class_idx == target->class_idx)
2185	print_deadlock_bug(current, src, target);
2186	else
2187	print_circular_bug(this: &src_entry, target: target_entry, check_src: src, check_tgt: target);
2188	}
2189
2190	return ret;
2191	}
2192
2193	#ifdef CONFIG_TRACE_IRQFLAGS
2194
2195	/*
2196	* Forwards and backwards subgraph searching, for the purposes of
2197	* proving that two subgraphs can be connected by a new dependency
2198	* without creating any illegal irq-safe -> irq-unsafe lock dependency.
2199	*
2200	* A irq safe->unsafe deadlock happens with the following conditions:
2201	*
2202	* 1) We have a strong dependency path A -> ... -> B
2203	*
2204	* 2) and we have ENABLED_IRQ usage of B and USED_IN_IRQ usage of A, therefore
2205	* irq can create a new dependency B -> A (consider the case that a holder
2206	* of B gets interrupted by an irq whose handler will try to acquire A).
2207	*
2208	* 3) the dependency circle A -> ... -> B -> A we get from 1) and 2) is a
2209	* strong circle:
2210	*
2211	* For the usage bits of B:
2212	* a) if A -> B is -(*N)->, then B -> A could be any type, so any
2213	* ENABLED_IRQ usage suffices.
2214	* b) if A -> B is -(*R)->, then B -> A must be -(E*)->, so only
2215	* ENABLED_IRQ_*_READ usage suffices.
2216	*
2217	* For the usage bits of A:
2218	* c) if A -> B is -(E*)->, then B -> A could be any type, so any
2219	* USED_IN_IRQ usage suffices.
2220	* d) if A -> B is -(S*)->, then B -> A must be -(*N)->, so only
2221	* USED_IN_IRQ_*_READ usage suffices.
2222	*/
2223
2224	/*
2225	* There is a strong dependency path in the dependency graph: A -> B, and now
2226	* we need to decide which usage bit of A should be accumulated to detect
2227	* safe->unsafe bugs.
2228	*
2229	* Note that usage_accumulate() is used in backwards search, so ->only_xr
2230	* stands for whether A -> B only has -(S*)-> (in this case ->only_xr is true).
2231	*
2232	* As above, if only_xr is false, which means A -> B has -(E*)-> dependency
2233	* path, any usage of A should be considered. Otherwise, we should only
2234	* consider _READ usage.
2235	*/
2236	static inline bool usage_accumulate(struct lock_list *entry, void *mask)
2237	{
2238	if (!entry->only_xr)
2239	*(unsigned long *)mask |= entry->class->usage_mask;
2240	else /* Mask out _READ usage bits */
2241	*(unsigned long *)mask |= (entry->class->usage_mask & LOCKF_IRQ);
2242
2243	return false;
2244	}
2245
2246	/*
2247	* There is a strong dependency path in the dependency graph: A -> B, and now
2248	* we need to decide which usage bit of B conflicts with the usage bits of A,
2249	* i.e. which usage bit of B may introduce safe->unsafe deadlocks.
2250	*
2251	* As above, if only_xr is false, which means A -> B has -(*N)-> dependency
2252	* path, any usage of B should be considered. Otherwise, we should only
2253	* consider _READ usage.
2254	*/
2255	static inline bool usage_match(struct lock_list *entry, void *mask)
2256	{
2257	if (!entry->only_xr)
2258	return !!(entry->class->usage_mask & *(unsigned long *)mask);
2259	else /* Mask out _READ usage bits */
2260	return !!((entry->class->usage_mask & LOCKF_IRQ) & *(unsigned long *)mask);
2261	}
2262
2263	static inline bool usage_skip(struct lock_list *entry, void *mask)
2264	{
2265	if (entry->class->lock_type == LD_LOCK_NORMAL)
2266	return false;
2267
2268	/*
2269	* Skip local_lock() for irq inversion detection.
2270	*
2271	* For !RT, local_lock() is not a real lock, so it won't carry any
2272	* dependency.
2273	*
2274	* For RT, an irq inversion happens when we have lock A and B, and on
2275	* some CPU we can have:
2276	*
2277	* lock(A);
2278	* <interrupted>
2279	* lock(B);
2280	*
2281	* where lock(B) cannot sleep, and we have a dependency B -> ... -> A.
2282	*
2283	* Now we prove local_lock() cannot exist in that dependency. First we
2284	* have the observation for any lock chain L1 -> ... -> Ln, for any
2285	* 1 <= i <= n, Li.inner_wait_type <= L1.inner_wait_type, otherwise
2286	* wait context check will complain. And since B is not a sleep lock,
2287	* therefore B.inner_wait_type >= 2, and since the inner_wait_type of
2288	* local_lock() is 3, which is greater than 2, therefore there is no
2289	* way the local_lock() exists in the dependency B -> ... -> A.
2290	*
2291	* As a result, we will skip local_lock(), when we search for irq
2292	* inversion bugs.
2293	*/
2294	if (entry->class->lock_type == LD_LOCK_PERCPU &&
2295	DEBUG_LOCKS_WARN_ON(entry->class->wait_type_inner < LD_WAIT_CONFIG))
2296	return false;
2297
2298	/*
2299	* Skip WAIT_OVERRIDE for irq inversion detection -- it's not actually
2300	* a lock and only used to override the wait_type.
2301	*/
2302
2303	return true;
2304	}
2305
2306	/*
2307	* Find a node in the forwards-direction dependency sub-graph starting
2308	* at @root->class that matches @bit.
2309	*
2310	* Return BFS_MATCH if such a node exists in the subgraph, and put that node
2311	* into *@target_entry.
2312	*/
2313	static enum bfs_result
2314	find_usage_forwards(struct lock_list *root, unsigned long usage_mask,
2315	struct lock_list **target_entry)
2316	{
2317	enum bfs_result result;
2318
2319	debug_atomic_inc(nr_find_usage_forwards_checks);
2320
2321	result = __bfs_forwards(src_entry: root, data: &usage_mask, match: usage_match, skip: usage_skip, target_entry);
2322
2323	return result;
2324	}
2325
2326	/*
2327	* Find a node in the backwards-direction dependency sub-graph starting
2328	* at @root->class that matches @bit.
2329	*/
2330	static enum bfs_result
2331	find_usage_backwards(struct lock_list *root, unsigned long usage_mask,
2332	struct lock_list **target_entry)
2333	{
2334	enum bfs_result result;
2335
2336	debug_atomic_inc(nr_find_usage_backwards_checks);
2337
2338	result = __bfs_backwards(src_entry: root, data: &usage_mask, match: usage_match, skip: usage_skip, target_entry);
2339
2340	return result;
2341	}
2342
2343	static void print_lock_class_header(struct lock_class *class, int depth)
2344	{
2345	int bit;
2346
2347	printk("%*s->", depth, "");
2348	print_lock_name(NULL, class);
2349	#ifdef CONFIG_DEBUG_LOCKDEP
2350	printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));
2351	#endif
2352	printk(KERN_CONT " {\n");
2353
2354	for (bit = 0; bit < LOCK_TRACE_STATES; bit++) {
2355	if (class->usage_mask & (1 << bit)) {
2356	int len = depth;
2357
2358	len += printk("%*s %s", depth, "", usage_str[bit]);
2359	len += printk(KERN_CONT " at:\n");
2360	print_lock_trace(trace: class->usage_traces[bit], spaces: len);
2361	}
2362	}
2363	printk("%*s }\n", depth, "");
2364
2365	printk("%*s ... key at: [<%px>] %pS\n",
2366	depth, "", class->key, class->key);
2367	}
2368
2369	/*
2370	* Dependency path printing:
2371	*
2372	* After BFS we get a lock dependency path (linked via ->parent of lock_list),
2373	* printing out each lock in the dependency path will help on understanding how
2374	* the deadlock could happen. Here are some details about dependency path
2375	* printing:
2376	*
2377	* 1) A lock_list can be either forwards or backwards for a lock dependency,
2378	* for a lock dependency A -> B, there are two lock_lists:
2379	*
2380	* a) lock_list in the ->locks_after list of A, whose ->class is B and
2381	* ->links_to is A. In this case, we can say the lock_list is
2382	* "A -> B" (forwards case).
2383	*
2384	* b) lock_list in the ->locks_before list of B, whose ->class is A
2385	* and ->links_to is B. In this case, we can say the lock_list is
2386	* "B <- A" (bacwards case).
2387	*
2388	* The ->trace of both a) and b) point to the call trace where B was
2389	* acquired with A held.
2390	*
2391	* 2) A "helper" lock_list is introduced during BFS, this lock_list doesn't
2392	* represent a certain lock dependency, it only provides an initial entry
2393	* for BFS. For example, BFS may introduce a "helper" lock_list whose
2394	* ->class is A, as a result BFS will search all dependencies starting with
2395	* A, e.g. A -> B or A -> C.
2396	*
2397	* The notation of a forwards helper lock_list is like "-> A", which means
2398	* we should search the forwards dependencies starting with "A", e.g A -> B
2399	* or A -> C.
2400	*
2401	* The notation of a bacwards helper lock_list is like "<- B", which means
2402	* we should search the backwards dependencies ending with "B", e.g.
2403	* B <- A or B <- C.
2404	*/
2405
2406	/*
2407	* printk the shortest lock dependencies from @root to @leaf in reverse order.
2408	*
2409	* We have a lock dependency path as follow:
2410	*
2411	* @root @leaf
2412	* | |
2413	* V V
2414	* ->parent ->parent
2415	* | lock_list | <--------- | lock_list | ... | lock_list | <--------- | lock_list |
2416	* | -> L1 | | L1 -> L2 | ... |Ln-2 -> Ln-1| | Ln-1 -> Ln|
2417	*
2418	* , so it's natural that we start from @leaf and print every ->class and
2419	* ->trace until we reach the @root.
2420	*/
2421	static void __used
2422	print_shortest_lock_dependencies(struct lock_list *leaf,
2423	struct lock_list *root)
2424	{
2425	struct lock_list *entry = leaf;
2426	int depth;
2427
2428	/*compute depth from generated tree by BFS*/
2429	depth = get_lock_depth(child: leaf);
2430
2431	do {
2432	print_lock_class_header(class: entry->class, depth);
2433	printk("%*s ... acquired at:\n", depth, "");
2434	print_lock_trace(trace: entry->trace, spaces: 2);
2435	printk("\n");
2436
2437	if (depth == 0 && (entry != root)) {
2438	printk("lockdep:%s bad path found in chain graph\n", __func__);
2439	break;
2440	}
2441
2442	entry = get_lock_parent(child: entry);
2443	depth--;
2444	} while (entry && (depth >= 0));
2445	}
2446
2447	/*
2448	* printk the shortest lock dependencies from @leaf to @root.
2449	*
2450	* We have a lock dependency path (from a backwards search) as follow:
2451	*
2452	* @leaf @root
2453	* | |
2454	* V V
2455	* ->parent ->parent
2456	* | lock_list | ---------> | lock_list | ... | lock_list | ---------> | lock_list |
2457	* | L2 <- L1 | | L3 <- L2 | ... | Ln <- Ln-1 | | <- Ln |
2458	*
2459	* , so when we iterate from @leaf to @root, we actually print the lock
2460	* dependency path L1 -> L2 -> .. -> Ln in the non-reverse order.
2461	*
2462	* Another thing to notice here is that ->class of L2 <- L1 is L1, while the
2463	* ->trace of L2 <- L1 is the call trace of L2, in fact we don't have the call
2464	* trace of L1 in the dependency path, which is alright, because most of the
2465	* time we can figure out where L1 is held from the call trace of L2.
2466	*/
2467	static void __used
2468	print_shortest_lock_dependencies_backwards(struct lock_list *leaf,
2469	struct lock_list *root)
2470	{
2471	struct lock_list *entry = leaf;
2472	const struct lock_trace *trace = NULL;
2473	int depth;
2474
2475	/*compute depth from generated tree by BFS*/
2476	depth = get_lock_depth(child: leaf);
2477
2478	do {
2479	print_lock_class_header(class: entry->class, depth);
2480	if (trace) {
2481	printk("%*s ... acquired at:\n", depth, "");
2482	print_lock_trace(trace, spaces: 2);
2483	printk("\n");
2484	}
2485
2486	/*
2487	* Record the pointer to the trace for the next lock_list
2488	* entry, see the comments for the function.
2489	*/
2490	trace = entry->trace;
2491
2492	if (depth == 0 && (entry != root)) {
2493	printk("lockdep:%s bad path found in chain graph\n", __func__);
2494	break;
2495	}
2496
2497	entry = get_lock_parent(child: entry);
2498	depth--;
2499	} while (entry && (depth >= 0));
2500	}
2501
2502	static void
2503	print_irq_lock_scenario(struct lock_list *safe_entry,
2504	struct lock_list *unsafe_entry,
2505	struct lock_class *prev_class,
2506	struct lock_class *next_class)
2507	{
2508	struct lock_class *safe_class = safe_entry->class;
2509	struct lock_class *unsafe_class = unsafe_entry->class;
2510	struct lock_class *middle_class = prev_class;
2511
2512	if (middle_class == safe_class)
2513	middle_class = next_class;
2514
2515	/*
2516	* A direct locking problem where unsafe_class lock is taken
2517	* directly by safe_class lock, then all we need to show
2518	* is the deadlock scenario, as it is obvious that the
2519	* unsafe lock is taken under the safe lock.
2520	*
2521	* But if there is a chain instead, where the safe lock takes
2522	* an intermediate lock (middle_class) where this lock is
2523	* not the same as the safe lock, then the lock chain is
2524	* used to describe the problem. Otherwise we would need
2525	* to show a different CPU case for each link in the chain
2526	* from the safe_class lock to the unsafe_class lock.
2527	*/
2528	if (middle_class != unsafe_class) {
2529	printk("Chain exists of:\n ");
2530	__print_lock_name(NULL, class: safe_class);
2531	printk(KERN_CONT " --> ");
2532	__print_lock_name(NULL, class: middle_class);
2533	printk(KERN_CONT " --> ");
2534	__print_lock_name(NULL, class: unsafe_class);
2535	printk(KERN_CONT "\n\n");
2536	}
2537
2538	printk(" Possible interrupt unsafe locking scenario:\n\n");
2539	printk(" CPU0 CPU1\n");
2540	printk(" ---- ----\n");
2541	printk(" lock(");
2542	__print_lock_name(NULL, class: unsafe_class);
2543	printk(KERN_CONT ");\n");
2544	printk(" local_irq_disable();\n");
2545	printk(" lock(");
2546	__print_lock_name(NULL, class: safe_class);
2547	printk(KERN_CONT ");\n");
2548	printk(" lock(");
2549	__print_lock_name(NULL, class: middle_class);
2550	printk(KERN_CONT ");\n");
2551	printk(" <Interrupt>\n");
2552	printk(" lock(");
2553	__print_lock_name(NULL, class: safe_class);
2554	printk(KERN_CONT ");\n");
2555	printk("\n *** DEADLOCK ***\n\n");
2556	}
2557
2558	static void
2559	print_bad_irq_dependency(struct task_struct *curr,
2560	struct lock_list *prev_root,
2561	struct lock_list *next_root,
2562	struct lock_list *backwards_entry,
2563	struct lock_list *forwards_entry,
2564	struct held_lock *prev,
2565	struct held_lock *next,
2566	enum lock_usage_bit bit1,
2567	enum lock_usage_bit bit2,
2568	const char *irqclass)
2569	{
2570	if (!debug_locks_off_graph_unlock() || debug_locks_silent)
2571	return;
2572
2573	pr_warn("\n");
2574	pr_warn("=====================================================\n");
2575	pr_warn("WARNING: %s-safe -> %s-unsafe lock order detected\n",
2576	irqclass, irqclass);
2577	print_kernel_ident();
2578	pr_warn("-----------------------------------------------------\n");
2579	pr_warn("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] is trying to acquire:\n",
2580	curr->comm, task_pid_nr(curr),
2581	lockdep_hardirq_context(), hardirq_count() >> HARDIRQ_SHIFT,
2582	curr->softirq_context, softirq_count() >> SOFTIRQ_SHIFT,
2583	lockdep_hardirqs_enabled(),
2584	curr->softirqs_enabled);
2585	print_lock(hlock: next);
2586
2587	pr_warn("\nand this task is already holding:\n");
2588	print_lock(hlock: prev);
2589	pr_warn("which would create a new lock dependency:\n");
2590	print_lock_name(hlock: prev, class: hlock_class(hlock: prev));
2591	pr_cont(" ->");
2592	print_lock_name(hlock: next, class: hlock_class(hlock: next));
2593	pr_cont("\n");
2594
2595	pr_warn("\nbut this new dependency connects a %s-irq-safe lock:\n",
2596	irqclass);
2597	print_lock_name(NULL, class: backwards_entry->class);
2598	pr_warn("\n... which became %s-irq-safe at:\n", irqclass);
2599
2600	print_lock_trace(trace: backwards_entry->class->usage_traces[bit1], spaces: 1);
2601
2602	pr_warn("\nto a %s-irq-unsafe lock:\n", irqclass);
2603	print_lock_name(NULL, class: forwards_entry->class);
2604	pr_warn("\n... which became %s-irq-unsafe at:\n", irqclass);
2605	pr_warn("...");
2606
2607	print_lock_trace(trace: forwards_entry->class->usage_traces[bit2], spaces: 1);
2608
2609	pr_warn("\nother info that might help us debug this:\n\n");
2610	print_irq_lock_scenario(safe_entry: backwards_entry, unsafe_entry: forwards_entry,
2611	prev_class: hlock_class(hlock: prev), next_class: hlock_class(hlock: next));
2612
2613	lockdep_print_held_locks(p: curr);
2614
2615	pr_warn("\nthe dependencies between %s-irq-safe lock and the holding lock:\n", irqclass);
2616	print_shortest_lock_dependencies_backwards(leaf: backwards_entry, root: prev_root);
2617
2618	pr_warn("\nthe dependencies between the lock to be acquired");
2619	pr_warn(" and %s-irq-unsafe lock:\n", irqclass);
2620	next_root->trace = save_trace();
2621	if (!next_root->trace)
2622	return;
2623	print_shortest_lock_dependencies(leaf: forwards_entry, root: next_root);
2624
2625	pr_warn("\nstack backtrace:\n");
2626	dump_stack();
2627	}
2628
2629	static const char *state_names[] = {
2630	#define LOCKDEP_STATE(__STATE) \
2631	__stringify(__STATE),
2632	#include "lockdep_states.h"
2633	#undef LOCKDEP_STATE
2634	};
2635
2636	static const char *state_rnames[] = {
2637	#define LOCKDEP_STATE(__STATE) \
2638	__stringify(__STATE)"-READ",
2639	#include "lockdep_states.h"
2640	#undef LOCKDEP_STATE
2641	};
2642
2643	static inline const char *state_name(enum lock_usage_bit bit)
2644	{
2645	if (bit & LOCK_USAGE_READ_MASK)
2646	return state_rnames[bit >> LOCK_USAGE_DIR_MASK];
2647	else
2648	return state_names[bit >> LOCK_USAGE_DIR_MASK];
2649	}
2650
2651	/*
2652	* The bit number is encoded like:
2653	*
2654	* bit0: 0 exclusive, 1 read lock
2655	* bit1: 0 used in irq, 1 irq enabled
2656	* bit2-n: state
2657	*/
2658	static int exclusive_bit(int new_bit)
2659	{
2660	int state = new_bit & LOCK_USAGE_STATE_MASK;
2661	int dir = new_bit & LOCK_USAGE_DIR_MASK;
2662
2663	/*
2664	* keep state, bit flip the direction and strip read.
2665	*/
2666	return state | (dir ^ LOCK_USAGE_DIR_MASK);
2667	}
2668
2669	/*
2670	* Observe that when given a bitmask where each bitnr is encoded as above, a
2671	* right shift of the mask transforms the individual bitnrs as -1 and
2672	* conversely, a left shift transforms into +1 for the individual bitnrs.
2673	*
2674	* So for all bits whose number have LOCK_ENABLED_* set (bitnr1 == 1), we can
2675	* create the mask with those bit numbers using LOCK_USED_IN_* (bitnr1 == 0)
2676	* instead by subtracting the bit number by 2, or shifting the mask right by 2.
2677	*
2678	* Similarly, bitnr1 == 0 becomes bitnr1 == 1 by adding 2, or shifting left 2.
2679	*
2680	* So split the mask (note that LOCKF_ENABLED_IRQ_ALL|LOCKF_USED_IN_IRQ_ALL is
2681	* all bits set) and recompose with bitnr1 flipped.
2682	*/
2683	static unsigned long invert_dir_mask(unsigned long mask)
2684	{
2685	unsigned long excl = 0;
2686
2687	/* Invert dir */
2688	excl |= (mask & LOCKF_ENABLED_IRQ_ALL) >> LOCK_USAGE_DIR_MASK;
2689	excl |= (mask & LOCKF_USED_IN_IRQ_ALL) << LOCK_USAGE_DIR_MASK;
2690
2691	return excl;
2692	}
2693
2694	/*
2695	* Note that a LOCK_ENABLED_IRQ_*_READ usage and a LOCK_USED_IN_IRQ_*_READ
2696	* usage may cause deadlock too, for example:
2697	*
2698	* P1 P2
2699	* <irq disabled>
2700	* write_lock(l1); <irq enabled>
2701	* read_lock(l2);
2702	* write_lock(l2);
2703	* <in irq>
2704	* read_lock(l1);
2705	*
2706	* , in above case, l1 will be marked as LOCK_USED_IN_IRQ_HARDIRQ_READ and l2
2707	* will marked as LOCK_ENABLE_IRQ_HARDIRQ_READ, and this is a possible
2708	* deadlock.
2709	*
2710	* In fact, all of the following cases may cause deadlocks:
2711	*
2712	* LOCK_USED_IN_IRQ_* -> LOCK_ENABLED_IRQ_*
2713	* LOCK_USED_IN_IRQ_*_READ -> LOCK_ENABLED_IRQ_*
2714	* LOCK_USED_IN_IRQ_* -> LOCK_ENABLED_IRQ_*_READ
2715	* LOCK_USED_IN_IRQ_*_READ -> LOCK_ENABLED_IRQ_*_READ
2716	*
2717	* As a result, to calculate the "exclusive mask", first we invert the
2718	* direction (USED_IN/ENABLED) of the original mask, and 1) for all bits with
2719	* bitnr0 set (LOCK_*_READ), add those with bitnr0 cleared (LOCK_*). 2) for all
2720	* bits with bitnr0 cleared (LOCK_*_READ), add those with bitnr0 set (LOCK_*).
2721	*/
2722	static unsigned long exclusive_mask(unsigned long mask)
2723	{
2724	unsigned long excl = invert_dir_mask(mask);
2725
2726	excl |= (excl & LOCKF_IRQ_READ) >> LOCK_USAGE_READ_MASK;
2727	excl |= (excl & LOCKF_IRQ) << LOCK_USAGE_READ_MASK;
2728
2729	return excl;
2730	}
2731
2732	/*
2733	* Retrieve the _possible_ original mask to which @mask is
2734	* exclusive. Ie: this is the opposite of exclusive_mask().
2735	* Note that 2 possible original bits can match an exclusive
2736	* bit: one has LOCK_USAGE_READ_MASK set, the other has it
2737	* cleared. So both are returned for each exclusive bit.
2738	*/
2739	static unsigned long original_mask(unsigned long mask)
2740	{
2741	unsigned long excl = invert_dir_mask(mask);
2742
2743	/* Include read in existing usages */
2744	excl |= (excl & LOCKF_IRQ_READ) >> LOCK_USAGE_READ_MASK;
2745	excl |= (excl & LOCKF_IRQ) << LOCK_USAGE_READ_MASK;
2746
2747	return excl;
2748	}
2749
2750	/*
2751	* Find the first pair of bit match between an original
2752	* usage mask and an exclusive usage mask.
2753	*/
2754	static int find_exclusive_match(unsigned long mask,
2755	unsigned long excl_mask,
2756	enum lock_usage_bit *bitp,
2757	enum lock_usage_bit *excl_bitp)
2758	{
2759	int bit, excl, excl_read;
2760
2761	for_each_set_bit(bit, &mask, LOCK_USED) {
2762	/*
2763	* exclusive_bit() strips the read bit, however,
2764	* LOCK_ENABLED_IRQ_*_READ may cause deadlocks too, so we need
2765	* to search excl | LOCK_USAGE_READ_MASK as well.
2766	*/
2767	excl = exclusive_bit(new_bit: bit);
2768	excl_read = excl | LOCK_USAGE_READ_MASK;
2769	if (excl_mask & lock_flag(bit: excl)) {
2770	*bitp = bit;
2771	*excl_bitp = excl;
2772	return 0;
2773	} else if (excl_mask & lock_flag(bit: excl_read)) {
2774	*bitp = bit;
2775	*excl_bitp = excl_read;
2776	return 0;
2777	}
2778	}
2779	return -1;
2780	}
2781
2782	/*
2783	* Prove that the new dependency does not connect a hardirq-safe(-read)
2784	* lock with a hardirq-unsafe lock - to achieve this we search
2785	* the backwards-subgraph starting at <prev>, and the
2786	* forwards-subgraph starting at <next>:
2787	*/
2788	static int check_irq_usage(struct task_struct *curr, struct held_lock *prev,
2789	struct held_lock *next)
2790	{
2791	unsigned long usage_mask = 0, forward_mask, backward_mask;
2792	enum lock_usage_bit forward_bit = 0, backward_bit = 0;
2793	struct lock_list *target_entry1;
2794	struct lock_list *target_entry;
2795	struct lock_list this, that;
2796	enum bfs_result ret;
2797
2798	/*
2799	* Step 1: gather all hard/soft IRQs usages backward in an
2800	* accumulated usage mask.
2801	*/
2802	bfs_init_rootb(lock: &this, hlock: prev);
2803
2804	ret = __bfs_backwards(src_entry: &this, data: &usage_mask, match: usage_accumulate, skip: usage_skip, NULL);
2805	if (bfs_error(res: ret)) {
2806	print_bfs_bug(ret);
2807	return 0;
2808	}
2809
2810	usage_mask &= LOCKF_USED_IN_IRQ_ALL;
2811	if (!usage_mask)
2812	return 1;
2813
2814	/*
2815	* Step 2: find exclusive uses forward that match the previous
2816	* backward accumulated mask.
2817	*/
2818	forward_mask = exclusive_mask(mask: usage_mask);
2819
2820	bfs_init_root(lock: &that, hlock: next);
2821
2822	ret = find_usage_forwards(root: &that, usage_mask: forward_mask, target_entry: &target_entry1);
2823	if (bfs_error(res: ret)) {
2824	print_bfs_bug(ret);
2825	return 0;
2826	}
2827	if (ret == BFS_RNOMATCH)
2828	return 1;
2829
2830	/*
2831	* Step 3: we found a bad match! Now retrieve a lock from the backward
2832	* list whose usage mask matches the exclusive usage mask from the
2833	* lock found on the forward list.
2834	*
2835	* Note, we should only keep the LOCKF_ENABLED_IRQ_ALL bits, considering
2836	* the follow case:
2837	*
2838	* When trying to add A -> B to the graph, we find that there is a
2839	* hardirq-safe L, that L -> ... -> A, and another hardirq-unsafe M,
2840	* that B -> ... -> M. However M is **softirq-safe**, if we use exact
2841	* invert bits of M's usage_mask, we will find another lock N that is
2842	* **softirq-unsafe** and N -> ... -> A, however N -> .. -> M will not
2843	* cause a inversion deadlock.
2844	*/
2845	backward_mask = original_mask(mask: target_entry1->class->usage_mask & LOCKF_ENABLED_IRQ_ALL);
2846
2847	ret = find_usage_backwards(root: &this, usage_mask: backward_mask, target_entry: &target_entry);
2848	if (bfs_error(res: ret)) {
2849	print_bfs_bug(ret);
2850	return 0;
2851	}
2852	if (DEBUG_LOCKS_WARN_ON(ret == BFS_RNOMATCH))
2853	return 1;
2854
2855	/*
2856	* Step 4: narrow down to a pair of incompatible usage bits
2857	* and report it.
2858	*/
2859	ret = find_exclusive_match(mask: target_entry->class->usage_mask,
2860	excl_mask: target_entry1->class->usage_mask,
2861	bitp: &backward_bit, excl_bitp: &forward_bit);
2862	if (DEBUG_LOCKS_WARN_ON(ret == -1))
2863	return 1;
2864
2865	print_bad_irq_dependency(curr, prev_root: &this, next_root: &that,
2866	backwards_entry: target_entry, forwards_entry: target_entry1,
2867	prev, next,
2868	bit1: backward_bit, bit2: forward_bit,
2869	irqclass: state_name(bit: backward_bit));
2870
2871	return 0;
2872	}
2873
2874	#else
2875
2876	static inline int check_irq_usage(struct task_struct *curr,
2877	struct held_lock *prev, struct held_lock *next)
2878	{
2879	return 1;
2880	}
2881
2882	static inline bool usage_skip(struct lock_list *entry, void *mask)
2883	{
2884	return false;
2885	}
2886
2887	#endif /* CONFIG_TRACE_IRQFLAGS */
2888
2889	#ifdef CONFIG_LOCKDEP_SMALL
2890	/*
2891	* Check that the dependency graph starting at <src> can lead to
2892	* <target> or not. If it can, <src> -> <target> dependency is already
2893	* in the graph.
2894	*
2895	* Return BFS_RMATCH if it does, or BFS_RNOMATCH if it does not, return BFS_E* if
2896	* any error appears in the bfs search.
2897	*/
2898	static noinline enum bfs_result
2899	check_redundant(struct held_lock *src, struct held_lock *target)
2900	{
2901	enum bfs_result ret;
2902	struct lock_list *target_entry;
2903	struct lock_list src_entry;
2904
2905	bfs_init_root(&src_entry, src);
2906	/*
2907	* Special setup for check_redundant().
2908	*
2909	* To report redundant, we need to find a strong dependency path that
2910	* is equal to or stronger than <src> -> <target>. So if <src> is E,
2911	* we need to let __bfs() only search for a path starting at a -(E*)->,
2912	* we achieve this by setting the initial node's ->only_xr to true in
2913	* that case. And if <prev> is S, we set initial ->only_xr to false
2914	* because both -(S*)-> (equal) and -(E*)-> (stronger) are redundant.
2915	*/
2916	src_entry.only_xr = src->read == 0;
2917
2918	debug_atomic_inc(nr_redundant_checks);
2919
2920	/*
2921	* Note: we skip local_lock() for redundant check, because as the
2922	* comment in usage_skip(), A -> local_lock() -> B and A -> B are not
2923	* the same.
2924	*/
2925	ret = check_path(target, &src_entry, hlock_equal, usage_skip, &target_entry);
2926
2927	if (ret == BFS_RMATCH)
2928	debug_atomic_inc(nr_redundant);
2929
2930	return ret;
2931	}
2932
2933	#else
2934
2935	static inline enum bfs_result
2936	check_redundant(struct held_lock *src, struct held_lock *target)
2937	{
2938	return BFS_RNOMATCH;
2939	}
2940
2941	#endif
2942
2943	static void inc_chains(int irq_context)
2944	{
2945	if (irq_context & LOCK_CHAIN_HARDIRQ_CONTEXT)
2946	nr_hardirq_chains++;
2947	else if (irq_context & LOCK_CHAIN_SOFTIRQ_CONTEXT)
2948	nr_softirq_chains++;
2949	else
2950	nr_process_chains++;
2951	}
2952
2953	static void dec_chains(int irq_context)
2954	{
2955	if (irq_context & LOCK_CHAIN_HARDIRQ_CONTEXT)
2956	nr_hardirq_chains--;
2957	else if (irq_context & LOCK_CHAIN_SOFTIRQ_CONTEXT)
2958	nr_softirq_chains--;
2959	else
2960	nr_process_chains--;
2961	}
2962
2963	static void
2964	print_deadlock_scenario(struct held_lock *nxt, struct held_lock *prv)
2965	{
2966	struct lock_class *next = hlock_class(hlock: nxt);
2967	struct lock_class *prev = hlock_class(hlock: prv);
2968
2969	printk(" Possible unsafe locking scenario:\n\n");
2970	printk(" CPU0\n");
2971	printk(" ----\n");
2972	printk(" lock(");
2973	__print_lock_name(hlock: prv, class: prev);
2974	printk(KERN_CONT ");\n");
2975	printk(" lock(");
2976	__print_lock_name(hlock: nxt, class: next);
2977	printk(KERN_CONT ");\n");
2978	printk("\n *** DEADLOCK ***\n\n");
2979	printk(" May be due to missing lock nesting notation\n\n");
2980	}
2981
2982	static void
2983	print_deadlock_bug(struct task_struct *curr, struct held_lock *prev,
2984	struct held_lock *next)
2985	{
2986	struct lock_class *class = hlock_class(hlock: prev);
2987
2988	if (!debug_locks_off_graph_unlock() || debug_locks_silent)
2989	return;
2990
2991	pr_warn("\n");
2992	pr_warn("============================================\n");
2993	pr_warn("WARNING: possible recursive locking detected\n");
2994	print_kernel_ident();
2995	pr_warn("--------------------------------------------\n");
2996	pr_warn("%s/%d is trying to acquire lock:\n",
2997	curr->comm, task_pid_nr(curr));
2998	print_lock(hlock: next);
2999	pr_warn("\nbut task is already holding lock:\n");
3000	print_lock(hlock: prev);
3001
3002	if (class->cmp_fn) {
3003	pr_warn("and the lock comparison function returns %i:\n",
3004	class->cmp_fn(prev->instance, next->instance));
3005	}
3006
3007	pr_warn("\nother info that might help us debug this:\n");
3008	print_deadlock_scenario(nxt: next, prv: prev);
3009	lockdep_print_held_locks(p: curr);
3010
3011	pr_warn("\nstack backtrace:\n");
3012	dump_stack();
3013	}
3014
3015	/*
3016	* Check whether we are holding such a class already.
3017	*
3018	* (Note that this has to be done separately, because the graph cannot
3019	* detect such classes of deadlocks.)
3020	*
3021	* Returns: 0 on deadlock detected, 1 on OK, 2 if another lock with the same
3022	* lock class is held but nest_lock is also held, i.e. we rely on the
3023	* nest_lock to avoid the deadlock.
3024	*/
3025	static int
3026	check_deadlock(struct task_struct *curr, struct held_lock *next)
3027	{
3028	struct lock_class *class;
3029	struct held_lock *prev;
3030	struct held_lock *nest = NULL;
3031	int i;
3032
3033	for (i = 0; i < curr->lockdep_depth; i++) {
3034	prev = curr->held_locks + i;
3035
3036	if (prev->instance == next->nest_lock)
3037	nest = prev;
3038
3039	if (hlock_class(hlock: prev) != hlock_class(hlock: next))
3040	continue;
3041
3042	/*
3043	* Allow read-after-read recursion of the same
3044	* lock class (i.e. read_lock(lock)+read_lock(lock)):
3045	*/
3046	if ((next->read == 2) && prev->read)
3047	continue;
3048
3049	class = hlock_class(hlock: prev);
3050
3051	if (class->cmp_fn &&
3052	class->cmp_fn(prev->instance, next->instance) < 0)
3053	continue;
3054
3055	/*
3056	* We're holding the nest_lock, which serializes this lock's
3057	* nesting behaviour.
3058	*/
3059	if (nest)
3060	return 2;
3061
3062	print_deadlock_bug(curr, prev, next);
3063	return 0;
3064	}
3065	return 1;
3066	}
3067
3068	/*
3069	* There was a chain-cache miss, and we are about to add a new dependency
3070	* to a previous lock. We validate the following rules:
3071	*
3072	* - would the adding of the <prev> -> <next> dependency create a
3073	* circular dependency in the graph? [== circular deadlock]
3074	*
3075	* - does the new prev->next dependency connect any hardirq-safe lock
3076	* (in the full backwards-subgraph starting at <prev>) with any
3077	* hardirq-unsafe lock (in the full forwards-subgraph starting at
3078	* <next>)? [== illegal lock inversion with hardirq contexts]
3079	*
3080	* - does the new prev->next dependency connect any softirq-safe lock
3081	* (in the full backwards-subgraph starting at <prev>) with any
3082	* softirq-unsafe lock (in the full forwards-subgraph starting at
3083	* <next>)? [== illegal lock inversion with softirq contexts]
3084	*
3085	* any of these scenarios could lead to a deadlock.
3086	*
3087	* Then if all the validations pass, we add the forwards and backwards
3088	* dependency.
3089	*/
3090	static int
3091	check_prev_add(struct task_struct *curr, struct held_lock *prev,
3092	struct held_lock *next, u16 distance,
3093	struct lock_trace **const trace)
3094	{
3095	struct lock_list *entry;
3096	enum bfs_result ret;
3097
3098	if (!hlock_class(hlock: prev)->key || !hlock_class(hlock: next)->key) {
3099	/*
3100	* The warning statements below may trigger a use-after-free
3101	* of the class name. It is better to trigger a use-after free
3102	* and to have the class name most of the time instead of not
3103	* having the class name available.
3104	*/
3105	WARN_ONCE(!debug_locks_silent && !hlock_class(prev)->key,
3106	"Detected use-after-free of lock class %px/%s\n",
3107	hlock_class(prev),
3108	hlock_class(prev)->name);
3109	WARN_ONCE(!debug_locks_silent && !hlock_class(next)->key,
3110	"Detected use-after-free of lock class %px/%s\n",
3111	hlock_class(next),
3112	hlock_class(next)->name);
3113	return 2;
3114	}
3115
3116	if (prev->class_idx == next->class_idx) {
3117	struct lock_class *class = hlock_class(hlock: prev);
3118
3119	if (class->cmp_fn &&
3120	class->cmp_fn(prev->instance, next->instance) < 0)
3121	return 2;
3122	}
3123
3124	/*
3125	* Prove that the new <prev> -> <next> dependency would not
3126	* create a circular dependency in the graph. (We do this by
3127	* a breadth-first search into the graph starting at <next>,
3128	* and check whether we can reach <prev>.)
3129	*
3130	* The search is limited by the size of the circular queue (i.e.,
3131	* MAX_CIRCULAR_QUEUE_SIZE) which keeps track of a breadth of nodes
3132	* in the graph whose neighbours are to be checked.
3133	*/
3134	ret = check_noncircular(src: next, target: prev, trace);
3135	if (unlikely(bfs_error(ret) || ret == BFS_RMATCH))
3136	return 0;
3137
3138	if (!check_irq_usage(curr, prev, next))
3139	return 0;
3140
3141	/*
3142	* Is the <prev> -> <next> dependency already present?
3143	*
3144	* (this may occur even though this is a new chain: consider
3145	* e.g. the L1 -> L2 -> L3 -> L4 and the L5 -> L1 -> L2 -> L3
3146	* chains - the second one will be new, but L1 already has
3147	* L2 added to its dependency list, due to the first chain.)
3148	*/
3149	list_for_each_entry(entry, &hlock_class(prev)->locks_after, entry) {
3150	if (entry->class == hlock_class(hlock: next)) {
3151	if (distance == 1)
3152	entry->distance = 1;
3153	entry->dep |= calc_dep(prev, next);
3154
3155	/*
3156	* Also, update the reverse dependency in @next's
3157	* ->locks_before list.
3158	*
3159	* Here we reuse @entry as the cursor, which is fine
3160	* because we won't go to the next iteration of the
3161	* outer loop:
3162	*
3163	* For normal cases, we return in the inner loop.
3164	*
3165	* If we fail to return, we have inconsistency, i.e.
3166	* <prev>::locks_after contains <next> while
3167	* <next>::locks_before doesn't contain <prev>. In
3168	* that case, we return after the inner and indicate
3169	* something is wrong.
3170	*/
3171	list_for_each_entry(entry, &hlock_class(next)->locks_before, entry) {
3172	if (entry->class == hlock_class(hlock: prev)) {
3173	if (distance == 1)
3174	entry->distance = 1;
3175	entry->dep |= calc_depb(prev, next);
3176	return 1;
3177	}
3178	}
3179
3180	/* <prev> is not found in <next>::locks_before */
3181	return 0;
3182	}
3183	}
3184
3185	/*
3186	* Is the <prev> -> <next> link redundant?
3187	*/
3188	ret = check_redundant(src: prev, target: next);
3189	if (bfs_error(res: ret))
3190	return 0;
3191	else if (ret == BFS_RMATCH)
3192	return 2;
3193
3194	if (!*trace) {
3195	*trace = save_trace();
3196	if (!*trace)
3197	return 0;
3198	}
3199
3200	/*
3201	* Ok, all validations passed, add the new lock
3202	* to the previous lock's dependency list:
3203	*/
3204	ret = add_lock_to_list(this: hlock_class(hlock: next), links_to: hlock_class(hlock: prev),
3205	head: &hlock_class(hlock: prev)->locks_after, distance,
3206	dep: calc_dep(prev, next), trace: *trace);
3207
3208	if (!ret)
3209	return 0;
3210
3211	ret = add_lock_to_list(this: hlock_class(hlock: prev), links_to: hlock_class(hlock: next),
3212	head: &hlock_class(hlock: next)->locks_before, distance,
3213	dep: calc_depb(prev, next), trace: *trace);
3214	if (!ret)
3215	return 0;
3216
3217	return 2;
3218	}
3219
3220	/*
3221	* Add the dependency to all directly-previous locks that are 'relevant'.
3222	* The ones that are relevant are (in increasing distance from curr):
3223	* all consecutive trylock entries and the final non-trylock entry - or
3224	* the end of this context's lock-chain - whichever comes first.
3225	*/
3226	static int
3227	check_prevs_add(struct task_struct *curr, struct held_lock *next)
3228	{
3229	struct lock_trace *trace = NULL;
3230	int depth = curr->lockdep_depth;
3231	struct held_lock *hlock;
3232
3233	/*
3234	* Debugging checks.
3235	*
3236	* Depth must not be zero for a non-head lock:
3237	*/
3238	if (!depth)
3239	goto out_bug;
3240	/*
3241	* At least two relevant locks must exist for this
3242	* to be a head:
3243	*/
3244	if (curr->held_locks[depth].irq_context !=
3245	curr->held_locks[depth-1].irq_context)
3246	goto out_bug;
3247
3248	for (;;) {
3249	u16 distance = curr->lockdep_depth - depth + 1;
3250	hlock = curr->held_locks + depth - 1;
3251
3252	if (hlock->check) {
3253	int ret = check_prev_add(curr, prev: hlock, next, distance, trace: &trace);
3254	if (!ret)
3255	return 0;
3256
3257	/*
3258	* Stop after the first non-trylock entry,
3259	* as non-trylock entries have added their
3260	* own direct dependencies already, so this
3261	* lock is connected to them indirectly:
3262	*/
3263	if (!hlock->trylock)
3264	break;
3265	}
3266
3267	depth--;
3268	/*
3269	* End of lock-stack?
3270	*/
3271	if (!depth)
3272	break;
3273	/*
3274	* Stop the search if we cross into another context:
3275	*/
3276	if (curr->held_locks[depth].irq_context !=
3277	curr->held_locks[depth-1].irq_context)
3278	break;
3279	}
3280	return 1;
3281	out_bug:
3282	if (!debug_locks_off_graph_unlock())
3283	return 0;
3284
3285	/*
3286	* Clearly we all shouldn't be here, but since we made it we
3287	* can reliable say we messed up our state. See the above two
3288	* gotos for reasons why we could possibly end up here.
3289	*/
3290	WARN_ON(1);
3291
3292	return 0;
3293	}
3294
3295	struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
3296	static DECLARE_BITMAP(lock_chains_in_use, MAX_LOCKDEP_CHAINS);
3297	static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS];
3298	unsigned long nr_zapped_lock_chains;
3299	unsigned int nr_free_chain_hlocks; /* Free chain_hlocks in buckets */
3300	unsigned int nr_lost_chain_hlocks; /* Lost chain_hlocks */
3301	unsigned int nr_large_chain_blocks; /* size > MAX_CHAIN_BUCKETS */
3302
3303	/*
3304	* The first 2 chain_hlocks entries in the chain block in the bucket
3305	* list contains the following meta data:
3306	*
3307	* entry[0]:
3308	* Bit 15 - always set to 1 (it is not a class index)
3309	* Bits 0-14 - upper 15 bits of the next block index
3310	* entry[1] - lower 16 bits of next block index
3311	*
3312	* A next block index of all 1 bits means it is the end of the list.
3313	*
3314	* On the unsized bucket (bucket-0), the 3rd and 4th entries contain
3315	* the chain block size:
3316	*
3317	* entry[2] - upper 16 bits of the chain block size
3318	* entry[3] - lower 16 bits of the chain block size
3319	*/
3320	#define MAX_CHAIN_BUCKETS 16
3321	#define CHAIN_BLK_FLAG (1U << 15)
3322	#define CHAIN_BLK_LIST_END 0xFFFFU
3323
3324	static int chain_block_buckets[MAX_CHAIN_BUCKETS];
3325
3326	static inline int size_to_bucket(int size)
3327	{
3328	if (size > MAX_CHAIN_BUCKETS)
3329	return 0;
3330
3331	return size - 1;
3332	}
3333
3334	/*
3335	* Iterate all the chain blocks in a bucket.
3336	*/
3337	#define for_each_chain_block(bucket, prev, curr) \
3338	for ((prev) = -1, (curr) = chain_block_buckets[bucket]; \
3339	(curr) >= 0; \
3340	(prev) = (curr), (curr) = chain_block_next(curr))
3341
3342	/*
3343	* next block or -1
3344	*/
3345	static inline int chain_block_next(int offset)
3346	{
3347	int next = chain_hlocks[offset];
3348
3349	WARN_ON_ONCE(!(next & CHAIN_BLK_FLAG));
3350
3351	if (next == CHAIN_BLK_LIST_END)
3352	return -1;
3353
3354	next &= ~CHAIN_BLK_FLAG;
3355	next <<= 16;
3356	next |= chain_hlocks[offset + 1];
3357
3358	return next;
3359	}
3360
3361	/*
3362	* bucket-0 only
3363	*/
3364	static inline int chain_block_size(int offset)
3365	{
3366	return (chain_hlocks[offset + 2] << 16) | chain_hlocks[offset + 3];
3367	}
3368
3369	static inline void init_chain_block(int offset, int next, int bucket, int size)
3370	{
3371	chain_hlocks[offset] = (next >> 16) | CHAIN_BLK_FLAG;
3372	chain_hlocks[offset + 1] = (u16)next;
3373
3374	if (size && !bucket) {
3375	chain_hlocks[offset + 2] = size >> 16;
3376	chain_hlocks[offset + 3] = (u16)size;
3377	}
3378	}
3379
3380	static inline void add_chain_block(int offset, int size)
3381	{
3382	int bucket = size_to_bucket(size);
3383	int next = chain_block_buckets[bucket];
3384	int prev, curr;
3385
3386	if (unlikely(size < 2)) {
3387	/*
3388	* We can't store single entries on the freelist. Leak them.
3389	*
3390	* One possible way out would be to uniquely mark them, other
3391	* than with CHAIN_BLK_FLAG, such that we can recover them when
3392	* the block before it is re-added.
3393	*/
3394	if (size)
3395	nr_lost_chain_hlocks++;
3396	return;
3397	}
3398
3399	nr_free_chain_hlocks += size;
3400	if (!bucket) {
3401	nr_large_chain_blocks++;
3402
3403	/*
3404	* Variable sized, sort large to small.
3405	*/
3406	for_each_chain_block(0, prev, curr) {
3407	if (size >= chain_block_size(offset: curr))
3408	break;
3409	}
3410	init_chain_block(offset, next: curr, bucket: 0, size);
3411	if (prev < 0)
3412	chain_block_buckets[0] = offset;
3413	else
3414	init_chain_block(offset: prev, next: offset, bucket: 0, size: 0);
3415	return;
3416	}
3417	/*
3418	* Fixed size, add to head.
3419	*/
3420	init_chain_block(offset, next, bucket, size);
3421	chain_block_buckets[bucket] = offset;
3422	}
3423
3424	/*
3425	* Only the first block in the list can be deleted.
3426	*
3427	* For the variable size bucket[0], the first block (the largest one) is
3428	* returned, broken up and put back into the pool. So if a chain block of
3429	* length > MAX_CHAIN_BUCKETS is ever used and zapped, it will just be
3430	* queued up after the primordial chain block and never be used until the
3431	* hlock entries in the primordial chain block is almost used up. That
3432	* causes fragmentation and reduce allocation efficiency. That can be
3433	* monitored by looking at the "large chain blocks" number in lockdep_stats.
3434	*/
3435	static inline void del_chain_block(int bucket, int size, int next)
3436	{
3437	nr_free_chain_hlocks -= size;
3438	chain_block_buckets[bucket] = next;
3439
3440	if (!bucket)
3441	nr_large_chain_blocks--;
3442	}
3443
3444	static void init_chain_block_buckets(void)
3445	{
3446	int i;
3447
3448	for (i = 0; i < MAX_CHAIN_BUCKETS; i++)
3449	chain_block_buckets[i] = -1;
3450
3451	add_chain_block(offset: 0, ARRAY_SIZE(chain_hlocks));
3452	}
3453
3454	/*
3455	* Return offset of a chain block of the right size or -1 if not found.
3456	*
3457	* Fairly simple worst-fit allocator with the addition of a number of size
3458	* specific free lists.
3459	*/
3460	static int alloc_chain_hlocks(int req)
3461	{
3462	int bucket, curr, size;
3463
3464	/*
3465	* We rely on the MSB to act as an escape bit to denote freelist
3466	* pointers. Make sure this bit isn't set in 'normal' class_idx usage.
3467	*/
3468	BUILD_BUG_ON((MAX_LOCKDEP_KEYS-1) & CHAIN_BLK_FLAG);
3469
3470	init_data_structures_once();
3471
3472	if (nr_free_chain_hlocks < req)
3473	return -1;
3474
3475	/*
3476	* We require a minimum of 2 (u16) entries to encode a freelist
3477	* 'pointer'.
3478	*/
3479	req = max(req, 2);
3480	bucket = size_to_bucket(size: req);
3481	curr = chain_block_buckets[bucket];
3482
3483	if (bucket) {
3484	if (curr >= 0) {
3485	del_chain_block(bucket, size: req, next: chain_block_next(offset: curr));
3486	return curr;
3487	}
3488	/* Try bucket 0 */
3489	curr = chain_block_buckets[0];
3490	}
3491
3492	/*
3493	* The variable sized freelist is sorted by size; the first entry is
3494	* the largest. Use it if it fits.
3495	*/
3496	if (curr >= 0) {
3497	size = chain_block_size(offset: curr);
3498	if (likely(size >= req)) {
3499	del_chain_block(bucket: 0, size, next: chain_block_next(offset: curr));
3500	if (size > req)
3501	add_chain_block(offset: curr + req, size: size - req);
3502	return curr;
3503	}
3504	}
3505
3506	/*
3507	* Last resort, split a block in a larger sized bucket.
3508	*/
3509	for (size = MAX_CHAIN_BUCKETS; size > req; size--) {
3510	bucket = size_to_bucket(size);
3511	curr = chain_block_buckets[bucket];
3512	if (curr < 0)
3513	continue;
3514
3515	del_chain_block(bucket, size, next: chain_block_next(offset: curr));
3516	add_chain_block(offset: curr + req, size: size - req);
3517	return curr;
3518	}
3519
3520	return -1;
3521	}
3522
3523	static inline void free_chain_hlocks(int base, int size)
3524	{
3525	add_chain_block(offset: base, max(size, 2));
3526	}
3527
3528	struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i)
3529	{
3530	u16 chain_hlock = chain_hlocks[chain->base + i];
3531	unsigned int class_idx = chain_hlock_class_idx(hlock_id: chain_hlock);
3532
3533	return lock_classes + class_idx;
3534	}
3535
3536	/*
3537	* Returns the index of the first held_lock of the current chain
3538	*/
3539	static inline int get_first_held_lock(struct task_struct *curr,
3540	struct held_lock *hlock)
3541	{
3542	int i;
3543	struct held_lock *hlock_curr;
3544
3545	for (i = curr->lockdep_depth - 1; i >= 0; i--) {
3546	hlock_curr = curr->held_locks + i;
3547	if (hlock_curr->irq_context != hlock->irq_context)
3548	break;
3549
3550	}
3551
3552	return ++i;
3553	}
3554
3555	#ifdef CONFIG_DEBUG_LOCKDEP
3556	/*
3557	* Returns the next chain_key iteration
3558	*/
3559	static u64 print_chain_key_iteration(u16 hlock_id, u64 chain_key)
3560	{
3561	u64 new_chain_key = iterate_chain_key(key: chain_key, idx: hlock_id);
3562
3563	printk(" hlock_id:%d -> chain_key:%016Lx",
3564	(unsigned int)hlock_id,
3565	(unsigned long long)new_chain_key);
3566	return new_chain_key;
3567	}
3568
3569	static void
3570	print_chain_keys_held_locks(struct task_struct *curr, struct held_lock *hlock_next)
3571	{
3572	struct held_lock *hlock;
3573	u64 chain_key = INITIAL_CHAIN_KEY;
3574	int depth = curr->lockdep_depth;
3575	int i = get_first_held_lock(curr, hlock: hlock_next);
3576
3577	printk("depth: %u (irq_context %u)\n", depth - i + 1,
3578	hlock_next->irq_context);
3579	for (; i < depth; i++) {
3580	hlock = curr->held_locks + i;
3581	chain_key = print_chain_key_iteration(hlock_id: hlock_id(hlock), chain_key);
3582
3583	print_lock(hlock);
3584	}
3585
3586	print_chain_key_iteration(hlock_id: hlock_id(hlock: hlock_next), chain_key);
3587	print_lock(hlock: hlock_next);
3588	}
3589
3590	static void print_chain_keys_chain(struct lock_chain *chain)
3591	{
3592	int i;
3593	u64 chain_key = INITIAL_CHAIN_KEY;
3594	u16 hlock_id;
3595
3596	printk("depth: %u\n", chain->depth);
3597	for (i = 0; i < chain->depth; i++) {
3598	hlock_id = chain_hlocks[chain->base + i];
3599	chain_key = print_chain_key_iteration(hlock_id, chain_key);
3600
3601	print_lock_name(NULL, class: lock_classes + chain_hlock_class_idx(hlock_id));
3602	printk("\n");
3603	}
3604	}
3605
3606	static void print_collision(struct task_struct *curr,
3607	struct held_lock *hlock_next,
3608	struct lock_chain *chain)
3609	{
3610	pr_warn("\n");
3611	pr_warn("============================\n");
3612	pr_warn("WARNING: chain_key collision\n");
3613	print_kernel_ident();
3614	pr_warn("----------------------------\n");
3615	pr_warn("%s/%d: ", current->comm, task_pid_nr(current));
3616	pr_warn("Hash chain already cached but the contents don't match!\n");
3617
3618	pr_warn("Held locks:");
3619	print_chain_keys_held_locks(curr, hlock_next);
3620
3621	pr_warn("Locks in cached chain:");
3622	print_chain_keys_chain(chain);
3623
3624	pr_warn("\nstack backtrace:\n");
3625	dump_stack();
3626	}
3627	#endif
3628
3629	/*
3630	* Checks whether the chain and the current held locks are consistent
3631	* in depth and also in content. If they are not it most likely means
3632	* that there was a collision during the calculation of the chain_key.
3633	* Returns: 0 not passed, 1 passed
3634	*/
3635	static int check_no_collision(struct task_struct *curr,
3636	struct held_lock *hlock,
3637	struct lock_chain *chain)
3638	{
3639	#ifdef CONFIG_DEBUG_LOCKDEP
3640	int i, j, id;
3641
3642	i = get_first_held_lock(curr, hlock);
3643
3644	if (DEBUG_LOCKS_WARN_ON(chain->depth != curr->lockdep_depth - (i - 1))) {
3645	print_collision(curr, hlock_next: hlock, chain);
3646	return 0;
3647	}
3648
3649	for (j = 0; j < chain->depth - 1; j++, i++) {
3650	id = hlock_id(hlock: &curr->held_locks[i]);
3651
3652	if (DEBUG_LOCKS_WARN_ON(chain_hlocks[chain->base + j] != id)) {
3653	print_collision(curr, hlock_next: hlock, chain);
3654	return 0;
3655	}
3656	}
3657	#endif
3658	return 1;
3659	}
3660
3661	/*
3662	* Given an index that is >= -1, return the index of the next lock chain.
3663	* Return -2 if there is no next lock chain.
3664	*/
3665	long lockdep_next_lockchain(long i)
3666	{
3667	i = find_next_bit(addr: lock_chains_in_use, ARRAY_SIZE(lock_chains), offset: i + 1);
3668	return i < ARRAY_SIZE(lock_chains) ? i : -2;
3669	}
3670
3671	unsigned long lock_chain_count(void)
3672	{
3673	return bitmap_weight(src: lock_chains_in_use, ARRAY_SIZE(lock_chains));
3674	}
3675
3676	/* Must be called with the graph lock held. */
3677	static struct lock_chain *alloc_lock_chain(void)
3678	{
3679	int idx = find_first_zero_bit(addr: lock_chains_in_use,
3680	ARRAY_SIZE(lock_chains));
3681
3682	if (unlikely(idx >= ARRAY_SIZE(lock_chains)))
3683	return NULL;
3684	__set_bit(idx, lock_chains_in_use);
3685	return lock_chains + idx;
3686	}
3687
3688	/*
3689	* Adds a dependency chain into chain hashtable. And must be called with
3690	* graph_lock held.
3691	*
3692	* Return 0 if fail, and graph_lock is released.
3693	* Return 1 if succeed, with graph_lock held.
3694	*/
3695	static inline int add_chain_cache(struct task_struct *curr,
3696	struct held_lock *hlock,
3697	u64 chain_key)
3698	{
3699	struct hlist_head *hash_head = chainhashentry(chain_key);
3700	struct lock_chain *chain;
3701	int i, j;
3702
3703	/*
3704	* The caller must hold the graph lock, ensure we've got IRQs
3705	* disabled to make this an IRQ-safe lock.. for recursion reasons
3706	* lockdep won't complain about its own locking errors.
3707	*/
3708	if (lockdep_assert_locked())
3709	return 0;
3710
3711	chain = alloc_lock_chain();
3712	if (!chain) {
3713	if (!debug_locks_off_graph_unlock())
3714	return 0;
3715
3716	print_lockdep_off(bug_msg: "BUG: MAX_LOCKDEP_CHAINS too low!");
3717	dump_stack();
3718	return 0;
3719	}
3720	chain->chain_key = chain_key;
3721	chain->irq_context = hlock->irq_context;
3722	i = get_first_held_lock(curr, hlock);
3723	chain->depth = curr->lockdep_depth + 1 - i;
3724
3725	BUILD_BUG_ON((1UL << 24) <= ARRAY_SIZE(chain_hlocks));
3726	BUILD_BUG_ON((1UL << 6) <= ARRAY_SIZE(curr->held_locks));
3727	BUILD_BUG_ON((1UL << 8*sizeof(chain_hlocks[0])) <= ARRAY_SIZE(lock_classes));
3728
3729	j = alloc_chain_hlocks(req: chain->depth);
3730	if (j < 0) {
3731	if (!debug_locks_off_graph_unlock())
3732	return 0;
3733
3734	print_lockdep_off(bug_msg: "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!");
3735	dump_stack();
3736	return 0;
3737	}
3738
3739	chain->base = j;
3740	for (j = 0; j < chain->depth - 1; j++, i++) {
3741	int lock_id = hlock_id(hlock: curr->held_locks + i);
3742
3743	chain_hlocks[chain->base + j] = lock_id;
3744	}
3745	chain_hlocks[chain->base + j] = hlock_id(hlock);
3746	hlist_add_head_rcu(n: &chain->entry, h: hash_head);
3747	debug_atomic_inc(chain_lookup_misses);
3748	inc_chains(irq_context: chain->irq_context);
3749
3750	return 1;
3751	}
3752
3753	/*
3754	* Look up a dependency chain. Must be called with either the graph lock or
3755	* the RCU read lock held.
3756	*/
3757	static inline struct lock_chain *lookup_chain_cache(u64 chain_key)
3758	{
3759	struct hlist_head *hash_head = chainhashentry(chain_key);
3760	struct lock_chain *chain;
3761
3762	hlist_for_each_entry_rcu(chain, hash_head, entry) {
3763	if (READ_ONCE(chain->chain_key) == chain_key) {
3764	debug_atomic_inc(chain_lookup_hits);
3765	return chain;
3766	}
3767	}
3768	return NULL;
3769	}
3770
3771	/*
3772	* If the key is not present yet in dependency chain cache then
3773	* add it and return 1 - in this case the new dependency chain is
3774	* validated. If the key is already hashed, return 0.
3775	* (On return with 1 graph_lock is held.)
3776	*/
3777	static inline int lookup_chain_cache_add(struct task_struct *curr,
3778	struct held_lock *hlock,
3779	u64 chain_key)
3780	{
3781	struct lock_class *class = hlock_class(hlock);
3782	struct lock_chain *chain = lookup_chain_cache(chain_key);
3783
3784	if (chain) {
3785	cache_hit:
3786	if (!check_no_collision(curr, hlock, chain))
3787	return 0;
3788
3789	if (very_verbose(class)) {
3790	printk("\nhash chain already cached, key: "
3791	"%016Lx tail class: [%px] %s\n",
3792	(unsigned long long)chain_key,
3793	class->key, class->name);
3794	}
3795
3796	return 0;
3797	}
3798
3799	if (very_verbose(class)) {
3800	printk("\nnew hash chain, key: %016Lx tail class: [%px] %s\n",
3801	(unsigned long long)chain_key, class->key, class->name);
3802	}
3803
3804	if (!graph_lock())
3805	return 0;
3806
3807	/*
3808	* We have to walk the chain again locked - to avoid duplicates:
3809	*/
3810	chain = lookup_chain_cache(chain_key);
3811	if (chain) {
3812	graph_unlock();
3813	goto cache_hit;
3814	}
3815
3816	if (!add_chain_cache(curr, hlock, chain_key))
3817	return 0;
3818
3819	return 1;
3820	}
3821
3822	static int validate_chain(struct task_struct *curr,
3823	struct held_lock *hlock,
3824	int chain_head, u64 chain_key)
3825	{
3826	/*
3827	* Trylock needs to maintain the stack of held locks, but it
3828	* does not add new dependencies, because trylock can be done
3829	* in any order.
3830	*
3831	* We look up the chain_key and do the O(N^2) check and update of
3832	* the dependencies only if this is a new dependency chain.
3833	* (If lookup_chain_cache_add() return with 1 it acquires
3834	* graph_lock for us)
3835	*/
3836	if (!hlock->trylock && hlock->check &&
3837	lookup_chain_cache_add(curr, hlock, chain_key)) {
3838	/*
3839	* Check whether last held lock:
3840	*
3841	* - is irq-safe, if this lock is irq-unsafe
3842	* - is softirq-safe, if this lock is hardirq-unsafe
3843	*
3844	* And check whether the new lock's dependency graph
3845	* could lead back to the previous lock:
3846	*
3847	* - within the current held-lock stack
3848	* - across our accumulated lock dependency records
3849	*
3850	* any of these scenarios could lead to a deadlock.
3851	*/
3852	/*
3853	* The simple case: does the current hold the same lock
3854	* already?
3855	*/
3856	int ret = check_deadlock(curr, next: hlock);
3857
3858	if (!ret)
3859	return 0;
3860	/*
3861	* Add dependency only if this lock is not the head
3862	* of the chain, and if the new lock introduces no more
3863	* lock dependency (because we already hold a lock with the
3864	* same lock class) nor deadlock (because the nest_lock
3865	* serializes nesting locks), see the comments for
3866	* check_deadlock().
3867	*/
3868	if (!chain_head && ret != 2) {
3869	if (!check_prevs_add(curr, next: hlock))
3870	return 0;
3871	}
3872
3873	graph_unlock();
3874	} else {
3875	/* after lookup_chain_cache_add(): */
3876	if (unlikely(!debug_locks))
3877	return 0;
3878	}
3879
3880	return 1;
3881	}
3882	#else
3883	static inline int validate_chain(struct task_struct *curr,
3884	struct held_lock *hlock,
3885	int chain_head, u64 chain_key)
3886	{
3887	return 1;
3888	}
3889
3890	static void init_chain_block_buckets(void) { }
3891	#endif /* CONFIG_PROVE_LOCKING */
3892
3893	/*
3894	* We are building curr_chain_key incrementally, so double-check
3895	* it from scratch, to make sure that it's done correctly:
3896	*/
3897	static void check_chain_key(struct task_struct *curr)
3898	{
3899	#ifdef CONFIG_DEBUG_LOCKDEP
3900	struct held_lock *hlock, *prev_hlock = NULL;
3901	unsigned int i;
3902	u64 chain_key = INITIAL_CHAIN_KEY;
3903
3904	for (i = 0; i < curr->lockdep_depth; i++) {
3905	hlock = curr->held_locks + i;
3906	if (chain_key != hlock->prev_chain_key) {
3907	debug_locks_off();
3908	/*
3909	* We got mighty confused, our chain keys don't match
3910	* with what we expect, someone trample on our task state?
3911	*/
3912	WARN(1, "hm#1, depth: %u [%u], %016Lx != %016Lx\n",
3913	curr->lockdep_depth, i,
3914	(unsigned long long)chain_key,
3915	(unsigned long long)hlock->prev_chain_key);
3916	return;
3917	}
3918
3919	/*
3920	* hlock->class_idx can't go beyond MAX_LOCKDEP_KEYS, but is
3921	* it registered lock class index?
3922	*/
3923	if (DEBUG_LOCKS_WARN_ON(!test_bit(hlock->class_idx, lock_classes_in_use)))
3924	return;
3925
3926	if (prev_hlock && (prev_hlock->irq_context !=
3927	hlock->irq_context))
3928	chain_key = INITIAL_CHAIN_KEY;
3929	chain_key = iterate_chain_key(key: chain_key, idx: hlock_id(hlock));
3930	prev_hlock = hlock;
3931	}
3932	if (chain_key != curr->curr_chain_key) {
3933	debug_locks_off();
3934	/*
3935	* More smoking hash instead of calculating it, damn see these
3936	* numbers float.. I bet that a pink elephant stepped on my memory.
3937	*/
3938	WARN(1, "hm#2, depth: %u [%u], %016Lx != %016Lx\n",
3939	curr->lockdep_depth, i,
3940	(unsigned long long)chain_key,
3941	(unsigned long long)curr->curr_chain_key);
3942	}
3943	#endif
3944	}
3945
3946	#ifdef CONFIG_PROVE_LOCKING
3947	static int mark_lock(struct task_struct *curr, struct held_lock *this,
3948	enum lock_usage_bit new_bit);
3949
3950	static void print_usage_bug_scenario(struct held_lock *lock)
3951	{
3952	struct lock_class *class = hlock_class(hlock: lock);
3953
3954	printk(" Possible unsafe locking scenario:\n\n");
3955	printk(" CPU0\n");
3956	printk(" ----\n");
3957	printk(" lock(");
3958	__print_lock_name(hlock: lock, class);
3959	printk(KERN_CONT ");\n");
3960	printk(" <Interrupt>\n");
3961	printk(" lock(");
3962	__print_lock_name(hlock: lock, class);
3963	printk(KERN_CONT ");\n");
3964	printk("\n *** DEADLOCK ***\n\n");
3965	}
3966
3967	static void
3968	print_usage_bug(struct task_struct *curr, struct held_lock *this,
3969	enum lock_usage_bit prev_bit, enum lock_usage_bit new_bit)
3970	{
3971	if (!debug_locks_off() || debug_locks_silent)
3972	return;
3973
3974	pr_warn("\n");
3975	pr_warn("================================\n");
3976	pr_warn("WARNING: inconsistent lock state\n");
3977	print_kernel_ident();
3978	pr_warn("--------------------------------\n");
3979
3980	pr_warn("inconsistent {%s} -> {%s} usage.\n",
3981	usage_str[prev_bit], usage_str[new_bit]);
3982
3983	pr_warn("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] takes:\n",
3984	curr->comm, task_pid_nr(curr),
3985	lockdep_hardirq_context(), hardirq_count() >> HARDIRQ_SHIFT,
3986	lockdep_softirq_context(curr), softirq_count() >> SOFTIRQ_SHIFT,
3987	lockdep_hardirqs_enabled(),
3988	lockdep_softirqs_enabled(curr));
3989	print_lock(hlock: this);
3990
3991	pr_warn("{%s} state was registered at:\n", usage_str[prev_bit]);
3992	print_lock_trace(trace: hlock_class(hlock: this)->usage_traces[prev_bit], spaces: 1);
3993
3994	print_irqtrace_events(curr);
3995	pr_warn("\nother info that might help us debug this:\n");
3996	print_usage_bug_scenario(lock: this);
3997
3998	lockdep_print_held_locks(p: curr);
3999
4000	pr_warn("\nstack backtrace:\n");
4001	dump_stack();
4002	}
4003
4004	/*
4005	* Print out an error if an invalid bit is set:
4006	*/
4007	static inline int
4008	valid_state(struct task_struct *curr, struct held_lock *this,
4009	enum lock_usage_bit new_bit, enum lock_usage_bit bad_bit)
4010	{
4011	if (unlikely(hlock_class(this)->usage_mask & (1 << bad_bit))) {
4012	graph_unlock();
4013	print_usage_bug(curr, this, prev_bit: bad_bit, new_bit);
4014	return 0;
4015	}
4016	return 1;
4017	}
4018
4019
4020	/*
4021	* print irq inversion bug:
4022	*/
4023	static void
4024	print_irq_inversion_bug(struct task_struct *curr,
4025	struct lock_list *root, struct lock_list *other,
4026	struct held_lock *this, int forwards,
4027	const char *irqclass)
4028	{
4029	struct lock_list *entry = other;
4030	struct lock_list *middle = NULL;
4031	int depth;
4032
4033	if (!debug_locks_off_graph_unlock() || debug_locks_silent)
4034	return;
4035
4036	pr_warn("\n");
4037	pr_warn("========================================================\n");
4038	pr_warn("WARNING: possible irq lock inversion dependency detected\n");
4039	print_kernel_ident();
4040	pr_warn("--------------------------------------------------------\n");
4041	pr_warn("%s/%d just changed the state of lock:\n",
4042	curr->comm, task_pid_nr(curr));
4043	print_lock(hlock: this);
4044	if (forwards)
4045	pr_warn("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
4046	else
4047	pr_warn("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
4048	print_lock_name(NULL, class: other->class);
4049	pr_warn("\n\nand interrupts could create inverse lock ordering between them.\n\n");
4050
4051	pr_warn("\nother info that might help us debug this:\n");
4052
4053	/* Find a middle lock (if one exists) */
4054	depth = get_lock_depth(child: other);
4055	do {
4056	if (depth == 0 && (entry != root)) {
4057	pr_warn("lockdep:%s bad path found in chain graph\n", __func__);
4058	break;
4059	}
4060	middle = entry;
4061	entry = get_lock_parent(child: entry);
4062	depth--;
4063	} while (entry && entry != root && (depth >= 0));
4064	if (forwards)
4065	print_irq_lock_scenario(safe_entry: root, unsafe_entry: other,
4066	prev_class: middle ? middle->class : root->class, next_class: other->class);
4067	else
4068	print_irq_lock_scenario(safe_entry: other, unsafe_entry: root,
4069	prev_class: middle ? middle->class : other->class, next_class: root->class);
4070
4071	lockdep_print_held_locks(p: curr);
4072
4073	pr_warn("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
4074	root->trace = save_trace();
4075	if (!root->trace)
4076	return;
4077	print_shortest_lock_dependencies(leaf: other, root);
4078
4079	pr_warn("\nstack backtrace:\n");
4080	dump_stack();
4081	}
4082
4083	/*
4084	* Prove that in the forwards-direction subgraph starting at <this>
4085	* there is no lock matching <mask>:
4086	*/
4087	static int
4088	check_usage_forwards(struct task_struct *curr, struct held_lock *this,
4089	enum lock_usage_bit bit)
4090	{
4091	enum bfs_result ret;
4092	struct lock_list root;
4093	struct lock_list *target_entry;
4094	enum lock_usage_bit read_bit = bit + LOCK_USAGE_READ_MASK;
4095	unsigned usage_mask = lock_flag(bit) | lock_flag(bit: read_bit);
4096
4097	bfs_init_root(lock: &root, hlock: this);
4098	ret = find_usage_forwards(root: &root, usage_mask, target_entry: &target_entry);
4099	if (bfs_error(res: ret)) {
4100	print_bfs_bug(ret);
4101	return 0;
4102	}
4103	if (ret == BFS_RNOMATCH)
4104	return 1;
4105
4106	/* Check whether write or read usage is the match */
4107	if (target_entry->class->usage_mask & lock_flag(bit)) {
4108	print_irq_inversion_bug(curr, root: &root, other: target_entry,
4109	this, forwards: 1, irqclass: state_name(bit));
4110	} else {
4111	print_irq_inversion_bug(curr, root: &root, other: target_entry,
4112	this, forwards: 1, irqclass: state_name(bit: read_bit));
4113	}
4114
4115	return 0;
4116	}
4117
4118	/*
4119	* Prove that in the backwards-direction subgraph starting at <this>
4120	* there is no lock matching <mask>:
4121	*/
4122	static int
4123	check_usage_backwards(struct task_struct *curr, struct held_lock *this,
4124	enum lock_usage_bit bit)
4125	{
4126	enum bfs_result ret;
4127	struct lock_list root;
4128	struct lock_list *target_entry;
4129	enum lock_usage_bit read_bit = bit + LOCK_USAGE_READ_MASK;
4130	unsigned usage_mask = lock_flag(bit) | lock_flag(bit: read_bit);
4131
4132	bfs_init_rootb(lock: &root, hlock: this);
4133	ret = find_usage_backwards(root: &root, usage_mask, target_entry: &target_entry);
4134	if (bfs_error(res: ret)) {
4135	print_bfs_bug(ret);
4136	return 0;
4137	}
4138	if (ret == BFS_RNOMATCH)
4139	return 1;
4140
4141	/* Check whether write or read usage is the match */
4142	if (target_entry->class->usage_mask & lock_flag(bit)) {
4143	print_irq_inversion_bug(curr, root: &root, other: target_entry,
4144	this, forwards: 0, irqclass: state_name(bit));
4145	} else {
4146	print_irq_inversion_bug(curr, root: &root, other: target_entry,
4147	this, forwards: 0, irqclass: state_name(bit: read_bit));
4148	}
4149
4150	return 0;
4151	}
4152
4153	void print_irqtrace_events(struct task_struct *curr)
4154	{
4155	const struct irqtrace_events *trace = &curr->irqtrace;
4156
4157	printk("irq event stamp: %u\n", trace->irq_events);
4158	printk("hardirqs last enabled at (%u): [<%px>] %pS\n",
4159	trace->hardirq_enable_event, (void *)trace->hardirq_enable_ip,
4160	(void *)trace->hardirq_enable_ip);
4161	printk("hardirqs last disabled at (%u): [<%px>] %pS\n",
4162	trace->hardirq_disable_event, (void *)trace->hardirq_disable_ip,
4163	(void *)trace->hardirq_disable_ip);
4164	printk("softirqs last enabled at (%u): [<%px>] %pS\n",
4165	trace->softirq_enable_event, (void *)trace->softirq_enable_ip,
4166	(void *)trace->softirq_enable_ip);
4167	printk("softirqs last disabled at (%u): [<%px>] %pS\n",
4168	trace->softirq_disable_event, (void *)trace->softirq_disable_ip,
4169	(void *)trace->softirq_disable_ip);
4170	}
4171
4172	static int HARDIRQ_verbose(struct lock_class *class)
4173	{
4174	#if HARDIRQ_VERBOSE
4175	return class_filter(class);
4176	#endif
4177	return 0;
4178	}
4179
4180	static int SOFTIRQ_verbose(struct lock_class *class)
4181	{
4182	#if SOFTIRQ_VERBOSE
4183	return class_filter(class);
4184	#endif
4185	return 0;
4186	}
4187
4188	static int (*state_verbose_f[])(struct lock_class *class) = {
4189	#define LOCKDEP_STATE(__STATE) \
4190	__STATE##_verbose,
4191	#include "lockdep_states.h"
4192	#undef LOCKDEP_STATE
4193	};
4194
4195	static inline int state_verbose(enum lock_usage_bit bit,
4196	struct lock_class *class)
4197	{
4198	return state_verbose_f[bit >> LOCK_USAGE_DIR_MASK](class);
4199	}
4200
4201	typedef int (*check_usage_f)(struct task_struct *, struct held_lock *,
4202	enum lock_usage_bit bit, const char *name);
4203
4204	static int
4205	mark_lock_irq(struct task_struct *curr, struct held_lock *this,
4206	enum lock_usage_bit new_bit)
4207	{
4208	int excl_bit = exclusive_bit(new_bit);
4209	int read = new_bit & LOCK_USAGE_READ_MASK;
4210	int dir = new_bit & LOCK_USAGE_DIR_MASK;
4211
4212	/*
4213	* Validate that this particular lock does not have conflicting
4214	* usage states.
4215	*/
4216	if (!valid_state(curr, this, new_bit, bad_bit: excl_bit))
4217	return 0;
4218
4219	/*
4220	* Check for read in write conflicts
4221	*/
4222	if (!read && !valid_state(curr, this, new_bit,
4223	bad_bit: excl_bit + LOCK_USAGE_READ_MASK))
4224	return 0;
4225
4226
4227	/*
4228	* Validate that the lock dependencies don't have conflicting usage
4229	* states.
4230	*/
4231	if (dir) {
4232	/*
4233	* mark ENABLED has to look backwards -- to ensure no dependee
4234	* has USED_IN state, which, again, would allow recursion deadlocks.
4235	*/
4236	if (!check_usage_backwards(curr, this, bit: excl_bit))
4237	return 0;
4238	} else {
4239	/*
4240	* mark USED_IN has to look forwards -- to ensure no dependency
4241	* has ENABLED state, which would allow recursion deadlocks.
4242	*/
4243	if (!check_usage_forwards(curr, this, bit: excl_bit))
4244	return 0;
4245	}
4246
4247	if (state_verbose(bit: new_bit, class: hlock_class(hlock: this)))
4248	return 2;
4249
4250	return 1;
4251	}
4252
4253	/*
4254	* Mark all held locks with a usage bit:
4255	*/
4256	static int
4257	mark_held_locks(struct task_struct *curr, enum lock_usage_bit base_bit)
4258	{
4259	struct held_lock *hlock;
4260	int i;
4261
4262	for (i = 0; i < curr->lockdep_depth; i++) {
4263	enum lock_usage_bit hlock_bit = base_bit;
4264	hlock = curr->held_locks + i;
4265
4266	if (hlock->read)
4267	hlock_bit += LOCK_USAGE_READ_MASK;
4268
4269	BUG_ON(hlock_bit >= LOCK_USAGE_STATES);
4270
4271	if (!hlock->check)
4272	continue;
4273
4274	if (!mark_lock(curr, this: hlock, new_bit: hlock_bit))
4275	return 0;
4276	}
4277
4278	return 1;
4279	}
4280
4281	/*
4282	* Hardirqs will be enabled:
4283	*/
4284	static void __trace_hardirqs_on_caller(void)
4285	{
4286	struct task_struct *curr = current;
4287
4288	/*
4289	* We are going to turn hardirqs on, so set the
4290	* usage bit for all held locks:
4291	*/
4292	if (!mark_held_locks(curr, base_bit: LOCK_ENABLED_HARDIRQ))
4293	return;
4294	/*
4295	* If we have softirqs enabled, then set the usage
4296	* bit for all held locks. (disabled hardirqs prevented
4297	* this bit from being set before)
4298	*/
4299	if (curr->softirqs_enabled)
4300	mark_held_locks(curr, base_bit: LOCK_ENABLED_SOFTIRQ);
4301	}
4302
4303	/**
4304	* lockdep_hardirqs_on_prepare - Prepare for enabling interrupts
4305	*
4306	* Invoked before a possible transition to RCU idle from exit to user or
4307	* guest mode. This ensures that all RCU operations are done before RCU
4308	* stops watching. After the RCU transition lockdep_hardirqs_on() has to be
4309	* invoked to set the final state.
4310	*/
4311	void lockdep_hardirqs_on_prepare(void)
4312	{
4313	if (unlikely(!debug_locks))
4314	return;
4315
4316	/*
4317	* NMIs do not (and cannot) track lock dependencies, nothing to do.
4318	*/
4319	if (unlikely(in_nmi()))
4320	return;
4321
4322	if (unlikely(this_cpu_read(lockdep_recursion)))
4323	return;
4324
4325	if (unlikely(lockdep_hardirqs_enabled())) {
4326	/*
4327	* Neither irq nor preemption are disabled here
4328	* so this is racy by nature but losing one hit
4329	* in a stat is not a big deal.
4330	*/
4331	__debug_atomic_inc(redundant_hardirqs_on);
4332	return;
4333	}
4334
4335	/*
4336	* We're enabling irqs and according to our state above irqs weren't
4337	* already enabled, yet we find the hardware thinks they are in fact
4338	* enabled.. someone messed up their IRQ state tracing.
4339	*/
4340	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
4341	return;
4342
4343	/*
4344	* See the fine text that goes along with this variable definition.
4345	*/
4346	if (DEBUG_LOCKS_WARN_ON(early_boot_irqs_disabled))
4347	return;
4348
4349	/*
4350	* Can't allow enabling interrupts while in an interrupt handler,
4351	* that's general bad form and such. Recursion, limited stack etc..
4352	*/
4353	if (DEBUG_LOCKS_WARN_ON(lockdep_hardirq_context()))
4354	return;
4355
4356	current->hardirq_chain_key = current->curr_chain_key;
4357
4358	lockdep_recursion_inc();
4359	__trace_hardirqs_on_caller();
4360	lockdep_recursion_finish();
4361	}
4362	EXPORT_SYMBOL_GPL(lockdep_hardirqs_on_prepare);
4363
4364	void noinstr lockdep_hardirqs_on(unsigned long ip)
4365	{
4366	struct irqtrace_events *trace = &current->irqtrace;
4367
4368	if (unlikely(!debug_locks))
4369	return;
4370
4371	/*
4372	* NMIs can happen in the middle of local_irq_{en,dis}able() where the
4373	* tracking state and hardware state are out of sync.
4374	*
4375	* NMIs must save lockdep_hardirqs_enabled() to restore IRQ state from,
4376	* and not rely on hardware state like normal interrupts.
4377	*/
4378	if (unlikely(in_nmi())) {
4379	if (!IS_ENABLED(CONFIG_TRACE_IRQFLAGS_NMI))
4380	return;
4381
4382	/*
4383	* Skip:
4384	* - recursion check, because NMI can hit lockdep;
4385	* - hardware state check, because above;
4386	* - chain_key check, see lockdep_hardirqs_on_prepare().
4387	*/
4388	goto skip_checks;
4389	}
4390
4391	if (unlikely(this_cpu_read(lockdep_recursion)))
4392	return;
4393
4394	if (lockdep_hardirqs_enabled()) {
4395	/*
4396	* Neither irq nor preemption are disabled here
4397	* so this is racy by nature but losing one hit
4398	* in a stat is not a big deal.
4399	*/
4400	__debug_atomic_inc(redundant_hardirqs_on);
4401	return;
4402	}
4403
4404	/*
4405	* We're enabling irqs and according to our state above irqs weren't
4406	* already enabled, yet we find the hardware thinks they are in fact
4407	* enabled.. someone messed up their IRQ state tracing.
4408	*/
4409	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
4410	return;
4411
4412	/*
4413	* Ensure the lock stack remained unchanged between
4414	* lockdep_hardirqs_on_prepare() and lockdep_hardirqs_on().
4415	*/
4416	DEBUG_LOCKS_WARN_ON(current->hardirq_chain_key !=
4417	current->curr_chain_key);
4418
4419	skip_checks:
4420	/* we'll do an OFF -> ON transition: */
4421	__this_cpu_write(hardirqs_enabled, 1);
4422	trace->hardirq_enable_ip = ip;
4423	trace->hardirq_enable_event = ++trace->irq_events;
4424	debug_atomic_inc(hardirqs_on_events);
4425	}
4426	EXPORT_SYMBOL_GPL(lockdep_hardirqs_on);
4427
4428	/*
4429	* Hardirqs were disabled:
4430	*/
4431	void noinstr lockdep_hardirqs_off(unsigned long ip)
4432	{
4433	if (unlikely(!debug_locks))
4434	return;
4435
4436	/*
4437	* Matching lockdep_hardirqs_on(), allow NMIs in the middle of lockdep;
4438	* they will restore the software state. This ensures the software
4439	* state is consistent inside NMIs as well.
4440	*/
4441	if (in_nmi()) {
4442	if (!IS_ENABLED(CONFIG_TRACE_IRQFLAGS_NMI))
4443	return;
4444	} else if (__this_cpu_read(lockdep_recursion))
4445	return;
4446
4447	/*
4448	* So we're supposed to get called after you mask local IRQs, but for
4449	* some reason the hardware doesn't quite think you did a proper job.
4450	*/
4451	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
4452	return;
4453
4454	if (lockdep_hardirqs_enabled()) {
4455	struct irqtrace_events *trace = &current->irqtrace;
4456
4457	/*
4458	* We have done an ON -> OFF transition:
4459	*/
4460	__this_cpu_write(hardirqs_enabled, 0);
4461	trace->hardirq_disable_ip = ip;
4462	trace->hardirq_disable_event = ++trace->irq_events;
4463	debug_atomic_inc(hardirqs_off_events);
4464	} else {
4465	debug_atomic_inc(redundant_hardirqs_off);
4466	}
4467	}
4468	EXPORT_SYMBOL_GPL(lockdep_hardirqs_off);
4469
4470	/*
4471	* Softirqs will be enabled:
4472	*/
4473	void lockdep_softirqs_on(unsigned long ip)
4474	{
4475	struct irqtrace_events *trace = &current->irqtrace;
4476
4477	if (unlikely(!lockdep_enabled()))
4478	return;
4479
4480	/*
4481	* We fancy IRQs being disabled here, see softirq.c, avoids
4482	* funny state and nesting things.
4483	*/
4484	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
4485	return;
4486
4487	if (current->softirqs_enabled) {
4488	debug_atomic_inc(redundant_softirqs_on);
4489	return;
4490	}
4491
4492	lockdep_recursion_inc();
4493	/*
4494	* We'll do an OFF -> ON transition:
4495	*/
4496	current->softirqs_enabled = 1;
4497	trace->softirq_enable_ip = ip;
4498	trace->softirq_enable_event = ++trace->irq_events;
4499	debug_atomic_inc(softirqs_on_events);
4500	/*
4501	* We are going to turn softirqs on, so set the
4502	* usage bit for all held locks, if hardirqs are
4503	* enabled too:
4504	*/
4505	if (lockdep_hardirqs_enabled())
4506	mark_held_locks(current, base_bit: LOCK_ENABLED_SOFTIRQ);
4507	lockdep_recursion_finish();
4508	}
4509
4510	/*
4511	* Softirqs were disabled:
4512	*/
4513	void lockdep_softirqs_off(unsigned long ip)
4514	{
4515	if (unlikely(!lockdep_enabled()))
4516	return;
4517
4518	/*
4519	* We fancy IRQs being disabled here, see softirq.c
4520	*/
4521	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
4522	return;
4523
4524	if (current->softirqs_enabled) {
4525	struct irqtrace_events *trace = &current->irqtrace;
4526
4527	/*
4528	* We have done an ON -> OFF transition:
4529	*/
4530	current->softirqs_enabled = 0;
4531	trace->softirq_disable_ip = ip;
4532	trace->softirq_disable_event = ++trace->irq_events;
4533	debug_atomic_inc(softirqs_off_events);
4534	/*
4535	* Whoops, we wanted softirqs off, so why aren't they?
4536	*/
4537	DEBUG_LOCKS_WARN_ON(!softirq_count());
4538	} else
4539	debug_atomic_inc(redundant_softirqs_off);
4540	}
4541
4542	static int
4543	mark_usage(struct task_struct *curr, struct held_lock *hlock, int check)
4544	{
4545	if (!check)
4546	goto lock_used;
4547
4548	/*
4549	* If non-trylock use in a hardirq or softirq context, then
4550	* mark the lock as used in these contexts:
4551	*/
4552	if (!hlock->trylock) {
4553	if (hlock->read) {
4554	if (lockdep_hardirq_context())
4555	if (!mark_lock(curr, this: hlock,
4556	new_bit: LOCK_USED_IN_HARDIRQ_READ))
4557	return 0;
4558	if (curr->softirq_context)
4559	if (!mark_lock(curr, this: hlock,
4560	new_bit: LOCK_USED_IN_SOFTIRQ_READ))
4561	return 0;
4562	} else {
4563	if (lockdep_hardirq_context())
4564	if (!mark_lock(curr, this: hlock, new_bit: LOCK_USED_IN_HARDIRQ))
4565	return 0;
4566	if (curr->softirq_context)
4567	if (!mark_lock(curr, this: hlock, new_bit: LOCK_USED_IN_SOFTIRQ))
4568	return 0;
4569	}
4570	}
4571
4572	/*
4573	* For lock_sync(), don't mark the ENABLED usage, since lock_sync()
4574	* creates no critical section and no extra dependency can be introduced
4575	* by interrupts
4576	*/
4577	if (!hlock->hardirqs_off && !hlock->sync) {
4578	if (hlock->read) {
4579	if (!mark_lock(curr, this: hlock,
4580	new_bit: LOCK_ENABLED_HARDIRQ_READ))
4581	return 0;
4582	if (curr->softirqs_enabled)
4583	if (!mark_lock(curr, this: hlock,
4584	new_bit: LOCK_ENABLED_SOFTIRQ_READ))
4585	return 0;
4586	} else {
4587	if (!mark_lock(curr, this: hlock,
4588	new_bit: LOCK_ENABLED_HARDIRQ))
4589	return 0;
4590	if (curr->softirqs_enabled)
4591	if (!mark_lock(curr, this: hlock,
4592	new_bit: LOCK_ENABLED_SOFTIRQ))
4593	return 0;
4594	}
4595	}
4596
4597	lock_used:
4598	/* mark it as used: */
4599	if (!mark_lock(curr, this: hlock, new_bit: LOCK_USED))
4600	return 0;
4601
4602	return 1;
4603	}
4604
4605	static inline unsigned int task_irq_context(struct task_struct *task)
4606	{
4607	return LOCK_CHAIN_HARDIRQ_CONTEXT * !!lockdep_hardirq_context() +
4608	LOCK_CHAIN_SOFTIRQ_CONTEXT * !!task->softirq_context;
4609	}
4610
4611	static int separate_irq_context(struct task_struct *curr,
4612	struct held_lock *hlock)
4613	{
4614	unsigned int depth = curr->lockdep_depth;
4615
4616	/*
4617	* Keep track of points where we cross into an interrupt context:
4618	*/
4619	if (depth) {
4620	struct held_lock *prev_hlock;
4621
4622	prev_hlock = curr->held_locks + depth-1;
4623	/*
4624	* If we cross into another context, reset the
4625	* hash key (this also prevents the checking and the
4626	* adding of the dependency to 'prev'):
4627	*/
4628	if (prev_hlock->irq_context != hlock->irq_context)
4629	return 1;
4630	}
4631	return 0;
4632	}
4633
4634	/*
4635	* Mark a lock with a usage bit, and validate the state transition:
4636	*/
4637	static int mark_lock(struct task_struct *curr, struct held_lock *this,
4638	enum lock_usage_bit new_bit)
4639	{
4640	unsigned int new_mask, ret = 1;
4641
4642	if (new_bit >= LOCK_USAGE_STATES) {
4643	DEBUG_LOCKS_WARN_ON(1);
4644	return 0;
4645	}
4646
4647	if (new_bit == LOCK_USED && this->read)
4648	new_bit = LOCK_USED_READ;
4649
4650	new_mask = 1 << new_bit;
4651
4652	/*
4653	* If already set then do not dirty the cacheline,
4654	* nor do any checks:
4655	*/
4656	if (likely(hlock_class(this)->usage_mask & new_mask))
4657	return 1;
4658
4659	if (!graph_lock())
4660	return 0;
4661	/*
4662	* Make sure we didn't race:
4663	*/
4664	if (unlikely(hlock_class(this)->usage_mask & new_mask))
4665	goto unlock;
4666
4667	if (!hlock_class(hlock: this)->usage_mask)
4668	debug_atomic_dec(nr_unused_locks);
4669
4670	hlock_class(hlock: this)->usage_mask |= new_mask;
4671
4672	if (new_bit < LOCK_TRACE_STATES) {
4673	if (!(hlock_class(hlock: this)->usage_traces[new_bit] = save_trace()))
4674	return 0;
4675	}
4676
4677	if (new_bit < LOCK_USED) {
4678	ret = mark_lock_irq(curr, this, new_bit);
4679	if (!ret)
4680	return 0;
4681	}
4682
4683	unlock:
4684	graph_unlock();
4685
4686	/*
4687	* We must printk outside of the graph_lock:
4688	*/
4689	if (ret == 2) {
4690	printk("\nmarked lock as {%s}:\n", usage_str[new_bit]);
4691	print_lock(hlock: this);
4692	print_irqtrace_events(curr);
4693	dump_stack();
4694	}
4695
4696	return ret;
4697	}
4698
4699	static inline short task_wait_context(struct task_struct *curr)
4700	{
4701	/*
4702	* Set appropriate wait type for the context; for IRQs we have to take
4703	* into account force_irqthread as that is implied by PREEMPT_RT.
4704	*/
4705	if (lockdep_hardirq_context()) {
4706	/*
4707	* Check if force_irqthreads will run us threaded.
4708	*/
4709	if (curr->hardirq_threaded || curr->irq_config)
4710	return LD_WAIT_CONFIG;
4711
4712	return LD_WAIT_SPIN;
4713	} else if (curr->softirq_context) {
4714	/*
4715	* Softirqs are always threaded.
4716	*/
4717	return LD_WAIT_CONFIG;
4718	}
4719
4720	return LD_WAIT_MAX;
4721	}
4722
4723	static int
4724	print_lock_invalid_wait_context(struct task_struct *curr,
4725	struct held_lock *hlock)
4726	{
4727	short curr_inner;
4728
4729	if (!debug_locks_off())
4730	return 0;
4731	if (debug_locks_silent)
4732	return 0;
4733
4734	pr_warn("\n");
4735	pr_warn("=============================\n");
4736	pr_warn("[ BUG: Invalid wait context ]\n");
4737	print_kernel_ident();
4738	pr_warn("-----------------------------\n");
4739
4740	pr_warn("%s/%d is trying to lock:\n", curr->comm, task_pid_nr(curr));
4741	print_lock(hlock);
4742
4743	pr_warn("other info that might help us debug this:\n");
4744
4745	curr_inner = task_wait_context(curr);
4746	pr_warn("context-{%d:%d}\n", curr_inner, curr_inner);
4747
4748	lockdep_print_held_locks(p: curr);
4749
4750	pr_warn("stack backtrace:\n");
4751	dump_stack();
4752
4753	return 0;
4754	}
4755
4756	/*
4757	* Verify the wait_type context.
4758	*
4759	* This check validates we take locks in the right wait-type order; that is it
4760	* ensures that we do not take mutexes inside spinlocks and do not attempt to
4761	* acquire spinlocks inside raw_spinlocks and the sort.
4762	*
4763	* The entire thing is slightly more complex because of RCU, RCU is a lock that
4764	* can be taken from (pretty much) any context but also has constraints.
4765	* However when taken in a stricter environment the RCU lock does not loosen
4766	* the constraints.
4767	*
4768	* Therefore we must look for the strictest environment in the lock stack and
4769	* compare that to the lock we're trying to acquire.
4770	*/
4771	static int check_wait_context(struct task_struct *curr, struct held_lock *next)
4772	{
4773	u8 next_inner = hlock_class(hlock: next)->wait_type_inner;
4774	u8 next_outer = hlock_class(hlock: next)->wait_type_outer;
4775	u8 curr_inner;
4776	int depth;
4777
4778	if (!next_inner || next->trylock)
4779	return 0;
4780
4781	if (!next_outer)
4782	next_outer = next_inner;
4783
4784	/*
4785	* Find start of current irq_context..
4786	*/
4787	for (depth = curr->lockdep_depth - 1; depth >= 0; depth--) {
4788	struct held_lock *prev = curr->held_locks + depth;
4789	if (prev->irq_context != next->irq_context)
4790	break;
4791	}
4792	depth++;
4793
4794	curr_inner = task_wait_context(curr);
4795
4796	for (; depth < curr->lockdep_depth; depth++) {
4797	struct held_lock *prev = curr->held_locks + depth;
4798	struct lock_class *class = hlock_class(hlock: prev);
4799	u8 prev_inner = class->wait_type_inner;
4800
4801	if (prev_inner) {
4802	/*
4803	* We can have a bigger inner than a previous one
4804	* when outer is smaller than inner, as with RCU.
4805	*
4806	* Also due to trylocks.
4807	*/
4808	curr_inner = min(curr_inner, prev_inner);
4809
4810	/*
4811	* Allow override for annotations -- this is typically
4812	* only valid/needed for code that only exists when
4813	* CONFIG_PREEMPT_RT=n.
4814	*/
4815	if (unlikely(class->lock_type == LD_LOCK_WAIT_OVERRIDE))
4816	curr_inner = prev_inner;
4817	}
4818	}
4819
4820	if (next_outer > curr_inner)
4821	return print_lock_invalid_wait_context(curr, hlock: next);
4822
4823	return 0;
4824	}
4825
4826	#else /* CONFIG_PROVE_LOCKING */
4827
4828	static inline int
4829	mark_usage(struct task_struct *curr, struct held_lock *hlock, int check)
4830	{
4831	return 1;
4832	}
4833
4834	static inline unsigned int task_irq_context(struct task_struct *task)
4835	{
4836	return 0;
4837	}
4838
4839	static inline int separate_irq_context(struct task_struct *curr,
4840	struct held_lock *hlock)
4841	{
4842	return 0;
4843	}
4844
4845	static inline int check_wait_context(struct task_struct *curr,
4846	struct held_lock *next)
4847	{
4848	return 0;
4849	}
4850
4851	#endif /* CONFIG_PROVE_LOCKING */
4852
4853	/*
4854	* Initialize a lock instance's lock-class mapping info:
4855	*/
4856	void lockdep_init_map_type(struct lockdep_map *lock, const char *name,
4857	struct lock_class_key *key, int subclass,
4858	u8 inner, u8 outer, u8 lock_type)
4859	{
4860	int i;
4861
4862	for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++)
4863	lock->class_cache[i] = NULL;
4864
4865	#ifdef CONFIG_LOCK_STAT
4866	lock->cpu = raw_smp_processor_id();
4867	#endif
4868
4869	/*
4870	* Can't be having no nameless bastards around this place!
4871	*/
4872	if (DEBUG_LOCKS_WARN_ON(!name)) {
4873	lock->name = "NULL";
4874	return;
4875	}
4876
4877	lock->name = name;
4878
4879	lock->wait_type_outer = outer;
4880	lock->wait_type_inner = inner;
4881	lock->lock_type = lock_type;
4882
4883	/*
4884	* No key, no joy, we need to hash something.
4885	*/
4886	if (DEBUG_LOCKS_WARN_ON(!key))
4887	return;
4888	/*
4889	* Sanity check, the lock-class key must either have been allocated
4890	* statically or must have been registered as a dynamic key.
4891	*/
4892	if (!static_obj(obj: key) && !is_dynamic_key(key)) {
4893	if (debug_locks)
4894	printk(KERN_ERR "BUG: key %px has not been registered!\n", key);
4895	DEBUG_LOCKS_WARN_ON(1);
4896	return;
4897	}
4898	lock->key = key;
4899
4900	if (unlikely(!debug_locks))
4901	return;
4902
4903	if (subclass) {
4904	unsigned long flags;
4905
4906	if (DEBUG_LOCKS_WARN_ON(!lockdep_enabled()))
4907	return;
4908
4909	raw_local_irq_save(flags);
4910	lockdep_recursion_inc();
4911	register_lock_class(lock, subclass, force: 1);
4912	lockdep_recursion_finish();
4913	raw_local_irq_restore(flags);
4914	}
4915	}
4916	EXPORT_SYMBOL_GPL(lockdep_init_map_type);
4917
4918	struct lock_class_key __lockdep_no_validate__;
4919	EXPORT_SYMBOL_GPL(__lockdep_no_validate__);
4920
4921	#ifdef CONFIG_PROVE_LOCKING
4922	void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn,
4923	lock_print_fn print_fn)
4924	{
4925	struct lock_class *class = lock->class_cache[0];
4926	unsigned long flags;
4927
4928	raw_local_irq_save(flags);
4929	lockdep_recursion_inc();
4930
4931	if (!class)
4932	class = register_lock_class(lock, subclass: 0, force: 0);
4933
4934	if (class) {
4935	WARN_ON(class->cmp_fn && class->cmp_fn != cmp_fn);
4936	WARN_ON(class->print_fn && class->print_fn != print_fn);
4937
4938	class->cmp_fn = cmp_fn;
4939	class->print_fn = print_fn;
4940	}
4941
4942	lockdep_recursion_finish();
4943	raw_local_irq_restore(flags);
4944	}
4945	EXPORT_SYMBOL_GPL(lockdep_set_lock_cmp_fn);
4946	#endif
4947
4948	static void
4949	print_lock_nested_lock_not_held(struct task_struct *curr,
4950	struct held_lock *hlock)
4951	{
4952	if (!debug_locks_off())
4953	return;
4954	if (debug_locks_silent)
4955	return;
4956
4957	pr_warn("\n");
4958	pr_warn("==================================\n");
4959	pr_warn("WARNING: Nested lock was not taken\n");
4960	print_kernel_ident();
4961	pr_warn("----------------------------------\n");
4962
4963	pr_warn("%s/%d is trying to lock:\n", curr->comm, task_pid_nr(curr));
4964	print_lock(hlock);
4965
4966	pr_warn("\nbut this task is not holding:\n");
4967	pr_warn("%s\n", hlock->nest_lock->name);
4968
4969	pr_warn("\nstack backtrace:\n");
4970	dump_stack();
4971
4972	pr_warn("\nother info that might help us debug this:\n");
4973	lockdep_print_held_locks(p: curr);
4974
4975	pr_warn("\nstack backtrace:\n");
4976	dump_stack();
4977	}
4978
4979	static int __lock_is_held(const struct lockdep_map *lock, int read);
4980
4981	/*
4982	* This gets called for every mutex_lock*()/spin_lock*() operation.
4983	* We maintain the dependency maps and validate the locking attempt:
4984	*
4985	* The callers must make sure that IRQs are disabled before calling it,
4986	* otherwise we could get an interrupt which would want to take locks,
4987	* which would end up in lockdep again.
4988	*/
4989	static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
4990	int trylock, int read, int check, int hardirqs_off,
4991	struct lockdep_map *nest_lock, unsigned long ip,
4992	int references, int pin_count, int sync)
4993	{
4994	struct task_struct *curr = current;
4995	struct lock_class *class = NULL;
4996	struct held_lock *hlock;
4997	unsigned int depth;
4998	int chain_head = 0;
4999	int class_idx;
5000	u64 chain_key;
5001
5002	if (unlikely(!debug_locks))
5003	return 0;
5004
5005	if (!prove_locking || lock->key == &__lockdep_no_validate__)
5006	check = 0;
5007
5008	if (subclass < NR_LOCKDEP_CACHING_CLASSES)
5009	class = lock->class_cache[subclass];
5010	/*
5011	* Not cached?
5012	*/
5013	if (unlikely(!class)) {
5014	class = register_lock_class(lock, subclass, force: 0);
5015	if (!class)
5016	return 0;
5017	}
5018
5019	debug_class_ops_inc(class);
5020
5021	if (very_verbose(class)) {
5022	printk("\nacquire class [%px] %s", class->key, class->name);
5023	if (class->name_version > 1)
5024	printk(KERN_CONT "#%d", class->name_version);
5025	printk(KERN_CONT "\n");
5026	dump_stack();
5027	}
5028
5029	/*
5030	* Add the lock to the list of currently held locks.
5031	* (we dont increase the depth just yet, up until the
5032	* dependency checks are done)
5033	*/
5034	depth = curr->lockdep_depth;
5035	/*
5036	* Ran out of static storage for our per-task lock stack again have we?
5037	*/
5038	if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH))
5039	return 0;
5040
5041	class_idx = class - lock_classes;
5042
5043	if (depth && !sync) {
5044	/* we're holding locks and the new held lock is not a sync */
5045	hlock = curr->held_locks + depth - 1;
5046	if (hlock->class_idx == class_idx && nest_lock) {
5047	if (!references)
5048	references++;
5049
5050	if (!hlock->references)
5051	hlock->references++;
5052
5053	hlock->references += references;
5054
5055	/* Overflow */
5056	if (DEBUG_LOCKS_WARN_ON(hlock->references < references))
5057	return 0;
5058
5059	return 2;
5060	}
5061	}
5062
5063	hlock = curr->held_locks + depth;
5064	/*
5065	* Plain impossible, we just registered it and checked it weren't no
5066	* NULL like.. I bet this mushroom I ate was good!
5067	*/
5068	if (DEBUG_LOCKS_WARN_ON(!class))
5069	return 0;
5070	hlock->class_idx = class_idx;
5071	hlock->acquire_ip = ip;
5072	hlock->instance = lock;
5073	hlock->nest_lock = nest_lock;
5074	hlock->irq_context = task_irq_context(task: curr);
5075	hlock->trylock = trylock;
5076	hlock->read = read;
5077	hlock->check = check;
5078	hlock->sync = !!sync;
5079	hlock->hardirqs_off = !!hardirqs_off;
5080	hlock->references = references;
5081	#ifdef CONFIG_LOCK_STAT
5082	hlock->waittime_stamp = 0;
5083	hlock->holdtime_stamp = lockstat_clock();
5084	#endif
5085	hlock->pin_count = pin_count;
5086
5087	if (check_wait_context(curr, next: hlock))
5088	return 0;
5089
5090	/* Initialize the lock usage bit */
5091	if (!mark_usage(curr, hlock, check))
5092	return 0;
5093
5094	/*
5095	* Calculate the chain hash: it's the combined hash of all the
5096	* lock keys along the dependency chain. We save the hash value
5097	* at every step so that we can get the current hash easily
5098	* after unlock. The chain hash is then used to cache dependency
5099	* results.
5100	*
5101	* The 'key ID' is what is the most compact key value to drive
5102	* the hash, not class->key.
5103	*/
5104	/*
5105	* Whoops, we did it again.. class_idx is invalid.
5106	*/
5107	if (DEBUG_LOCKS_WARN_ON(!test_bit(class_idx, lock_classes_in_use)))
5108	return 0;
5109
5110	chain_key = curr->curr_chain_key;
5111	if (!depth) {
5112	/*
5113	* How can we have a chain hash when we ain't got no keys?!
5114	*/
5115	if (DEBUG_LOCKS_WARN_ON(chain_key != INITIAL_CHAIN_KEY))
5116	return 0;
5117	chain_head = 1;
5118	}
5119
5120	hlock->prev_chain_key = chain_key;
5121	if (separate_irq_context(curr, hlock)) {
5122	chain_key = INITIAL_CHAIN_KEY;
5123	chain_head = 1;
5124	}
5125	chain_key = iterate_chain_key(key: chain_key, idx: hlock_id(hlock));
5126
5127	if (nest_lock && !__lock_is_held(lock: nest_lock, read: -1)) {
5128	print_lock_nested_lock_not_held(curr, hlock);
5129	return 0;
5130	}
5131
5132	if (!debug_locks_silent) {
5133	WARN_ON_ONCE(depth && !hlock_class(hlock - 1)->key);
5134	WARN_ON_ONCE(!hlock_class(hlock)->key);
5135	}
5136
5137	if (!validate_chain(curr, hlock, chain_head, chain_key))
5138	return 0;
5139
5140	/* For lock_sync(), we are done here since no actual critical section */
5141	if (hlock->sync)
5142	return 1;
5143
5144	curr->curr_chain_key = chain_key;
5145	curr->lockdep_depth++;
5146	check_chain_key(curr);
5147	#ifdef CONFIG_DEBUG_LOCKDEP
5148	if (unlikely(!debug_locks))
5149	return 0;
5150	#endif
5151	if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) {
5152	debug_locks_off();
5153	print_lockdep_off(bug_msg: "BUG: MAX_LOCK_DEPTH too low!");
5154	printk(KERN_DEBUG "depth: %i max: %lu!\n",
5155	curr->lockdep_depth, MAX_LOCK_DEPTH);
5156
5157	lockdep_print_held_locks(current);
5158	debug_show_all_locks();
5159	dump_stack();
5160
5161	return 0;
5162	}
5163
5164	if (unlikely(curr->lockdep_depth > max_lockdep_depth))
5165	max_lockdep_depth = curr->lockdep_depth;
5166
5167	return 1;
5168	}
5169
5170	static void print_unlock_imbalance_bug(struct task_struct *curr,
5171	struct lockdep_map *lock,
5172	unsigned long ip)
5173	{
5174	if (!debug_locks_off())
5175	return;
5176	if (debug_locks_silent)
5177	return;
5178
5179	pr_warn("\n");
5180	pr_warn("=====================================\n");
5181	pr_warn("WARNING: bad unlock balance detected!\n");
5182	print_kernel_ident();
5183	pr_warn("-------------------------------------\n");
5184	pr_warn("%s/%d is trying to release lock (",
5185	curr->comm, task_pid_nr(curr));
5186	print_lockdep_cache(lock);
5187	pr_cont(") at:\n");
5188	print_ip_sym(KERN_WARNING, ip);
5189	pr_warn("but there are no more locks to release!\n");
5190	pr_warn("\nother info that might help us debug this:\n");
5191	lockdep_print_held_locks(p: curr);
5192
5193	pr_warn("\nstack backtrace:\n");
5194	dump_stack();
5195	}
5196
5197	static noinstr int match_held_lock(const struct held_lock *hlock,
5198	const struct lockdep_map *lock)
5199	{
5200	if (hlock->instance == lock)
5201	return 1;
5202
5203	if (hlock->references) {
5204	const struct lock_class *class = lock->class_cache[0];
5205
5206	if (!class)
5207	class = look_up_lock_class(lock, subclass: 0);
5208
5209	/*
5210	* If look_up_lock_class() failed to find a class, we're trying
5211	* to test if we hold a lock that has never yet been acquired.
5212	* Clearly if the lock hasn't been acquired _ever_, we're not
5213	* holding it either, so report failure.
5214	*/
5215	if (!class)
5216	return 0;
5217
5218	/*
5219	* References, but not a lock we're actually ref-counting?
5220	* State got messed up, follow the sites that change ->references
5221	* and try to make sense of it.
5222	*/
5223	if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))
5224	return 0;
5225
5226	if (hlock->class_idx == class - lock_classes)
5227	return 1;
5228	}
5229
5230	return 0;
5231	}
5232
5233	/* @depth must not be zero */
5234	static struct held_lock *find_held_lock(struct task_struct *curr,
5235	struct lockdep_map *lock,
5236	unsigned int depth, int *idx)
5237	{
5238	struct held_lock *ret, *hlock, *prev_hlock;
5239	int i;
5240
5241	i = depth - 1;
5242	hlock = curr->held_locks + i;
5243	ret = hlock;
5244	if (match_held_lock(hlock, lock))
5245	goto out;
5246
5247	ret = NULL;
5248	for (i--, prev_hlock = hlock--;
5249	i >= 0;
5250	i--, prev_hlock = hlock--) {
5251	/*
5252	* We must not cross into another context:
5253	*/
5254	if (prev_hlock->irq_context != hlock->irq_context) {
5255	ret = NULL;
5256	break;
5257	}
5258	if (match_held_lock(hlock, lock)) {
5259	ret = hlock;
5260	break;
5261	}
5262	}
5263
5264	out:
5265	*idx = i;
5266	return ret;
5267	}
5268
5269	static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
5270	int idx, unsigned int *merged)
5271	{
5272	struct held_lock *hlock;
5273	int first_idx = idx;
5274
5275	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
5276	return 0;
5277
5278	for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) {
5279	switch (__lock_acquire(lock: hlock->instance,
5280	subclass: hlock_class(hlock)->subclass,
5281	trylock: hlock->trylock,
5282	read: hlock->read, check: hlock->check,
5283	hardirqs_off: hlock->hardirqs_off,
5284	nest_lock: hlock->nest_lock, ip: hlock->acquire_ip,
5285	references: hlock->references, pin_count: hlock->pin_count, sync: 0)) {
5286	case 0:
5287	return 1;
5288	case 1:
5289	break;
5290	case 2:
5291	*merged += (idx == first_idx);
5292	break;
5293	default:
5294	WARN_ON(1);
5295	return 0;
5296	}
5297	}
5298	return 0;
5299	}
5300
5301	static int
5302	__lock_set_class(struct lockdep_map *lock, const char *name,
5303	struct lock_class_key *key, unsigned int subclass,
5304	unsigned long ip)
5305	{
5306	struct task_struct *curr = current;
5307	unsigned int depth, merged = 0;
5308	struct held_lock *hlock;
5309	struct lock_class *class;
5310	int i;
5311
5312	if (unlikely(!debug_locks))
5313	return 0;
5314
5315	depth = curr->lockdep_depth;
5316	/*
5317	* This function is about (re)setting the class of a held lock,
5318	* yet we're not actually holding any locks. Naughty user!
5319	*/
5320	if (DEBUG_LOCKS_WARN_ON(!depth))
5321	return 0;
5322
5323	hlock = find_held_lock(curr, lock, depth, idx: &i);
5324	if (!hlock) {
5325	print_unlock_imbalance_bug(curr, lock, ip);
5326	return 0;
5327	}
5328
5329	lockdep_init_map_type(lock, name, key, 0,
5330	lock->wait_type_inner,
5331	lock->wait_type_outer,
5332	lock->lock_type);
5333	class = register_lock_class(lock, subclass, force: 0);
5334	hlock->class_idx = class - lock_classes;
5335
5336	curr->lockdep_depth = i;
5337	curr->curr_chain_key = hlock->prev_chain_key;
5338
5339	if (reacquire_held_locks(curr, depth, idx: i, merged: &merged))
5340	return 0;
5341
5342	/*
5343	* I took it apart and put it back together again, except now I have
5344	* these 'spare' parts.. where shall I put them.
5345	*/
5346	if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - merged))
5347	return 0;
5348	return 1;
5349	}
5350
5351	static int __lock_downgrade(struct lockdep_map *lock, unsigned long ip)
5352	{
5353	struct task_struct *curr = current;
5354	unsigned int depth, merged = 0;
5355	struct held_lock *hlock;
5356	int i;
5357
5358	if (unlikely(!debug_locks))
5359	return 0;
5360
5361	depth = curr->lockdep_depth;
5362	/*
5363	* This function is about (re)setting the class of a held lock,
5364	* yet we're not actually holding any locks. Naughty user!
5365	*/
5366	if (DEBUG_LOCKS_WARN_ON(!depth))
5367	return 0;
5368
5369	hlock = find_held_lock(curr, lock, depth, idx: &i);
5370	if (!hlock) {
5371	print_unlock_imbalance_bug(curr, lock, ip);
5372	return 0;
5373	}
5374
5375	curr->lockdep_depth = i;
5376	curr->curr_chain_key = hlock->prev_chain_key;
5377
5378	WARN(hlock->read, "downgrading a read lock");
5379	hlock->read = 1;
5380	hlock->acquire_ip = ip;
5381
5382	if (reacquire_held_locks(curr, depth, idx: i, merged: &merged))
5383	return 0;
5384
5385	/* Merging can't happen with unchanged classes.. */
5386	if (DEBUG_LOCKS_WARN_ON(merged))
5387	return 0;
5388
5389	/*
5390	* I took it apart and put it back together again, except now I have
5391	* these 'spare' parts.. where shall I put them.
5392	*/
5393	if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth))
5394	return 0;
5395
5396	return 1;
5397	}
5398
5399	/*
5400	* Remove the lock from the list of currently held locks - this gets
5401	* called on mutex_unlock()/spin_unlock*() (or on a failed
5402	* mutex_lock_interruptible()).
5403	*/
5404	static int
5405	__lock_release(struct lockdep_map *lock, unsigned long ip)
5406	{
5407	struct task_struct *curr = current;
5408	unsigned int depth, merged = 1;
5409	struct held_lock *hlock;
5410	int i;
5411
5412	if (unlikely(!debug_locks))
5413	return 0;
5414
5415	depth = curr->lockdep_depth;
5416	/*
5417	* So we're all set to release this lock.. wait what lock? We don't
5418	* own any locks, you've been drinking again?
5419	*/
5420	if (depth <= 0) {
5421	print_unlock_imbalance_bug(curr, lock, ip);
5422	return 0;
5423	}
5424
5425	/*
5426	* Check whether the lock exists in the current stack
5427	* of held locks:
5428	*/
5429	hlock = find_held_lock(curr, lock, depth, idx: &i);
5430	if (!hlock) {
5431	print_unlock_imbalance_bug(curr, lock, ip);
5432	return 0;
5433	}
5434
5435	if (hlock->instance == lock)
5436	lock_release_holdtime(hlock);
5437
5438	WARN(hlock->pin_count, "releasing a pinned lock\n");
5439
5440	if (hlock->references) {
5441	hlock->references--;
5442	if (hlock->references) {
5443	/*
5444	* We had, and after removing one, still have
5445	* references, the current lock stack is still
5446	* valid. We're done!
5447	*/
5448	return 1;
5449	}
5450	}
5451
5452	/*
5453	* We have the right lock to unlock, 'hlock' points to it.
5454	* Now we remove it from the stack, and add back the other
5455	* entries (if any), recalculating the hash along the way:
5456	*/
5457
5458	curr->lockdep_depth = i;
5459	curr->curr_chain_key = hlock->prev_chain_key;
5460
5461	/*
5462	* The most likely case is when the unlock is on the innermost
5463	* lock. In this case, we are done!
5464	*/
5465	if (i == depth-1)
5466	return 1;
5467
5468	if (reacquire_held_locks(curr, depth, idx: i + 1, merged: &merged))
5469	return 0;
5470
5471	/*
5472	* We had N bottles of beer on the wall, we drank one, but now
5473	* there's not N-1 bottles of beer left on the wall...
5474	* Pouring two of the bottles together is acceptable.
5475	*/
5476	DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - merged);
5477
5478	/*
5479	* Since reacquire_held_locks() would have called check_chain_key()
5480	* indirectly via __lock_acquire(), we don't need to do it again
5481	* on return.
5482	*/
5483	return 0;
5484	}
5485
5486	static __always_inline
5487	int __lock_is_held(const struct lockdep_map *lock, int read)
5488	{
5489	struct task_struct *curr = current;
5490	int i;
5491
5492	for (i = 0; i < curr->lockdep_depth; i++) {
5493	struct held_lock *hlock = curr->held_locks + i;
5494
5495	if (match_held_lock(hlock, lock)) {
5496	if (read == -1 || !!hlock->read == read)
5497	return LOCK_STATE_HELD;
5498
5499	return LOCK_STATE_NOT_HELD;
5500	}
5501	}
5502
5503	return LOCK_STATE_NOT_HELD;
5504	}
5505
5506	static struct pin_cookie __lock_pin_lock(struct lockdep_map *lock)
5507	{
5508	struct pin_cookie cookie = NIL_COOKIE;
5509	struct task_struct *curr = current;
5510	int i;
5511
5512	if (unlikely(!debug_locks))
5513	return cookie;
5514
5515	for (i = 0; i < curr->lockdep_depth; i++) {
5516	struct held_lock *hlock = curr->held_locks + i;
5517
5518	if (match_held_lock(hlock, lock)) {
5519	/*
5520	* Grab 16bits of randomness; this is sufficient to not
5521	* be guessable and still allows some pin nesting in
5522	* our u32 pin_count.
5523	*/
5524	cookie.val = 1 + (sched_clock() & 0xffff);
5525	hlock->pin_count += cookie.val;
5526	return cookie;
5527	}
5528	}
5529
5530	WARN(1, "pinning an unheld lock\n");
5531	return cookie;
5532	}
5533
5534	static void __lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
5535	{
5536	struct task_struct *curr = current;
5537	int i;
5538
5539	if (unlikely(!debug_locks))
5540	return;
5541
5542	for (i = 0; i < curr->lockdep_depth; i++) {
5543	struct held_lock *hlock = curr->held_locks + i;
5544
5545	if (match_held_lock(hlock, lock)) {
5546	hlock->pin_count += cookie.val;
5547	return;
5548	}
5549	}
5550
5551	WARN(1, "pinning an unheld lock\n");
5552	}
5553
5554	static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
5555	{
5556	struct task_struct *curr = current;
5557	int i;
5558
5559	if (unlikely(!debug_locks))
5560	return;
5561
5562	for (i = 0; i < curr->lockdep_depth; i++) {
5563	struct held_lock *hlock = curr->held_locks + i;
5564
5565	if (match_held_lock(hlock, lock)) {
5566	if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))
5567	return;
5568
5569	hlock->pin_count -= cookie.val;
5570
5571	if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))
5572	hlock->pin_count = 0;
5573
5574	return;
5575	}
5576	}
5577
5578	WARN(1, "unpinning an unheld lock\n");
5579	}
5580
5581	/*
5582	* Check whether we follow the irq-flags state precisely:
5583	*/
5584	static noinstr void check_flags(unsigned long flags)
5585	{
5586	#if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_DEBUG_LOCKDEP)
5587	if (!debug_locks)
5588	return;
5589
5590	/* Get the warning out.. */
5591	instrumentation_begin();
5592
5593	if (irqs_disabled_flags(flags)) {
5594	if (DEBUG_LOCKS_WARN_ON(lockdep_hardirqs_enabled())) {
5595	printk("possible reason: unannotated irqs-off.\n");
5596	}
5597	} else {
5598	if (DEBUG_LOCKS_WARN_ON(!lockdep_hardirqs_enabled())) {
5599	printk("possible reason: unannotated irqs-on.\n");
5600	}
5601	}
5602
5603	#ifndef CONFIG_PREEMPT_RT
5604	/*
5605	* We dont accurately track softirq state in e.g.
5606	* hardirq contexts (such as on 4KSTACKS), so only
5607	* check if not in hardirq contexts:
5608	*/
5609	if (!hardirq_count()) {
5610	if (softirq_count()) {
5611	/* like the above, but with softirqs */
5612	DEBUG_LOCKS_WARN_ON(current->softirqs_enabled);
5613	} else {
5614	/* lick the above, does it taste good? */
5615	DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
5616	}
5617	}
5618	#endif
5619
5620	if (!debug_locks)
5621	print_irqtrace_events(current);
5622
5623	instrumentation_end();
5624	#endif
5625	}
5626
5627	void lock_set_class(struct lockdep_map *lock, const char *name,
5628	struct lock_class_key *key, unsigned int subclass,
5629	unsigned long ip)
5630	{
5631	unsigned long flags;
5632
5633	if (unlikely(!lockdep_enabled()))
5634	return;
5635
5636	raw_local_irq_save(flags);
5637	lockdep_recursion_inc();
5638	check_flags(flags);
5639	if (__lock_set_class(lock, name, key, subclass, ip))
5640	check_chain_key(current);
5641	lockdep_recursion_finish();
5642	raw_local_irq_restore(flags);
5643	}
5644	EXPORT_SYMBOL_GPL(lock_set_class);
5645
5646	void lock_downgrade(struct lockdep_map *lock, unsigned long ip)
5647	{
5648	unsigned long flags;
5649
5650	if (unlikely(!lockdep_enabled()))
5651	return;
5652
5653	raw_local_irq_save(flags);
5654	lockdep_recursion_inc();
5655	check_flags(flags);
5656	if (__lock_downgrade(lock, ip))
5657	check_chain_key(current);
5658	lockdep_recursion_finish();
5659	raw_local_irq_restore(flags);
5660	}
5661	EXPORT_SYMBOL_GPL(lock_downgrade);
5662
5663	/* NMI context !!! */
5664	static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass)
5665	{
5666	#ifdef CONFIG_PROVE_LOCKING
5667	struct lock_class *class = look_up_lock_class(lock, subclass);
5668	unsigned long mask = LOCKF_USED;
5669
5670	/* if it doesn't have a class (yet), it certainly hasn't been used yet */
5671	if (!class)
5672	return;
5673
5674	/*
5675	* READ locks only conflict with USED, such that if we only ever use
5676	* READ locks, there is no deadlock possible -- RCU.
5677	*/
5678	if (!hlock->read)
5679	mask |= LOCKF_USED_READ;
5680
5681	if (!(class->usage_mask & mask))
5682	return;
5683
5684	hlock->class_idx = class - lock_classes;
5685
5686	print_usage_bug(current, this: hlock, prev_bit: LOCK_USED, new_bit: LOCK_USAGE_STATES);
5687	#endif
5688	}
5689
5690	static bool lockdep_nmi(void)
5691	{
5692	if (raw_cpu_read(lockdep_recursion))
5693	return false;
5694
5695	if (!in_nmi())
5696	return false;
5697
5698	return true;
5699	}
5700
5701	/*
5702	* read_lock() is recursive if:
5703	* 1. We force lockdep think this way in selftests or
5704	* 2. The implementation is not queued read/write lock or
5705	* 3. The locker is at an in_interrupt() context.
5706	*/
5707	bool read_lock_is_recursive(void)
5708	{
5709	return force_read_lock_recursive ||
5710	!IS_ENABLED(CONFIG_QUEUED_RWLOCKS) ||
5711	in_interrupt();
5712	}
5713	EXPORT_SYMBOL_GPL(read_lock_is_recursive);
5714
5715	/*
5716	* We are not always called with irqs disabled - do that here,
5717	* and also avoid lockdep recursion:
5718	*/
5719	void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
5720	int trylock, int read, int check,
5721	struct lockdep_map *nest_lock, unsigned long ip)
5722	{
5723	unsigned long flags;
5724
5725	trace_lock_acquire(lock, subclass, trylock, read, check, next_lock: nest_lock, ip);
5726
5727	if (!debug_locks)
5728	return;
5729
5730	if (unlikely(!lockdep_enabled())) {
5731	/* XXX allow trylock from NMI ?!? */
5732	if (lockdep_nmi() && !trylock) {
5733	struct held_lock hlock;
5734
5735	hlock.acquire_ip = ip;
5736	hlock.instance = lock;
5737	hlock.nest_lock = nest_lock;
5738	hlock.irq_context = 2; // XXX
5739	hlock.trylock = trylock;
5740	hlock.read = read;
5741	hlock.check = check;
5742	hlock.hardirqs_off = true;
5743	hlock.references = 0;
5744
5745	verify_lock_unused(lock, hlock: &hlock, subclass);
5746	}
5747	return;
5748	}
5749
5750	raw_local_irq_save(flags);
5751	check_flags(flags);
5752
5753	lockdep_recursion_inc();
5754	__lock_acquire(lock, subclass, trylock, read, check,
5755	irqs_disabled_flags(flags), nest_lock, ip, references: 0, pin_count: 0, sync: 0);
5756	lockdep_recursion_finish();
5757	raw_local_irq_restore(flags);
5758	}
5759	EXPORT_SYMBOL_GPL(lock_acquire);
5760
5761	void lock_release(struct lockdep_map *lock, unsigned long ip)
5762	{
5763	unsigned long flags;
5764
5765	trace_lock_release(lock, ip);
5766
5767	if (unlikely(!lockdep_enabled()))
5768	return;
5769
5770	raw_local_irq_save(flags);
5771	check_flags(flags);
5772
5773	lockdep_recursion_inc();
5774	if (__lock_release(lock, ip))
5775	check_chain_key(current);
5776	lockdep_recursion_finish();
5777	raw_local_irq_restore(flags);
5778	}
5779	EXPORT_SYMBOL_GPL(lock_release);
5780
5781	/*
5782	* lock_sync() - A special annotation for synchronize_{s,}rcu()-like API.
5783	*
5784	* No actual critical section is created by the APIs annotated with this: these
5785	* APIs are used to wait for one or multiple critical sections (on other CPUs
5786	* or threads), and it means that calling these APIs inside these critical
5787	* sections is potential deadlock.
5788	*/
5789	void lock_sync(struct lockdep_map *lock, unsigned subclass, int read,
5790	int check, struct lockdep_map *nest_lock, unsigned long ip)
5791	{
5792	unsigned long flags;
5793
5794	if (unlikely(!lockdep_enabled()))
5795	return;
5796
5797	raw_local_irq_save(flags);
5798	check_flags(flags);
5799
5800	lockdep_recursion_inc();
5801	__lock_acquire(lock, subclass, trylock: 0, read, check,
5802	irqs_disabled_flags(flags), nest_lock, ip, references: 0, pin_count: 0, sync: 1);
5803	check_chain_key(current);
5804	lockdep_recursion_finish();
5805	raw_local_irq_restore(flags);
5806	}
5807	EXPORT_SYMBOL_GPL(lock_sync);
5808
5809	noinstr int lock_is_held_type(const struct lockdep_map *lock, int read)
5810	{
5811	unsigned long flags;
5812	int ret = LOCK_STATE_NOT_HELD;
5813
5814	/*
5815	* Avoid false negative lockdep_assert_held() and
5816	* lockdep_assert_not_held().
5817	*/
5818	if (unlikely(!lockdep_enabled()))
5819	return LOCK_STATE_UNKNOWN;
5820
5821	raw_local_irq_save(flags);
5822	check_flags(flags);
5823
5824	lockdep_recursion_inc();
5825	ret = __lock_is_held(lock, read);
5826	lockdep_recursion_finish();
5827	raw_local_irq_restore(flags);
5828
5829	return ret;
5830	}
5831	EXPORT_SYMBOL_GPL(lock_is_held_type);
5832	NOKPROBE_SYMBOL(lock_is_held_type);
5833
5834	struct pin_cookie lock_pin_lock(struct lockdep_map *lock)
5835	{
5836	struct pin_cookie cookie = NIL_COOKIE;
5837	unsigned long flags;
5838
5839	if (unlikely(!lockdep_enabled()))
5840	return cookie;
5841
5842	raw_local_irq_save(flags);
5843	check_flags(flags);
5844
5845	lockdep_recursion_inc();
5846	cookie = __lock_pin_lock(lock);
5847	lockdep_recursion_finish();
5848	raw_local_irq_restore(flags);
5849
5850	return cookie;
5851	}
5852	EXPORT_SYMBOL_GPL(lock_pin_lock);
5853
5854	void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
5855	{
5856	unsigned long flags;
5857
5858	if (unlikely(!lockdep_enabled()))
5859	return;
5860
5861	raw_local_irq_save(flags);
5862	check_flags(flags);
5863
5864	lockdep_recursion_inc();
5865	__lock_repin_lock(lock, cookie);
5866	lockdep_recursion_finish();
5867	raw_local_irq_restore(flags);
5868	}
5869	EXPORT_SYMBOL_GPL(lock_repin_lock);
5870
5871	void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
5872	{
5873	unsigned long flags;
5874
5875	if (unlikely(!lockdep_enabled()))
5876	return;
5877
5878	raw_local_irq_save(flags);
5879	check_flags(flags);
5880
5881	lockdep_recursion_inc();
5882	__lock_unpin_lock(lock, cookie);
5883	lockdep_recursion_finish();
5884	raw_local_irq_restore(flags);
5885	}
5886	EXPORT_SYMBOL_GPL(lock_unpin_lock);
5887
5888	#ifdef CONFIG_LOCK_STAT
5889	static void print_lock_contention_bug(struct task_struct *curr,
5890	struct lockdep_map *lock,
5891	unsigned long ip)
5892	{
5893	if (!debug_locks_off())
5894	return;
5895	if (debug_locks_silent)
5896	return;
5897
5898	pr_warn("\n");
5899	pr_warn("=================================\n");
5900	pr_warn("WARNING: bad contention detected!\n");
5901	print_kernel_ident();
5902	pr_warn("---------------------------------\n");
5903	pr_warn("%s/%d is trying to contend lock (",
5904	curr->comm, task_pid_nr(curr));
5905	print_lockdep_cache(lock);
5906	pr_cont(") at:\n");
5907	print_ip_sym(KERN_WARNING, ip);
5908	pr_warn("but there are no locks held!\n");
5909	pr_warn("\nother info that might help us debug this:\n");
5910	lockdep_print_held_locks(p: curr);
5911
5912	pr_warn("\nstack backtrace:\n");
5913	dump_stack();
5914	}
5915
5916	static void
5917	__lock_contended(struct lockdep_map *lock, unsigned long ip)
5918	{
5919	struct task_struct *curr = current;
5920	struct held_lock *hlock;
5921	struct lock_class_stats *stats;
5922	unsigned int depth;
5923	int i, contention_point, contending_point;
5924
5925	depth = curr->lockdep_depth;
5926	/*
5927	* Whee, we contended on this lock, except it seems we're not
5928	* actually trying to acquire anything much at all..
5929	*/
5930	if (DEBUG_LOCKS_WARN_ON(!depth))
5931	return;
5932
5933	hlock = find_held_lock(curr, lock, depth, idx: &i);
5934	if (!hlock) {
5935	print_lock_contention_bug(curr, lock, ip);
5936	return;
5937	}
5938
5939	if (hlock->instance != lock)
5940	return;
5941
5942	hlock->waittime_stamp = lockstat_clock();
5943
5944	contention_point = lock_point(points: hlock_class(hlock)->contention_point, ip);
5945	contending_point = lock_point(points: hlock_class(hlock)->contending_point,
5946	ip: lock->ip);
5947
5948	stats = get_lock_stats(class: hlock_class(hlock));
5949	if (contention_point < LOCKSTAT_POINTS)
5950	stats->contention_point[contention_point]++;
5951	if (contending_point < LOCKSTAT_POINTS)
5952	stats->contending_point[contending_point]++;
5953	if (lock->cpu != smp_processor_id())
5954	stats->bounces[bounce_contended + !!hlock->read]++;
5955	}
5956
5957	static void
5958	__lock_acquired(struct lockdep_map *lock, unsigned long ip)
5959	{
5960	struct task_struct *curr = current;
5961	struct held_lock *hlock;
5962	struct lock_class_stats *stats;
5963	unsigned int depth;
5964	u64 now, waittime = 0;
5965	int i, cpu;
5966
5967	depth = curr->lockdep_depth;
5968	/*
5969	* Yay, we acquired ownership of this lock we didn't try to
5970	* acquire, how the heck did that happen?
5971	*/
5972	if (DEBUG_LOCKS_WARN_ON(!depth))
5973	return;
5974
5975	hlock = find_held_lock(curr, lock, depth, idx: &i);
5976	if (!hlock) {
5977	print_lock_contention_bug(curr, lock, _RET_IP_);
5978	return;
5979	}
5980
5981	if (hlock->instance != lock)
5982	return;
5983
5984	cpu = smp_processor_id();
5985	if (hlock->waittime_stamp) {
5986	now = lockstat_clock();
5987	waittime = now - hlock->waittime_stamp;
5988	hlock->holdtime_stamp = now;
5989	}
5990
5991	stats = get_lock_stats(class: hlock_class(hlock));
5992	if (waittime) {
5993	if (hlock->read)
5994	lock_time_inc(lt: &stats->read_waittime, time: waittime);
5995	else
5996	lock_time_inc(lt: &stats->write_waittime, time: waittime);
5997	}
5998	if (lock->cpu != cpu)
5999	stats->bounces[bounce_acquired + !!hlock->read]++;
6000
6001	lock->cpu = cpu;
6002	lock->ip = ip;
6003	}
6004
6005	void lock_contended(struct lockdep_map *lock, unsigned long ip)
6006	{
6007	unsigned long flags;
6008
6009	trace_lock_contended(lock, ip);
6010
6011	if (unlikely(!lock_stat || !lockdep_enabled()))
6012	return;
6013
6014	raw_local_irq_save(flags);
6015	check_flags(flags);
6016	lockdep_recursion_inc();
6017	__lock_contended(lock, ip);
6018	lockdep_recursion_finish();
6019	raw_local_irq_restore(flags);
6020	}
6021	EXPORT_SYMBOL_GPL(lock_contended);
6022
6023	void lock_acquired(struct lockdep_map *lock, unsigned long ip)
6024	{
6025	unsigned long flags;
6026
6027	trace_lock_acquired(lock, ip);
6028
6029	if (unlikely(!lock_stat || !lockdep_enabled()))
6030	return;
6031
6032	raw_local_irq_save(flags);
6033	check_flags(flags);
6034	lockdep_recursion_inc();
6035	__lock_acquired(lock, ip);
6036	lockdep_recursion_finish();
6037	raw_local_irq_restore(flags);
6038	}
6039	EXPORT_SYMBOL_GPL(lock_acquired);
6040	#endif
6041
6042	/*
6043	* Used by the testsuite, sanitize the validator state
6044	* after a simulated failure:
6045	*/
6046
6047	void lockdep_reset(void)
6048	{
6049	unsigned long flags;
6050	int i;
6051
6052	raw_local_irq_save(flags);
6053	lockdep_init_task(current);
6054	memset(current->held_locks, 0, MAX_LOCK_DEPTH*sizeof(struct held_lock));
6055	nr_hardirq_chains = 0;
6056	nr_softirq_chains = 0;
6057	nr_process_chains = 0;
6058	debug_locks = 1;
6059	for (i = 0; i < CHAINHASH_SIZE; i++)
6060	INIT_HLIST_HEAD(chainhash_table + i);
6061	raw_local_irq_restore(flags);
6062	}
6063
6064	/* Remove a class from a lock chain. Must be called with the graph lock held. */
6065	static void remove_class_from_lock_chain(struct pending_free *pf,
6066	struct lock_chain *chain,
6067	struct lock_class *class)
6068	{
6069	#ifdef CONFIG_PROVE_LOCKING
6070	int i;
6071
6072	for (i = chain->base; i < chain->base + chain->depth; i++) {
6073	if (chain_hlock_class_idx(hlock_id: chain_hlocks[i]) != class - lock_classes)
6074	continue;
6075	/*
6076	* Each lock class occurs at most once in a lock chain so once
6077	* we found a match we can break out of this loop.
6078	*/
6079	goto free_lock_chain;
6080	}
6081	/* Since the chain has not been modified, return. */
6082	return;
6083
6084	free_lock_chain:
6085	free_chain_hlocks(base: chain->base, size: chain->depth);
6086	/* Overwrite the chain key for concurrent RCU readers. */
6087	WRITE_ONCE(chain->chain_key, INITIAL_CHAIN_KEY);
6088	dec_chains(irq_context: chain->irq_context);
6089
6090	/*
6091	* Note: calling hlist_del_rcu() from inside a
6092	* hlist_for_each_entry_rcu() loop is safe.
6093	*/
6094	hlist_del_rcu(n: &chain->entry);
6095	__set_bit(chain - lock_chains, pf->lock_chains_being_freed);
6096	nr_zapped_lock_chains++;
6097	#endif
6098	}
6099
6100	/* Must be called with the graph lock held. */
6101	static void remove_class_from_lock_chains(struct pending_free *pf,
6102	struct lock_class *class)
6103	{
6104	struct lock_chain *chain;
6105	struct hlist_head *head;
6106	int i;
6107
6108	for (i = 0; i < ARRAY_SIZE(chainhash_table); i++) {
6109	head = chainhash_table + i;
6110	hlist_for_each_entry_rcu(chain, head, entry) {
6111	remove_class_from_lock_chain(pf, chain, class);
6112	}
6113	}
6114	}
6115
6116	/*
6117	* Remove all references to a lock class. The caller must hold the graph lock.
6118	*/
6119	static void zap_class(struct pending_free *pf, struct lock_class *class)
6120	{
6121	struct lock_list *entry;
6122	int i;
6123
6124	WARN_ON_ONCE(!class->key);
6125
6126	/*
6127	* Remove all dependencies this lock is
6128	* involved in:
6129	*/
6130	for_each_set_bit(i, list_entries_in_use, ARRAY_SIZE(list_entries)) {
6131	entry = list_entries + i;
6132	if (entry->class != class && entry->links_to != class)
6133	continue;
6134	__clear_bit(i, list_entries_in_use);
6135	nr_list_entries--;
6136	list_del_rcu(entry: &entry->entry);
6137	}
6138	if (list_empty(head: &class->locks_after) &&
6139	list_empty(head: &class->locks_before)) {
6140	list_move_tail(list: &class->lock_entry, head: &pf->zapped);
6141	hlist_del_rcu(n: &class->hash_entry);
6142	WRITE_ONCE(class->key, NULL);
6143	WRITE_ONCE(class->name, NULL);
6144	nr_lock_classes--;
6145	__clear_bit(class - lock_classes, lock_classes_in_use);
6146	if (class - lock_classes == max_lock_class_idx)
6147	max_lock_class_idx--;
6148	} else {
6149	WARN_ONCE(true, "%s() failed for class %s\n", __func__,
6150	class->name);
6151	}
6152
6153	remove_class_from_lock_chains(pf, class);
6154	nr_zapped_classes++;
6155	}
6156
6157	static void reinit_class(struct lock_class *class)
6158	{
6159	WARN_ON_ONCE(!class->lock_entry.next);
6160	WARN_ON_ONCE(!list_empty(&class->locks_after));
6161	WARN_ON_ONCE(!list_empty(&class->locks_before));
6162	memset_startat(class, 0, key);
6163	WARN_ON_ONCE(!class->lock_entry.next);
6164	WARN_ON_ONCE(!list_empty(&class->locks_after));
6165	WARN_ON_ONCE(!list_empty(&class->locks_before));
6166	}
6167
6168	static inline int within(const void *addr, void *start, unsigned long size)
6169	{
6170	return addr >= start && addr < start + size;
6171	}
6172
6173	static bool inside_selftest(void)
6174	{
6175	return current == lockdep_selftest_task_struct;
6176	}
6177
6178	/* The caller must hold the graph lock. */
6179	static struct pending_free *get_pending_free(void)
6180	{
6181	return delayed_free.pf + delayed_free.index;
6182	}
6183
6184	static void free_zapped_rcu(struct rcu_head *cb);
6185
6186	/*
6187	* Schedule an RCU callback if no RCU callback is pending. Must be called with
6188	* the graph lock held.
6189	*/
6190	static void call_rcu_zapped(struct pending_free *pf)
6191	{
6192	WARN_ON_ONCE(inside_selftest());
6193
6194	if (list_empty(head: &pf->zapped))
6195	return;
6196
6197	if (delayed_free.scheduled)
6198	return;
6199
6200	delayed_free.scheduled = true;
6201
6202	WARN_ON_ONCE(delayed_free.pf + delayed_free.index != pf);
6203	delayed_free.index ^= 1;
6204
6205	call_rcu(head: &delayed_free.rcu_head, func: free_zapped_rcu);
6206	}
6207
6208	/* The caller must hold the graph lock. May be called from RCU context. */
6209	static void __free_zapped_classes(struct pending_free *pf)
6210	{
6211	struct lock_class *class;
6212
6213	check_data_structures();
6214
6215	list_for_each_entry(class, &pf->zapped, lock_entry)
6216	reinit_class(class);
6217
6218	list_splice_init(list: &pf->zapped, head: &free_lock_classes);
6219
6220	#ifdef CONFIG_PROVE_LOCKING
6221	bitmap_andnot(dst: lock_chains_in_use, src1: lock_chains_in_use,
6222	src2: pf->lock_chains_being_freed, ARRAY_SIZE(lock_chains));
6223	bitmap_clear(map: pf->lock_chains_being_freed, start: 0, ARRAY_SIZE(lock_chains));
6224	#endif
6225	}
6226
6227	static void free_zapped_rcu(struct rcu_head *ch)
6228	{
6229	struct pending_free *pf;
6230	unsigned long flags;
6231
6232	if (WARN_ON_ONCE(ch != &delayed_free.rcu_head))
6233	return;
6234
6235	raw_local_irq_save(flags);
6236	lockdep_lock();
6237
6238	/* closed head */
6239	pf = delayed_free.pf + (delayed_free.index ^ 1);
6240	__free_zapped_classes(pf);
6241	delayed_free.scheduled = false;
6242
6243	/*
6244	* If there's anything on the open list, close and start a new callback.
6245	*/
6246	call_rcu_zapped(pf: delayed_free.pf + delayed_free.index);
6247
6248	lockdep_unlock();
6249	raw_local_irq_restore(flags);
6250	}
6251
6252	/*
6253	* Remove all lock classes from the class hash table and from the
6254	* all_lock_classes list whose key or name is in the address range [start,
6255	* start + size). Move these lock classes to the zapped_classes list. Must
6256	* be called with the graph lock held.
6257	*/
6258	static void __lockdep_free_key_range(struct pending_free *pf, void *start,
6259	unsigned long size)
6260	{
6261	struct lock_class *class;
6262	struct hlist_head *head;
6263	int i;
6264
6265	/* Unhash all classes that were created by a module. */
6266	for (i = 0; i < CLASSHASH_SIZE; i++) {
6267	head = classhash_table + i;
6268	hlist_for_each_entry_rcu(class, head, hash_entry) {
6269	if (!within(addr: class->key, start, size) &&
6270	!within(addr: class->name, start, size))
6271	continue;
6272	zap_class(pf, class);
6273	}
6274	}
6275	}
6276
6277	/*
6278	* Used in module.c to remove lock classes from memory that is going to be
6279	* freed; and possibly re-used by other modules.
6280	*
6281	* We will have had one synchronize_rcu() before getting here, so we're
6282	* guaranteed nobody will look up these exact classes -- they're properly dead
6283	* but still allocated.
6284	*/
6285	static void lockdep_free_key_range_reg(void *start, unsigned long size)
6286	{
6287	struct pending_free *pf;
6288	unsigned long flags;
6289
6290	init_data_structures_once();
6291
6292	raw_local_irq_save(flags);
6293	lockdep_lock();
6294	pf = get_pending_free();
6295	__lockdep_free_key_range(pf, start, size);
6296	call_rcu_zapped(pf);
6297	lockdep_unlock();
6298	raw_local_irq_restore(flags);
6299
6300	/*
6301	* Wait for any possible iterators from look_up_lock_class() to pass
6302	* before continuing to free the memory they refer to.
6303	*/
6304	synchronize_rcu();
6305	}
6306
6307	/*
6308	* Free all lockdep keys in the range [start, start+size). Does not sleep.
6309	* Ignores debug_locks. Must only be used by the lockdep selftests.
6310	*/
6311	static void lockdep_free_key_range_imm(void *start, unsigned long size)
6312	{
6313	struct pending_free *pf = delayed_free.pf;
6314	unsigned long flags;
6315
6316	init_data_structures_once();
6317
6318	raw_local_irq_save(flags);
6319	lockdep_lock();
6320	__lockdep_free_key_range(pf, start, size);
6321	__free_zapped_classes(pf);
6322	lockdep_unlock();
6323	raw_local_irq_restore(flags);
6324	}
6325
6326	void lockdep_free_key_range(void *start, unsigned long size)
6327	{
6328	init_data_structures_once();
6329
6330	if (inside_selftest())
6331	lockdep_free_key_range_imm(start, size);
6332	else
6333	lockdep_free_key_range_reg(start, size);
6334	}
6335
6336	/*
6337	* Check whether any element of the @lock->class_cache[] array refers to a
6338	* registered lock class. The caller must hold either the graph lock or the
6339	* RCU read lock.
6340	*/
6341	static bool lock_class_cache_is_registered(struct lockdep_map *lock)
6342	{
6343	struct lock_class *class;
6344	struct hlist_head *head;
6345	int i, j;
6346
6347	for (i = 0; i < CLASSHASH_SIZE; i++) {
6348	head = classhash_table + i;
6349	hlist_for_each_entry_rcu(class, head, hash_entry) {
6350	for (j = 0; j < NR_LOCKDEP_CACHING_CLASSES; j++)
6351	if (lock->class_cache[j] == class)
6352	return true;
6353	}
6354	}
6355	return false;
6356	}
6357
6358	/* The caller must hold the graph lock. Does not sleep. */
6359	static void __lockdep_reset_lock(struct pending_free *pf,
6360	struct lockdep_map *lock)
6361	{
6362	struct lock_class *class;
6363	int j;
6364
6365	/*
6366	* Remove all classes this lock might have:
6367	*/
6368	for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
6369	/*
6370	* If the class exists we look it up and zap it:
6371	*/
6372	class = look_up_lock_class(lock, subclass: j);
6373	if (class)
6374	zap_class(pf, class);
6375	}
6376	/*
6377	* Debug check: in the end all mapped classes should
6378	* be gone.
6379	*/
6380	if (WARN_ON_ONCE(lock_class_cache_is_registered(lock)))
6381	debug_locks_off();
6382	}
6383
6384	/*
6385	* Remove all information lockdep has about a lock if debug_locks == 1. Free
6386	* released data structures from RCU context.
6387	*/
6388	static void lockdep_reset_lock_reg(struct lockdep_map *lock)
6389	{
6390	struct pending_free *pf;
6391	unsigned long flags;
6392	int locked;
6393
6394	raw_local_irq_save(flags);
6395	locked = graph_lock();
6396	if (!locked)
6397	goto out_irq;
6398
6399	pf = get_pending_free();
6400	__lockdep_reset_lock(pf, lock);
6401	call_rcu_zapped(pf);
6402
6403	graph_unlock();
6404	out_irq:
6405	raw_local_irq_restore(flags);
6406	}
6407
6408	/*
6409	* Reset a lock. Does not sleep. Ignores debug_locks. Must only be used by the
6410	* lockdep selftests.
6411	*/
6412	static void lockdep_reset_lock_imm(struct lockdep_map *lock)
6413	{
6414	struct pending_free *pf = delayed_free.pf;
6415	unsigned long flags;
6416
6417	raw_local_irq_save(flags);
6418	lockdep_lock();
6419	__lockdep_reset_lock(pf, lock);
6420	__free_zapped_classes(pf);
6421	lockdep_unlock();
6422	raw_local_irq_restore(flags);
6423	}
6424
6425	void lockdep_reset_lock(struct lockdep_map *lock)
6426	{
6427	init_data_structures_once();
6428
6429	if (inside_selftest())
6430	lockdep_reset_lock_imm(lock);
6431	else
6432	lockdep_reset_lock_reg(lock);
6433	}
6434
6435	/*
6436	* Unregister a dynamically allocated key.
6437	*
6438	* Unlike lockdep_register_key(), a search is always done to find a matching
6439	* key irrespective of debug_locks to avoid potential invalid access to freed
6440	* memory in lock_class entry.
6441	*/
6442	void lockdep_unregister_key(struct lock_class_key *key)
6443	{
6444	struct hlist_head *hash_head = keyhashentry(key);
6445	struct lock_class_key *k;
6446	struct pending_free *pf;
6447	unsigned long flags;
6448	bool found = false;
6449
6450	might_sleep();
6451
6452	if (WARN_ON_ONCE(static_obj(key)))
6453	return;
6454
6455	raw_local_irq_save(flags);
6456	lockdep_lock();
6457
6458	hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
6459	if (k == key) {
6460	hlist_del_rcu(n: &k->hash_entry);
6461	found = true;
6462	break;
6463	}
6464	}
6465	WARN_ON_ONCE(!found && debug_locks);
6466	if (found) {
6467	pf = get_pending_free();
6468	__lockdep_free_key_range(pf, start: key, size: 1);
6469	call_rcu_zapped(pf);
6470	}
6471	lockdep_unlock();
6472	raw_local_irq_restore(flags);
6473
6474	/* Wait until is_dynamic_key() has finished accessing k->hash_entry. */
6475	synchronize_rcu();
6476	}
6477	EXPORT_SYMBOL_GPL(lockdep_unregister_key);
6478
6479	void __init lockdep_init(void)
6480	{
6481	printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n");
6482
6483	printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES);
6484	printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH);
6485	printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS);
6486	printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE);
6487	printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES);
6488	printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS);
6489	printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE);
6490
6491	printk(" memory used by lock dependency info: %zu kB\n",
6492	(sizeof(lock_classes) +
6493	sizeof(lock_classes_in_use) +
6494	sizeof(classhash_table) +
6495	sizeof(list_entries) +
6496	sizeof(list_entries_in_use) +
6497	sizeof(chainhash_table) +
6498	sizeof(delayed_free)
6499	#ifdef CONFIG_PROVE_LOCKING
6500	+ sizeof(lock_cq)
6501	+ sizeof(lock_chains)
6502	+ sizeof(lock_chains_in_use)
6503	+ sizeof(chain_hlocks)
6504	#endif
6505	) / 1024
6506	);
6507
6508	#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
6509	printk(" memory used for stack traces: %zu kB\n",
6510	(sizeof(stack_trace) + sizeof(stack_trace_hash)) / 1024
6511	);
6512	#endif
6513
6514	printk(" per task-struct memory footprint: %zu bytes\n",
6515	sizeof(((struct task_struct *)NULL)->held_locks));
6516	}
6517
6518	static void
6519	print_freed_lock_bug(struct task_struct *curr, const void *mem_from,
6520	const void *mem_to, struct held_lock *hlock)
6521	{
6522	if (!debug_locks_off())
6523	return;
6524	if (debug_locks_silent)
6525	return;
6526
6527	pr_warn("\n");
6528	pr_warn("=========================\n");
6529	pr_warn("WARNING: held lock freed!\n");
6530	print_kernel_ident();
6531	pr_warn("-------------------------\n");
6532	pr_warn("%s/%d is freeing memory %px-%px, with a lock still held there!\n",
6533	curr->comm, task_pid_nr(curr), mem_from, mem_to-1);
6534	print_lock(hlock);
6535	lockdep_print_held_locks(p: curr);
6536
6537	pr_warn("\nstack backtrace:\n");
6538	dump_stack();
6539	}
6540
6541	static inline int not_in_range(const void* mem_from, unsigned long mem_len,
6542	const void* lock_from, unsigned long lock_len)
6543	{
6544	return lock_from + lock_len <= mem_from ||
6545	mem_from + mem_len <= lock_from;
6546	}
6547
6548	/*
6549	* Called when kernel memory is freed (or unmapped), or if a lock
6550	* is destroyed or reinitialized - this code checks whether there is
6551	* any held lock in the memory range of <from> to <to>:
6552	*/
6553	void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len)
6554	{
6555	struct task_struct *curr = current;
6556	struct held_lock *hlock;
6557	unsigned long flags;
6558	int i;
6559
6560	if (unlikely(!debug_locks))
6561	return;
6562
6563	raw_local_irq_save(flags);
6564	for (i = 0; i < curr->lockdep_depth; i++) {
6565	hlock = curr->held_locks + i;
6566
6567	if (not_in_range(mem_from, mem_len, lock_from: hlock->instance,
6568	lock_len: sizeof(*hlock->instance)))
6569	continue;
6570
6571	print_freed_lock_bug(curr, mem_from, mem_to: mem_from + mem_len, hlock);
6572	break;
6573	}
6574	raw_local_irq_restore(flags);
6575	}
6576	EXPORT_SYMBOL_GPL(debug_check_no_locks_freed);
6577
6578	static void print_held_locks_bug(void)
6579	{
6580	if (!debug_locks_off())
6581	return;
6582	if (debug_locks_silent)
6583	return;
6584
6585	pr_warn("\n");
6586	pr_warn("====================================\n");
6587	pr_warn("WARNING: %s/%d still has locks held!\n",
6588	current->comm, task_pid_nr(current));
6589	print_kernel_ident();
6590	pr_warn("------------------------------------\n");
6591	lockdep_print_held_locks(current);
6592	pr_warn("\nstack backtrace:\n");
6593	dump_stack();
6594	}
6595
6596	void debug_check_no_locks_held(void)
6597	{
6598	if (unlikely(current->lockdep_depth > 0))
6599	print_held_locks_bug();
6600	}
6601	EXPORT_SYMBOL_GPL(debug_check_no_locks_held);
6602
6603	#ifdef __KERNEL__
6604	void debug_show_all_locks(void)
6605	{
6606	struct task_struct *g, *p;
6607
6608	if (unlikely(!debug_locks)) {
6609	pr_warn("INFO: lockdep is turned off.\n");
6610	return;
6611	}
6612	pr_warn("\nShowing all locks held in the system:\n");
6613
6614	rcu_read_lock();
6615	for_each_process_thread(g, p) {
6616	if (!p->lockdep_depth)
6617	continue;
6618	lockdep_print_held_locks(p);
6619	touch_nmi_watchdog();
6620	touch_all_softlockup_watchdogs();
6621	}
6622	rcu_read_unlock();
6623
6624	pr_warn("\n");
6625	pr_warn("=============================================\n\n");
6626	}
6627	EXPORT_SYMBOL_GPL(debug_show_all_locks);
6628	#endif
6629
6630	/*
6631	* Careful: only use this function if you are sure that
6632	* the task cannot run in parallel!
6633	*/
6634	void debug_show_held_locks(struct task_struct *task)
6635	{
6636	if (unlikely(!debug_locks)) {
6637	printk("INFO: lockdep is turned off.\n");
6638	return;
6639	}
6640	lockdep_print_held_locks(p: task);
6641	}
6642	EXPORT_SYMBOL_GPL(debug_show_held_locks);
6643
6644	asmlinkage __visible void lockdep_sys_exit(void)
6645	{
6646	struct task_struct *curr = current;
6647
6648	if (unlikely(curr->lockdep_depth)) {
6649	if (!debug_locks_off())
6650	return;
6651	pr_warn("\n");
6652	pr_warn("================================================\n");
6653	pr_warn("WARNING: lock held when returning to user space!\n");
6654	print_kernel_ident();
6655	pr_warn("------------------------------------------------\n");
6656	pr_warn("%s/%d is leaving the kernel with locks still held!\n",
6657	curr->comm, curr->pid);
6658	lockdep_print_held_locks(p: curr);
6659	}
6660
6661	/*
6662	* The lock history for each syscall should be independent. So wipe the
6663	* slate clean on return to userspace.
6664	*/
6665	lockdep_invariant_state(force: false);
6666	}
6667
6668	void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
6669	{
6670	struct task_struct *curr = current;
6671	int dl = READ_ONCE(debug_locks);
6672	bool rcu = warn_rcu_enter();
6673
6674	/* Note: the following can be executed concurrently, so be careful. */
6675	pr_warn("\n");
6676	pr_warn("=============================\n");
6677	pr_warn("WARNING: suspicious RCU usage\n");
6678	print_kernel_ident();
6679	pr_warn("-----------------------------\n");
6680	pr_warn("%s:%d %s!\n", file, line, s);
6681	pr_warn("\nother info that might help us debug this:\n\n");
6682	pr_warn("\n%srcu_scheduler_active = %d, debug_locks = %d\n%s",
6683	!rcu_lockdep_current_cpu_online()
6684	? "RCU used illegally from offline CPU!\n"
6685	: "",
6686	rcu_scheduler_active, dl,
6687	dl ? "" : "Possible false positive due to lockdep disabling via debug_locks = 0\n");
6688
6689	/*
6690	* If a CPU is in the RCU-free window in idle (ie: in the section
6691	* between ct_idle_enter() and ct_idle_exit(), then RCU
6692	* considers that CPU to be in an "extended quiescent state",
6693	* which means that RCU will be completely ignoring that CPU.
6694	* Therefore, rcu_read_lock() and friends have absolutely no
6695	* effect on a CPU running in that state. In other words, even if
6696	* such an RCU-idle CPU has called rcu_read_lock(), RCU might well
6697	* delete data structures out from under it. RCU really has no
6698	* choice here: we need to keep an RCU-free window in idle where
6699	* the CPU may possibly enter into low power mode. This way we can
6700	* notice an extended quiescent state to other CPUs that started a grace
6701	* period. Otherwise we would delay any grace period as long as we run
6702	* in the idle task.
6703	*
6704	* So complain bitterly if someone does call rcu_read_lock(),
6705	* rcu_read_lock_bh() and so on from extended quiescent states.
6706	*/
6707	if (!rcu_is_watching())
6708	pr_warn("RCU used illegally from extended quiescent state!\n");
6709
6710	lockdep_print_held_locks(p: curr);
6711	pr_warn("\nstack backtrace:\n");
6712	dump_stack();
6713	warn_rcu_exit(rcu);
6714	}
6715	EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);
6716