1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* SLUB: A slab allocator that limits cache line use instead of queuing
4	* objects in per cpu and per node lists.
5	*
6	* The allocator synchronizes using per slab locks or atomic operations
7	* and only uses a centralized lock to manage a pool of partial slabs.
8	*
9	* (C) 2007 SGI, Christoph Lameter
10	* (C) 2011 Linux Foundation, Christoph Lameter
11	*/
12
13	#include <linux/mm.h>
14	#include <linux/swap.h> /* mm_account_reclaimed_pages() */
15	#include <linux/module.h>
16	#include <linux/bit_spinlock.h>
17	#include <linux/interrupt.h>
18	#include <linux/swab.h>
19	#include <linux/bitops.h>
20	#include <linux/slab.h>
21	#include "slab.h"
22	#include <linux/proc_fs.h>
23	#include <linux/seq_file.h>
24	#include <linux/kasan.h>
25	#include <linux/kmsan.h>
26	#include <linux/cpu.h>
27	#include <linux/cpuset.h>
28	#include <linux/mempolicy.h>
29	#include <linux/ctype.h>
30	#include <linux/stackdepot.h>
31	#include <linux/debugobjects.h>
32	#include <linux/kallsyms.h>
33	#include <linux/kfence.h>
34	#include <linux/memory.h>
35	#include <linux/math64.h>
36	#include <linux/fault-inject.h>
37	#include <linux/kmemleak.h>
38	#include <linux/stacktrace.h>
39	#include <linux/prefetch.h>
40	#include <linux/memcontrol.h>
41	#include <linux/random.h>
42	#include <kunit/test.h>
43	#include <kunit/test-bug.h>
44	#include <linux/sort.h>
45
46	#include <linux/debugfs.h>
47	#include <trace/events/kmem.h>
48
49	#include "internal.h"
50
51	/*
52	* Lock order:
53	* 1. slab_mutex (Global Mutex)
54	* 2. node->list_lock (Spinlock)
55	* 3. kmem_cache->cpu_slab->lock (Local lock)
56	* 4. slab_lock(slab) (Only on some arches)
57	* 5. object_map_lock (Only for debugging)
58	*
59	* slab_mutex
60	*
61	* The role of the slab_mutex is to protect the list of all the slabs
62	* and to synchronize major metadata changes to slab cache structures.
63	* Also synchronizes memory hotplug callbacks.
64	*
65	* slab_lock
66	*
67	* The slab_lock is a wrapper around the page lock, thus it is a bit
68	* spinlock.
69	*
70	* The slab_lock is only used on arches that do not have the ability
71	* to do a cmpxchg_double. It only protects:
72	*
73	* A. slab->freelist -> List of free objects in a slab
74	* B. slab->inuse -> Number of objects in use
75	* C. slab->objects -> Number of objects in slab
76	* D. slab->frozen -> frozen state
77	*
78	* Frozen slabs
79	*
80	* If a slab is frozen then it is exempt from list management. It is
81	* the cpu slab which is actively allocated from by the processor that
82	* froze it and it is not on any list. The processor that froze the
83	* slab is the one who can perform list operations on the slab. Other
84	* processors may put objects onto the freelist but the processor that
85	* froze the slab is the only one that can retrieve the objects from the
86	* slab's freelist.
87	*
88	* CPU partial slabs
89	*
90	* The partially empty slabs cached on the CPU partial list are used
91	* for performance reasons, which speeds up the allocation process.
92	* These slabs are not frozen, but are also exempt from list management,
93	* by clearing the PG_workingset flag when moving out of the node
94	* partial list. Please see __slab_free() for more details.
95	*
96	* To sum up, the current scheme is:
97	* - node partial slab: PG_Workingset && !frozen
98	* - cpu partial slab: !PG_Workingset && !frozen
99	* - cpu slab: !PG_Workingset && frozen
100	* - full slab: !PG_Workingset && !frozen
101	*
102	* list_lock
103	*
104	* The list_lock protects the partial and full list on each node and
105	* the partial slab counter. If taken then no new slabs may be added or
106	* removed from the lists nor make the number of partial slabs be modified.
107	* (Note that the total number of slabs is an atomic value that may be
108	* modified without taking the list lock).
109	*
110	* The list_lock is a centralized lock and thus we avoid taking it as
111	* much as possible. As long as SLUB does not have to handle partial
112	* slabs, operations can continue without any centralized lock. F.e.
113	* allocating a long series of objects that fill up slabs does not require
114	* the list lock.
115	*
116	* For debug caches, all allocations are forced to go through a list_lock
117	* protected region to serialize against concurrent validation.
118	*
119	* cpu_slab->lock local lock
120	*
121	* This locks protect slowpath manipulation of all kmem_cache_cpu fields
122	* except the stat counters. This is a percpu structure manipulated only by
123	* the local cpu, so the lock protects against being preempted or interrupted
124	* by an irq. Fast path operations rely on lockless operations instead.
125	*
126	* On PREEMPT_RT, the local lock neither disables interrupts nor preemption
127	* which means the lockless fastpath cannot be used as it might interfere with
128	* an in-progress slow path operations. In this case the local lock is always
129	* taken but it still utilizes the freelist for the common operations.
130	*
131	* lockless fastpaths
132	*
133	* The fast path allocation (slab_alloc_node()) and freeing (do_slab_free())
134	* are fully lockless when satisfied from the percpu slab (and when
135	* cmpxchg_double is possible to use, otherwise slab_lock is taken).
136	* They also don't disable preemption or migration or irqs. They rely on
137	* the transaction id (tid) field to detect being preempted or moved to
138	* another cpu.
139	*
140	* irq, preemption, migration considerations
141	*
142	* Interrupts are disabled as part of list_lock or local_lock operations, or
143	* around the slab_lock operation, in order to make the slab allocator safe
144	* to use in the context of an irq.
145	*
146	* In addition, preemption (or migration on PREEMPT_RT) is disabled in the
147	* allocation slowpath, bulk allocation, and put_cpu_partial(), so that the
148	* local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer
149	* doesn't have to be revalidated in each section protected by the local lock.
150	*
151	* SLUB assigns one slab for allocation to each processor.
152	* Allocations only occur from these slabs called cpu slabs.
153	*
154	* Slabs with free elements are kept on a partial list and during regular
155	* operations no list for full slabs is used. If an object in a full slab is
156	* freed then the slab will show up again on the partial lists.
157	* We track full slabs for debugging purposes though because otherwise we
158	* cannot scan all objects.
159	*
160	* Slabs are freed when they become empty. Teardown and setup is
161	* minimal so we rely on the page allocators per cpu caches for
162	* fast frees and allocs.
163	*
164	* slab->frozen The slab is frozen and exempt from list processing.
165	* This means that the slab is dedicated to a purpose
166	* such as satisfying allocations for a specific
167	* processor. Objects may be freed in the slab while
168	* it is frozen but slab_free will then skip the usual
169	* list operations. It is up to the processor holding
170	* the slab to integrate the slab into the slab lists
171	* when the slab is no longer needed.
172	*
173	* One use of this flag is to mark slabs that are
174	* used for allocations. Then such a slab becomes a cpu
175	* slab. The cpu slab may be equipped with an additional
176	* freelist that allows lockless access to
177	* free objects in addition to the regular freelist
178	* that requires the slab lock.
179	*
180	* SLAB_DEBUG_FLAGS Slab requires special handling due to debug
181	* options set. This moves slab handling out of
182	* the fast path and disables lockless freelists.
183	*/
184
185	/*
186	* We could simply use migrate_disable()/enable() but as long as it's a
187	* function call even on !PREEMPT_RT, use inline preempt_disable() there.
188	*/
189	#ifndef CONFIG_PREEMPT_RT
190	#define slub_get_cpu_ptr(var) get_cpu_ptr(var)
191	#define slub_put_cpu_ptr(var) put_cpu_ptr(var)
192	#define USE_LOCKLESS_FAST_PATH() (true)
193	#else
194	#define slub_get_cpu_ptr(var) \
195	({ \
196	migrate_disable(); \
197	this_cpu_ptr(var); \
198	})
199	#define slub_put_cpu_ptr(var) \
200	do { \
201	(void)(var); \
202	migrate_enable(); \
203	} while (0)
204	#define USE_LOCKLESS_FAST_PATH() (false)
205	#endif
206
207	#ifndef CONFIG_SLUB_TINY
208	#define __fastpath_inline __always_inline
209	#else
210	#define __fastpath_inline
211	#endif
212
213	#ifdef CONFIG_SLUB_DEBUG
214	#ifdef CONFIG_SLUB_DEBUG_ON
215	DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
216	#else
217	DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
218	#endif
219	#endif /* CONFIG_SLUB_DEBUG */
220
221	/* Structure holding parameters for get_partial() call chain */
222	struct partial_context {
223	gfp_t flags;
224	unsigned int orig_size;
225	void *object;
226	};
227
228	static inline bool kmem_cache_debug(struct kmem_cache *s)
229	{
230	return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
231	}
232
233	static inline bool slub_debug_orig_size(struct kmem_cache *s)
234	{
235	return (kmem_cache_debug_flags(s, SLAB_STORE_USER) &&
236	(s->flags & SLAB_KMALLOC));
237	}
238
239	void *fixup_red_left(struct kmem_cache *s, void *p)
240	{
241	if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
242	p += s->red_left_pad;
243
244	return p;
245	}
246
247	static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
248	{
249	#ifdef CONFIG_SLUB_CPU_PARTIAL
250	return !kmem_cache_debug(s);
251	#else
252	return false;
253	#endif
254	}
255
256	/*
257	* Issues still to be resolved:
258	*
259	* - Support PAGE_ALLOC_DEBUG. Should be easy to do.
260	*
261	* - Variable sizing of the per node arrays
262	*/
263
264	/* Enable to log cmpxchg failures */
265	#undef SLUB_DEBUG_CMPXCHG
266
267	#ifndef CONFIG_SLUB_TINY
268	/*
269	* Minimum number of partial slabs. These will be left on the partial
270	* lists even if they are empty. kmem_cache_shrink may reclaim them.
271	*/
272	#define MIN_PARTIAL 5
273
274	/*
275	* Maximum number of desirable partial slabs.
276	* The existence of more partial slabs makes kmem_cache_shrink
277	* sort the partial list by the number of objects in use.
278	*/
279	#define MAX_PARTIAL 10
280	#else
281	#define MIN_PARTIAL 0
282	#define MAX_PARTIAL 0
283	#endif
284
285	#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
286	SLAB_POISON | SLAB_STORE_USER)
287
288	/*
289	* These debug flags cannot use CMPXCHG because there might be consistency
290	* issues when checking or reading debug information
291	*/
292	#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
293	SLAB_TRACE)
294
295
296	/*
297	* Debugging flags that require metadata to be stored in the slab. These get
298	* disabled when slab_debug=O is used and a cache's min order increases with
299	* metadata.
300	*/
301	#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
302
303	#define OO_SHIFT 16
304	#define OO_MASK ((1 << OO_SHIFT) - 1)
305	#define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */
306
307	/* Internal SLUB flags */
308	/* Poison object */
309	#define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON)
310	/* Use cmpxchg_double */
311
312	#ifdef system_has_freelist_aba
313	#define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE)
314	#else
315	#define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED
316	#endif
317
318	/*
319	* Tracking user of a slab.
320	*/
321	#define TRACK_ADDRS_COUNT 16
322	struct track {
323	unsigned long addr; /* Called from address */
324	#ifdef CONFIG_STACKDEPOT
325	depot_stack_handle_t handle;
326	#endif
327	int cpu; /* Was running on cpu */
328	int pid; /* Pid context */
329	unsigned long when; /* When did the operation occur */
330	};
331
332	enum track_item { TRACK_ALLOC, TRACK_FREE };
333
334	#ifdef SLAB_SUPPORTS_SYSFS
335	static int sysfs_slab_add(struct kmem_cache *);
336	static int sysfs_slab_alias(struct kmem_cache *, const char *);
337	#else
338	static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
339	static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
340	{ return 0; }
341	#endif
342
343	#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
344	static void debugfs_slab_add(struct kmem_cache *);
345	#else
346	static inline void debugfs_slab_add(struct kmem_cache *s) { }
347	#endif
348
349	enum stat_item {
350	ALLOC_FASTPATH, /* Allocation from cpu slab */
351	ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */
352	FREE_FASTPATH, /* Free to cpu slab */
353	FREE_SLOWPATH, /* Freeing not to cpu slab */
354	FREE_FROZEN, /* Freeing to frozen slab */
355	FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */
356	FREE_REMOVE_PARTIAL, /* Freeing removes last object */
357	ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */
358	ALLOC_SLAB, /* Cpu slab acquired from page allocator */
359	ALLOC_REFILL, /* Refill cpu slab from slab freelist */
360	ALLOC_NODE_MISMATCH, /* Switching cpu slab */
361	FREE_SLAB, /* Slab freed to the page allocator */
362	CPUSLAB_FLUSH, /* Abandoning of the cpu slab */
363	DEACTIVATE_FULL, /* Cpu slab was full when deactivated */
364	DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */
365	DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */
366	DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */
367	DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
368	DEACTIVATE_BYPASS, /* Implicit deactivation */
369	ORDER_FALLBACK, /* Number of times fallback was necessary */
370	CMPXCHG_DOUBLE_CPU_FAIL,/* Failures of this_cpu_cmpxchg_double */
371	CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */
372	CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */
373	CPU_PARTIAL_FREE, /* Refill cpu partial on free */
374	CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */
375	CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */
376	NR_SLUB_STAT_ITEMS
377	};
378
379	#ifndef CONFIG_SLUB_TINY
380	/*
381	* When changing the layout, make sure freelist and tid are still compatible
382	* with this_cpu_cmpxchg_double() alignment requirements.
383	*/
384	struct kmem_cache_cpu {
385	union {
386	struct {
387	void **freelist; /* Pointer to next available object */
388	unsigned long tid; /* Globally unique transaction id */
389	};
390	freelist_aba_t freelist_tid;
391	};
392	struct slab *slab; /* The slab from which we are allocating */
393	#ifdef CONFIG_SLUB_CPU_PARTIAL
394	struct slab *partial; /* Partially allocated slabs */
395	#endif
396	local_lock_t lock; /* Protects the fields above */
397	#ifdef CONFIG_SLUB_STATS
398	unsigned int stat[NR_SLUB_STAT_ITEMS];
399	#endif
400	};
401	#endif /* CONFIG_SLUB_TINY */
402
403	static inline void stat(const struct kmem_cache *s, enum stat_item si)
404	{
405	#ifdef CONFIG_SLUB_STATS
406	/*
407	* The rmw is racy on a preemptible kernel but this is acceptable, so
408	* avoid this_cpu_add()'s irq-disable overhead.
409	*/
410	raw_cpu_inc(s->cpu_slab->stat[si]);
411	#endif
412	}
413
414	static inline
415	void stat_add(const struct kmem_cache *s, enum stat_item si, int v)
416	{
417	#ifdef CONFIG_SLUB_STATS
418	raw_cpu_add(s->cpu_slab->stat[si], v);
419	#endif
420	}
421
422	/*
423	* The slab lists for all objects.
424	*/
425	struct kmem_cache_node {
426	spinlock_t list_lock;
427	unsigned long nr_partial;
428	struct list_head partial;
429	#ifdef CONFIG_SLUB_DEBUG
430	atomic_long_t nr_slabs;
431	atomic_long_t total_objects;
432	struct list_head full;
433	#endif
434	};
435
436	static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
437	{
438	return s->node[node];
439	}
440
441	/*
442	* Iterator over all nodes. The body will be executed for each node that has
443	* a kmem_cache_node structure allocated (which is true for all online nodes)
444	*/
445	#define for_each_kmem_cache_node(__s, __node, __n) \
446	for (__node = 0; __node < nr_node_ids; __node++) \
447	if ((__n = get_node(__s, __node)))
448
449	/*
450	* Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
451	* Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily
452	* differ during memory hotplug/hotremove operations.
453	* Protected by slab_mutex.
454	*/
455	static nodemask_t slab_nodes;
456
457	#ifndef CONFIG_SLUB_TINY
458	/*
459	* Workqueue used for flush_cpu_slab().
460	*/
461	static struct workqueue_struct *flushwq;
462	#endif
463
464	/********************************************************************
465	* Core slab cache functions
466	*******************************************************************/
467
468	/*
469	* freeptr_t represents a SLUB freelist pointer, which might be encoded
470	* and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
471	*/
472	typedef struct { unsigned long v; } freeptr_t;
473
474	/*
475	* Returns freelist pointer (ptr). With hardening, this is obfuscated
476	* with an XOR of the address where the pointer is held and a per-cache
477	* random number.
478	*/
479	static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,
480	void *ptr, unsigned long ptr_addr)
481	{
482	unsigned long encoded;
483
484	#ifdef CONFIG_SLAB_FREELIST_HARDENED
485	encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr);
486	#else
487	encoded = (unsigned long)ptr;
488	#endif
489	return (freeptr_t){.v = encoded};
490	}
491
492	static inline void *freelist_ptr_decode(const struct kmem_cache *s,
493	freeptr_t ptr, unsigned long ptr_addr)
494	{
495	void *decoded;
496
497	#ifdef CONFIG_SLAB_FREELIST_HARDENED
498	decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr));
499	#else
500	decoded = (void *)ptr.v;
501	#endif
502	return decoded;
503	}
504
505	static inline void *get_freepointer(struct kmem_cache *s, void *object)
506	{
507	unsigned long ptr_addr;
508	freeptr_t p;
509
510	object = kasan_reset_tag(addr: object);
511	ptr_addr = (unsigned long)object + s->offset;
512	p = *(freeptr_t *)(ptr_addr);
513	return freelist_ptr_decode(s, ptr: p, ptr_addr);
514	}
515
516	#ifndef CONFIG_SLUB_TINY
517	static void prefetch_freepointer(const struct kmem_cache *s, void *object)
518	{
519	prefetchw(object + s->offset);
520	}
521	#endif
522
523	/*
524	* When running under KMSAN, get_freepointer_safe() may return an uninitialized
525	* pointer value in the case the current thread loses the race for the next
526	* memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in
527	* slab_alloc_node() will fail, so the uninitialized value won't be used, but
528	* KMSAN will still check all arguments of cmpxchg because of imperfect
529	* handling of inline assembly.
530	* To work around this problem, we apply __no_kmsan_checks to ensure that
531	* get_freepointer_safe() returns initialized memory.
532	*/
533	__no_kmsan_checks
534	static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
535	{
536	unsigned long freepointer_addr;
537	freeptr_t p;
538
539	if (!debug_pagealloc_enabled_static())
540	return get_freepointer(s, object);
541
542	object = kasan_reset_tag(addr: object);
543	freepointer_addr = (unsigned long)object + s->offset;
544	copy_from_kernel_nofault(dst: &p, src: (freeptr_t *)freepointer_addr, size: sizeof(p));
545	return freelist_ptr_decode(s, ptr: p, ptr_addr: freepointer_addr);
546	}
547
548	static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
549	{
550	unsigned long freeptr_addr = (unsigned long)object + s->offset;
551
552	#ifdef CONFIG_SLAB_FREELIST_HARDENED
553	BUG_ON(object == fp); /* naive detection of double free or corruption */
554	#endif
555
556	freeptr_addr = (unsigned long)kasan_reset_tag(addr: (void *)freeptr_addr);
557	*(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, ptr: fp, ptr_addr: freeptr_addr);
558	}
559
560	/* Loop over all objects in a slab */
561	#define for_each_object(__p, __s, __addr, __objects) \
562	for (__p = fixup_red_left(__s, __addr); \
563	__p < (__addr) + (__objects) * (__s)->size; \
564	__p += (__s)->size)
565
566	static inline unsigned int order_objects(unsigned int order, unsigned int size)
567	{
568	return ((unsigned int)PAGE_SIZE << order) / size;
569	}
570
571	static inline struct kmem_cache_order_objects oo_make(unsigned int order,
572	unsigned int size)
573	{
574	struct kmem_cache_order_objects x = {
575	(order << OO_SHIFT) + order_objects(order, size)
576	};
577
578	return x;
579	}
580
581	static inline unsigned int oo_order(struct kmem_cache_order_objects x)
582	{
583	return x.x >> OO_SHIFT;
584	}
585
586	static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
587	{
588	return x.x & OO_MASK;
589	}
590
591	#ifdef CONFIG_SLUB_CPU_PARTIAL
592	static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
593	{
594	unsigned int nr_slabs;
595
596	s->cpu_partial = nr_objects;
597
598	/*
599	* We take the number of objects but actually limit the number of
600	* slabs on the per cpu partial list, in order to limit excessive
601	* growth of the list. For simplicity we assume that the slabs will
602	* be half-full.
603	*/
604	nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
605	s->cpu_partial_slabs = nr_slabs;
606	}
607	#else
608	static inline void
609	slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
610	{
611	}
612	#endif /* CONFIG_SLUB_CPU_PARTIAL */
613
614	/*
615	* Per slab locking using the pagelock
616	*/
617	static __always_inline void slab_lock(struct slab *slab)
618	{
619	struct page *page = slab_page(slab);
620
621	VM_BUG_ON_PAGE(PageTail(page), page);
622	bit_spin_lock(bitnum: PG_locked, addr: &page->flags);
623	}
624
625	static __always_inline void slab_unlock(struct slab *slab)
626	{
627	struct page *page = slab_page(slab);
628
629	VM_BUG_ON_PAGE(PageTail(page), page);
630	bit_spin_unlock(bitnum: PG_locked, addr: &page->flags);
631	}
632
633	static inline bool
634	__update_freelist_fast(struct slab *slab,
635	void *freelist_old, unsigned long counters_old,
636	void *freelist_new, unsigned long counters_new)
637	{
638	#ifdef system_has_freelist_aba
639	freelist_aba_t old = { .freelist = freelist_old, .counter = counters_old };
640	freelist_aba_t new = { .freelist = freelist_new, .counter = counters_new };
641
642	return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full);
643	#else
644	return false;
645	#endif
646	}
647
648	static inline bool
649	__update_freelist_slow(struct slab *slab,
650	void *freelist_old, unsigned long counters_old,
651	void *freelist_new, unsigned long counters_new)
652	{
653	bool ret = false;
654
655	slab_lock(slab);
656	if (slab->freelist == freelist_old &&
657	slab->counters == counters_old) {
658	slab->freelist = freelist_new;
659	slab->counters = counters_new;
660	ret = true;
661	}
662	slab_unlock(slab);
663
664	return ret;
665	}
666
667	/*
668	* Interrupts must be disabled (for the fallback code to work right), typically
669	* by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is
670	* part of bit_spin_lock(), is sufficient because the policy is not to allow any
671	* allocation/ free operation in hardirq context. Therefore nothing can
672	* interrupt the operation.
673	*/
674	static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
675	void *freelist_old, unsigned long counters_old,
676	void *freelist_new, unsigned long counters_new,
677	const char *n)
678	{
679	bool ret;
680
681	if (USE_LOCKLESS_FAST_PATH())
682	lockdep_assert_irqs_disabled();
683
684	if (s->flags & __CMPXCHG_DOUBLE) {
685	ret = __update_freelist_fast(slab, freelist_old, counters_old,
686	freelist_new, counters_new);
687	} else {
688	ret = __update_freelist_slow(slab, freelist_old, counters_old,
689	freelist_new, counters_new);
690	}
691	if (likely(ret))
692	return true;
693
694	cpu_relax();
695	stat(s, si: CMPXCHG_DOUBLE_FAIL);
696
697	#ifdef SLUB_DEBUG_CMPXCHG
698	pr_info("%s %s: cmpxchg double redo ", n, s->name);
699	#endif
700
701	return false;
702	}
703
704	static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
705	void *freelist_old, unsigned long counters_old,
706	void *freelist_new, unsigned long counters_new,
707	const char *n)
708	{
709	bool ret;
710
711	if (s->flags & __CMPXCHG_DOUBLE) {
712	ret = __update_freelist_fast(slab, freelist_old, counters_old,
713	freelist_new, counters_new);
714	} else {
715	unsigned long flags;
716
717	local_irq_save(flags);
718	ret = __update_freelist_slow(slab, freelist_old, counters_old,
719	freelist_new, counters_new);
720	local_irq_restore(flags);
721	}
722	if (likely(ret))
723	return true;
724
725	cpu_relax();
726	stat(s, si: CMPXCHG_DOUBLE_FAIL);
727
728	#ifdef SLUB_DEBUG_CMPXCHG
729	pr_info("%s %s: cmpxchg double redo ", n, s->name);
730	#endif
731
732	return false;
733	}
734
735	#ifdef CONFIG_SLUB_DEBUG
736	static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
737	static DEFINE_SPINLOCK(object_map_lock);
738
739	static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
740	struct slab *slab)
741	{
742	void *addr = slab_address(slab);
743	void *p;
744
745	bitmap_zero(obj_map, slab->objects);
746
747	for (p = slab->freelist; p; p = get_freepointer(s, p))
748	set_bit(__obj_to_index(s, addr, p), obj_map);
749	}
750
751	#if IS_ENABLED(CONFIG_KUNIT)
752	static bool slab_add_kunit_errors(void)
753	{
754	struct kunit_resource *resource;
755
756	if (!kunit_get_current_test())
757	return false;
758
759	resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
760	if (!resource)
761	return false;
762
763	(*(int *)resource->data)++;
764	kunit_put_resource(resource);
765	return true;
766	}
767	#else
768	static inline bool slab_add_kunit_errors(void) { return false; }
769	#endif
770
771	static inline unsigned int size_from_object(struct kmem_cache *s)
772	{
773	if (s->flags & SLAB_RED_ZONE)
774	return s->size - s->red_left_pad;
775
776	return s->size;
777	}
778
779	static inline void *restore_red_left(struct kmem_cache *s, void *p)
780	{
781	if (s->flags & SLAB_RED_ZONE)
782	p -= s->red_left_pad;
783
784	return p;
785	}
786
787	/*
788	* Debug settings:
789	*/
790	#if defined(CONFIG_SLUB_DEBUG_ON)
791	static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
792	#else
793	static slab_flags_t slub_debug;
794	#endif
795
796	static char *slub_debug_string;
797	static int disable_higher_order_debug;
798
799	/*
800	* slub is about to manipulate internal object metadata. This memory lies
801	* outside the range of the allocated object, so accessing it would normally
802	* be reported by kasan as a bounds error. metadata_access_enable() is used
803	* to tell kasan that these accesses are OK.
804	*/
805	static inline void metadata_access_enable(void)
806	{
807	kasan_disable_current();
808	}
809
810	static inline void metadata_access_disable(void)
811	{
812	kasan_enable_current();
813	}
814
815	/*
816	* Object debugging
817	*/
818
819	/* Verify that a pointer has an address that is valid within a slab page */
820	static inline int check_valid_pointer(struct kmem_cache *s,
821	struct slab *slab, void *object)
822	{
823	void *base;
824
825	if (!object)
826	return 1;
827
828	base = slab_address(slab);
829	object = kasan_reset_tag(object);
830	object = restore_red_left(s, object);
831	if (object < base || object >= base + slab->objects * s->size ||
832	(object - base) % s->size) {
833	return 0;
834	}
835
836	return 1;
837	}
838
839	static void print_section(char *level, char *text, u8 *addr,
840	unsigned int length)
841	{
842	metadata_access_enable();
843	print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
844	16, 1, kasan_reset_tag((void *)addr), length, 1);
845	metadata_access_disable();
846	}
847
848	/*
849	* See comment in calculate_sizes().
850	*/
851	static inline bool freeptr_outside_object(struct kmem_cache *s)
852	{
853	return s->offset >= s->inuse;
854	}
855
856	/*
857	* Return offset of the end of info block which is inuse + free pointer if
858	* not overlapping with object.
859	*/
860	static inline unsigned int get_info_end(struct kmem_cache *s)
861	{
862	if (freeptr_outside_object(s))
863	return s->inuse + sizeof(void *);
864	else
865	return s->inuse;
866	}
867
868	static struct track *get_track(struct kmem_cache *s, void *object,
869	enum track_item alloc)
870	{
871	struct track *p;
872
873	p = object + get_info_end(s);
874
875	return kasan_reset_tag(p + alloc);
876	}
877
878	#ifdef CONFIG_STACKDEPOT
879	static noinline depot_stack_handle_t set_track_prepare(void)
880	{
881	depot_stack_handle_t handle;
882	unsigned long entries[TRACK_ADDRS_COUNT];
883	unsigned int nr_entries;
884
885	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
886	handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);
887
888	return handle;
889	}
890	#else
891	static inline depot_stack_handle_t set_track_prepare(void)
892	{
893	return 0;
894	}
895	#endif
896
897	static void set_track_update(struct kmem_cache *s, void *object,
898	enum track_item alloc, unsigned long addr,
899	depot_stack_handle_t handle)
900	{
901	struct track *p = get_track(s, object, alloc);
902
903	#ifdef CONFIG_STACKDEPOT
904	p->handle = handle;
905	#endif
906	p->addr = addr;
907	p->cpu = smp_processor_id();
908	p->pid = current->pid;
909	p->when = jiffies;
910	}
911
912	static __always_inline void set_track(struct kmem_cache *s, void *object,
913	enum track_item alloc, unsigned long addr)
914	{
915	depot_stack_handle_t handle = set_track_prepare();
916
917	set_track_update(s, object, alloc, addr, handle);
918	}
919
920	static void init_tracking(struct kmem_cache *s, void *object)
921	{
922	struct track *p;
923
924	if (!(s->flags & SLAB_STORE_USER))
925	return;
926
927	p = get_track(s, object, TRACK_ALLOC);
928	memset(p, 0, 2*sizeof(struct track));
929	}
930
931	static void print_track(const char *s, struct track *t, unsigned long pr_time)
932	{
933	depot_stack_handle_t handle __maybe_unused;
934
935	if (!t->addr)
936	return;
937
938	pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
939	s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
940	#ifdef CONFIG_STACKDEPOT
941	handle = READ_ONCE(t->handle);
942	if (handle)
943	stack_depot_print(handle);
944	else
945	pr_err("object allocation/free stack trace missing\n");
946	#endif
947	}
948
949	void print_tracking(struct kmem_cache *s, void *object)
950	{
951	unsigned long pr_time = jiffies;
952	if (!(s->flags & SLAB_STORE_USER))
953	return;
954
955	print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
956	print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
957	}
958
959	static void print_slab_info(const struct slab *slab)
960	{
961	struct folio *folio = (struct folio *)slab_folio(slab);
962
963	pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",
964	slab, slab->objects, slab->inuse, slab->freelist,
965	folio_flags(folio, 0));
966	}
967
968	/*
969	* kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API
970	* family will round up the real request size to these fixed ones, so
971	* there could be an extra area than what is requested. Save the original
972	* request size in the meta data area, for better debug and sanity check.
973	*/
974	static inline void set_orig_size(struct kmem_cache *s,
975	void *object, unsigned int orig_size)
976	{
977	void *p = kasan_reset_tag(object);
978	unsigned int kasan_meta_size;
979
980	if (!slub_debug_orig_size(s))
981	return;
982
983	/*
984	* KASAN can save its free meta data inside of the object at offset 0.
985	* If this meta data size is larger than 'orig_size', it will overlap
986	* the data redzone in [orig_size+1, object_size]. Thus, we adjust
987	* 'orig_size' to be as at least as big as KASAN's meta data.
988	*/
989	kasan_meta_size = kasan_metadata_size(s, true);
990	if (kasan_meta_size > orig_size)
991	orig_size = kasan_meta_size;
992
993	p += get_info_end(s);
994	p += sizeof(struct track) * 2;
995
996	*(unsigned int *)p = orig_size;
997	}
998
999	static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)
1000	{
1001	void *p = kasan_reset_tag(object);
1002
1003	if (!slub_debug_orig_size(s))
1004	return s->object_size;
1005
1006	p += get_info_end(s);
1007	p += sizeof(struct track) * 2;
1008
1009	return *(unsigned int *)p;
1010	}
1011
1012	void skip_orig_size_check(struct kmem_cache *s, const void *object)
1013	{
1014	set_orig_size(s, (void *)object, s->object_size);
1015	}
1016
1017	static void slab_bug(struct kmem_cache *s, char *fmt, ...)
1018	{
1019	struct va_format vaf;
1020	va_list args;
1021
1022	va_start(args, fmt);
1023	vaf.fmt = fmt;
1024	vaf.va = &args;
1025	pr_err("=============================================================================\n");
1026	pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
1027	pr_err("-----------------------------------------------------------------------------\n\n");
1028	va_end(args);
1029	}
1030
1031	__printf(2, 3)
1032	static void slab_fix(struct kmem_cache *s, char *fmt, ...)
1033	{
1034	struct va_format vaf;
1035	va_list args;
1036
1037	if (slab_add_kunit_errors())
1038	return;
1039
1040	va_start(args, fmt);
1041	vaf.fmt = fmt;
1042	vaf.va = &args;
1043	pr_err("FIX %s: %pV\n", s->name, &vaf);
1044	va_end(args);
1045	}
1046
1047	static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
1048	{
1049	unsigned int off; /* Offset of last byte */
1050	u8 *addr = slab_address(slab);
1051
1052	print_tracking(s, p);
1053
1054	print_slab_info(slab);
1055
1056	pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
1057	p, p - addr, get_freepointer(s, p));
1058
1059	if (s->flags & SLAB_RED_ZONE)
1060	print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
1061	s->red_left_pad);
1062	else if (p > addr + 16)
1063	print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
1064
1065	print_section(KERN_ERR, "Object ", p,
1066	min_t(unsigned int, s->object_size, PAGE_SIZE));
1067	if (s->flags & SLAB_RED_ZONE)
1068	print_section(KERN_ERR, "Redzone ", p + s->object_size,
1069	s->inuse - s->object_size);
1070
1071	off = get_info_end(s);
1072
1073	if (s->flags & SLAB_STORE_USER)
1074	off += 2 * sizeof(struct track);
1075
1076	if (slub_debug_orig_size(s))
1077	off += sizeof(unsigned int);
1078
1079	off += kasan_metadata_size(s, false);
1080
1081	if (off != size_from_object(s))
1082	/* Beginning of the filler is the free pointer */
1083	print_section(KERN_ERR, "Padding ", p + off,
1084	size_from_object(s) - off);
1085
1086	dump_stack();
1087	}
1088
1089	static void object_err(struct kmem_cache *s, struct slab *slab,
1090	u8 *object, char *reason)
1091	{
1092	if (slab_add_kunit_errors())
1093	return;
1094
1095	slab_bug(s, "%s", reason);
1096	print_trailer(s, slab, object);
1097	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1098	}
1099
1100	static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1101	void **freelist, void *nextfree)
1102	{
1103	if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
1104	!check_valid_pointer(s, slab, nextfree) && freelist) {
1105	object_err(s, slab, *freelist, "Freechain corrupt");
1106	*freelist = NULL;
1107	slab_fix(s, "Isolate corrupted freechain");
1108	return true;
1109	}
1110
1111	return false;
1112	}
1113
1114	static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
1115	const char *fmt, ...)
1116	{
1117	va_list args;
1118	char buf[100];
1119
1120	if (slab_add_kunit_errors())
1121	return;
1122
1123	va_start(args, fmt);
1124	vsnprintf(buf, sizeof(buf), fmt, args);
1125	va_end(args);
1126	slab_bug(s, "%s", buf);
1127	print_slab_info(slab);
1128	dump_stack();
1129	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1130	}
1131
1132	static void init_object(struct kmem_cache *s, void *object, u8 val)
1133	{
1134	u8 *p = kasan_reset_tag(object);
1135	unsigned int poison_size = s->object_size;
1136
1137	if (s->flags & SLAB_RED_ZONE) {
1138	memset(p - s->red_left_pad, val, s->red_left_pad);
1139
1140	if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1141	/*
1142	* Redzone the extra allocated space by kmalloc than
1143	* requested, and the poison size will be limited to
1144	* the original request size accordingly.
1145	*/
1146	poison_size = get_orig_size(s, object);
1147	}
1148	}
1149
1150	if (s->flags & __OBJECT_POISON) {
1151	memset(p, POISON_FREE, poison_size - 1);
1152	p[poison_size - 1] = POISON_END;
1153	}
1154
1155	if (s->flags & SLAB_RED_ZONE)
1156	memset(p + poison_size, val, s->inuse - poison_size);
1157	}
1158
1159	static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
1160	void *from, void *to)
1161	{
1162	slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
1163	memset(from, data, to - from);
1164	}
1165
1166	static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
1167	u8 *object, char *what,
1168	u8 *start, unsigned int value, unsigned int bytes)
1169	{
1170	u8 *fault;
1171	u8 *end;
1172	u8 *addr = slab_address(slab);
1173
1174	metadata_access_enable();
1175	fault = memchr_inv(kasan_reset_tag(start), value, bytes);
1176	metadata_access_disable();
1177	if (!fault)
1178	return 1;
1179
1180	end = start + bytes;
1181	while (end > fault && end[-1] == value)
1182	end--;
1183
1184	if (slab_add_kunit_errors())
1185	goto skip_bug_print;
1186
1187	slab_bug(s, "%s overwritten", what);
1188	pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
1189	fault, end - 1, fault - addr,
1190	fault[0], value);
1191	print_trailer(s, slab, object);
1192	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1193
1194	skip_bug_print:
1195	restore_bytes(s, what, value, fault, end);
1196	return 0;
1197	}
1198
1199	/*
1200	* Object layout:
1201	*
1202	* object address
1203	* Bytes of the object to be managed.
1204	* If the freepointer may overlay the object then the free
1205	* pointer is at the middle of the object.
1206	*
1207	* Poisoning uses 0x6b (POISON_FREE) and the last byte is
1208	* 0xa5 (POISON_END)
1209	*
1210	* object + s->object_size
1211	* Padding to reach word boundary. This is also used for Redzoning.
1212	* Padding is extended by another word if Redzoning is enabled and
1213	* object_size == inuse.
1214	*
1215	* We fill with 0xbb (RED_INACTIVE) for inactive objects and with
1216	* 0xcc (RED_ACTIVE) for objects in use.
1217	*
1218	* object + s->inuse
1219	* Meta data starts here.
1220	*
1221	* A. Free pointer (if we cannot overwrite object on free)
1222	* B. Tracking data for SLAB_STORE_USER
1223	* C. Original request size for kmalloc object (SLAB_STORE_USER enabled)
1224	* D. Padding to reach required alignment boundary or at minimum
1225	* one word if debugging is on to be able to detect writes
1226	* before the word boundary.
1227	*
1228	* Padding is done using 0x5a (POISON_INUSE)
1229	*
1230	* object + s->size
1231	* Nothing is used beyond s->size.
1232	*
1233	* If slabcaches are merged then the object_size and inuse boundaries are mostly
1234	* ignored. And therefore no slab options that rely on these boundaries
1235	* may be used with merged slabcaches.
1236	*/
1237
1238	static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
1239	{
1240	unsigned long off = get_info_end(s); /* The end of info */
1241
1242	if (s->flags & SLAB_STORE_USER) {
1243	/* We also have user information there */
1244	off += 2 * sizeof(struct track);
1245
1246	if (s->flags & SLAB_KMALLOC)
1247	off += sizeof(unsigned int);
1248	}
1249
1250	off += kasan_metadata_size(s, false);
1251
1252	if (size_from_object(s) == off)
1253	return 1;
1254
1255	return check_bytes_and_report(s, slab, p, "Object padding",
1256	p + off, POISON_INUSE, size_from_object(s) - off);
1257	}
1258
1259	/* Check the pad bytes at the end of a slab page */
1260	static void slab_pad_check(struct kmem_cache *s, struct slab *slab)
1261	{
1262	u8 *start;
1263	u8 *fault;
1264	u8 *end;
1265	u8 *pad;
1266	int length;
1267	int remainder;
1268
1269	if (!(s->flags & SLAB_POISON))
1270	return;
1271
1272	start = slab_address(slab);
1273	length = slab_size(slab);
1274	end = start + length;
1275	remainder = length % s->size;
1276	if (!remainder)
1277	return;
1278
1279	pad = end - remainder;
1280	metadata_access_enable();
1281	fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
1282	metadata_access_disable();
1283	if (!fault)
1284	return;
1285	while (end > fault && end[-1] == POISON_INUSE)
1286	end--;
1287
1288	slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu",
1289	fault, end - 1, fault - start);
1290	print_section(KERN_ERR, "Padding ", pad, remainder);
1291
1292	restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
1293	}
1294
1295	static int check_object(struct kmem_cache *s, struct slab *slab,
1296	void *object, u8 val)
1297	{
1298	u8 *p = object;
1299	u8 *endobject = object + s->object_size;
1300	unsigned int orig_size, kasan_meta_size;
1301
1302	if (s->flags & SLAB_RED_ZONE) {
1303	if (!check_bytes_and_report(s, slab, object, "Left Redzone",
1304	object - s->red_left_pad, val, s->red_left_pad))
1305	return 0;
1306
1307	if (!check_bytes_and_report(s, slab, object, "Right Redzone",
1308	endobject, val, s->inuse - s->object_size))
1309	return 0;
1310
1311	if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1312	orig_size = get_orig_size(s, object);
1313
1314	if (s->object_size > orig_size &&
1315	!check_bytes_and_report(s, slab, object,
1316	"kmalloc Redzone", p + orig_size,
1317	val, s->object_size - orig_size)) {
1318	return 0;
1319	}
1320	}
1321	} else {
1322	if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
1323	check_bytes_and_report(s, slab, p, "Alignment padding",
1324	endobject, POISON_INUSE,
1325	s->inuse - s->object_size);
1326	}
1327	}
1328
1329	if (s->flags & SLAB_POISON) {
1330	if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) {
1331	/*
1332	* KASAN can save its free meta data inside of the
1333	* object at offset 0. Thus, skip checking the part of
1334	* the redzone that overlaps with the meta data.
1335	*/
1336	kasan_meta_size = kasan_metadata_size(s, true);
1337	if (kasan_meta_size < s->object_size - 1 &&
1338	!check_bytes_and_report(s, slab, p, "Poison",
1339	p + kasan_meta_size, POISON_FREE,
1340	s->object_size - kasan_meta_size - 1))
1341	return 0;
1342	if (kasan_meta_size < s->object_size &&
1343	!check_bytes_and_report(s, slab, p, "End Poison",
1344	p + s->object_size - 1, POISON_END, 1))
1345	return 0;
1346	}
1347	/*
1348	* check_pad_bytes cleans up on its own.
1349	*/
1350	check_pad_bytes(s, slab, p);
1351	}
1352
1353	if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
1354	/*
1355	* Object and freepointer overlap. Cannot check
1356	* freepointer while object is allocated.
1357	*/
1358	return 1;
1359
1360	/* Check free pointer validity */
1361	if (!check_valid_pointer(s, slab, get_freepointer(s, p))) {
1362	object_err(s, slab, p, "Freepointer corrupt");
1363	/*
1364	* No choice but to zap it and thus lose the remainder
1365	* of the free objects in this slab. May cause
1366	* another error because the object count is now wrong.
1367	*/
1368	set_freepointer(s, p, NULL);
1369	return 0;
1370	}
1371	return 1;
1372	}
1373
1374	static int check_slab(struct kmem_cache *s, struct slab *slab)
1375	{
1376	int maxobj;
1377
1378	if (!folio_test_slab(slab_folio(slab))) {
1379	slab_err(s, slab, "Not a valid slab page");
1380	return 0;
1381	}
1382
1383	maxobj = order_objects(slab_order(slab), s->size);
1384	if (slab->objects > maxobj) {
1385	slab_err(s, slab, "objects %u > max %u",
1386	slab->objects, maxobj);
1387	return 0;
1388	}
1389	if (slab->inuse > slab->objects) {
1390	slab_err(s, slab, "inuse %u > max %u",
1391	slab->inuse, slab->objects);
1392	return 0;
1393	}
1394	/* Slab_pad_check fixes things up after itself */
1395	slab_pad_check(s, slab);
1396	return 1;
1397	}
1398
1399	/*
1400	* Determine if a certain object in a slab is on the freelist. Must hold the
1401	* slab lock to guarantee that the chains are in a consistent state.
1402	*/
1403	static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
1404	{
1405	int nr = 0;
1406	void *fp;
1407	void *object = NULL;
1408	int max_objects;
1409
1410	fp = slab->freelist;
1411	while (fp && nr <= slab->objects) {
1412	if (fp == search)
1413	return 1;
1414	if (!check_valid_pointer(s, slab, fp)) {
1415	if (object) {
1416	object_err(s, slab, object,
1417	"Freechain corrupt");
1418	set_freepointer(s, object, NULL);
1419	} else {
1420	slab_err(s, slab, "Freepointer corrupt");
1421	slab->freelist = NULL;
1422	slab->inuse = slab->objects;
1423	slab_fix(s, "Freelist cleared");
1424	return 0;
1425	}
1426	break;
1427	}
1428	object = fp;
1429	fp = get_freepointer(s, object);
1430	nr++;
1431	}
1432
1433	max_objects = order_objects(slab_order(slab), s->size);
1434	if (max_objects > MAX_OBJS_PER_PAGE)
1435	max_objects = MAX_OBJS_PER_PAGE;
1436
1437	if (slab->objects != max_objects) {
1438	slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
1439	slab->objects, max_objects);
1440	slab->objects = max_objects;
1441	slab_fix(s, "Number of objects adjusted");
1442	}
1443	if (slab->inuse != slab->objects - nr) {
1444	slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
1445	slab->inuse, slab->objects - nr);
1446	slab->inuse = slab->objects - nr;
1447	slab_fix(s, "Object count adjusted");
1448	}
1449	return search == NULL;
1450	}
1451
1452	static void trace(struct kmem_cache *s, struct slab *slab, void *object,
1453	int alloc)
1454	{
1455	if (s->flags & SLAB_TRACE) {
1456	pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
1457	s->name,
1458	alloc ? "alloc" : "free",
1459	object, slab->inuse,
1460	slab->freelist);
1461
1462	if (!alloc)
1463	print_section(KERN_INFO, "Object ", (void *)object,
1464	s->object_size);
1465
1466	dump_stack();
1467	}
1468	}
1469
1470	/*
1471	* Tracking of fully allocated slabs for debugging purposes.
1472	*/
1473	static void add_full(struct kmem_cache *s,
1474	struct kmem_cache_node *n, struct slab *slab)
1475	{
1476	if (!(s->flags & SLAB_STORE_USER))
1477	return;
1478
1479	lockdep_assert_held(&n->list_lock);
1480	list_add(&slab->slab_list, &n->full);
1481	}
1482
1483	static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
1484	{
1485	if (!(s->flags & SLAB_STORE_USER))
1486	return;
1487
1488	lockdep_assert_held(&n->list_lock);
1489	list_del(&slab->slab_list);
1490	}
1491
1492	static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1493	{
1494	return atomic_long_read(&n->nr_slabs);
1495	}
1496
1497	static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1498	{
1499	struct kmem_cache_node *n = get_node(s, node);
1500
1501	atomic_long_inc(&n->nr_slabs);
1502	atomic_long_add(objects, &n->total_objects);
1503	}
1504	static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1505	{
1506	struct kmem_cache_node *n = get_node(s, node);
1507
1508	atomic_long_dec(&n->nr_slabs);
1509	atomic_long_sub(objects, &n->total_objects);
1510	}
1511
1512	/* Object debug checks for alloc/free paths */
1513	static void setup_object_debug(struct kmem_cache *s, void *object)
1514	{
1515	if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
1516	return;
1517
1518	init_object(s, object, SLUB_RED_INACTIVE);
1519	init_tracking(s, object);
1520	}
1521
1522	static
1523	void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
1524	{
1525	if (!kmem_cache_debug_flags(s, SLAB_POISON))
1526	return;
1527
1528	metadata_access_enable();
1529	memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
1530	metadata_access_disable();
1531	}
1532
1533	static inline int alloc_consistency_checks(struct kmem_cache *s,
1534	struct slab *slab, void *object)
1535	{
1536	if (!check_slab(s, slab))
1537	return 0;
1538
1539	if (!check_valid_pointer(s, slab, object)) {
1540	object_err(s, slab, object, "Freelist Pointer check fails");
1541	return 0;
1542	}
1543
1544	if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
1545	return 0;
1546
1547	return 1;
1548	}
1549
1550	static noinline bool alloc_debug_processing(struct kmem_cache *s,
1551	struct slab *slab, void *object, int orig_size)
1552	{
1553	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1554	if (!alloc_consistency_checks(s, slab, object))
1555	goto bad;
1556	}
1557
1558	/* Success. Perform special debug activities for allocs */
1559	trace(s, slab, object, 1);
1560	set_orig_size(s, object, orig_size);
1561	init_object(s, object, SLUB_RED_ACTIVE);
1562	return true;
1563
1564	bad:
1565	if (folio_test_slab(slab_folio(slab))) {
1566	/*
1567	* If this is a slab page then lets do the best we can
1568	* to avoid issues in the future. Marking all objects
1569	* as used avoids touching the remaining objects.
1570	*/
1571	slab_fix(s, "Marking all objects used");
1572	slab->inuse = slab->objects;
1573	slab->freelist = NULL;
1574	}
1575	return false;
1576	}
1577
1578	static inline int free_consistency_checks(struct kmem_cache *s,
1579	struct slab *slab, void *object, unsigned long addr)
1580	{
1581	if (!check_valid_pointer(s, slab, object)) {
1582	slab_err(s, slab, "Invalid object pointer 0x%p", object);
1583	return 0;
1584	}
1585
1586	if (on_freelist(s, slab, object)) {
1587	object_err(s, slab, object, "Object already free");
1588	return 0;
1589	}
1590
1591	if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
1592	return 0;
1593
1594	if (unlikely(s != slab->slab_cache)) {
1595	if (!folio_test_slab(slab_folio(slab))) {
1596	slab_err(s, slab, "Attempt to free object(0x%p) outside of slab",
1597	object);
1598	} else if (!slab->slab_cache) {
1599	pr_err("SLUB <none>: no slab for object 0x%p.\n",
1600	object);
1601	dump_stack();
1602	} else
1603	object_err(s, slab, object,
1604	"page slab pointer corrupt.");
1605	return 0;
1606	}
1607	return 1;
1608	}
1609
1610	/*
1611	* Parse a block of slab_debug options. Blocks are delimited by ';'
1612	*
1613	* @str: start of block
1614	* @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
1615	* @slabs: return start of list of slabs, or NULL when there's no list
1616	* @init: assume this is initial parsing and not per-kmem-create parsing
1617	*
1618	* returns the start of next block if there's any, or NULL
1619	*/
1620	static char *
1621	parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
1622	{
1623	bool higher_order_disable = false;
1624
1625	/* Skip any completely empty blocks */
1626	while (*str && *str == ';')
1627	str++;
1628
1629	if (*str == ',') {
1630	/*
1631	* No options but restriction on slabs. This means full
1632	* debugging for slabs matching a pattern.
1633	*/
1634	*flags = DEBUG_DEFAULT_FLAGS;
1635	goto check_slabs;
1636	}
1637	*flags = 0;
1638
1639	/* Determine which debug features should be switched on */
1640	for (; *str && *str != ',' && *str != ';'; str++) {
1641	switch (tolower(*str)) {
1642	case '-':
1643	*flags = 0;
1644	break;
1645	case 'f':
1646	*flags |= SLAB_CONSISTENCY_CHECKS;
1647	break;
1648	case 'z':
1649	*flags |= SLAB_RED_ZONE;
1650	break;
1651	case 'p':
1652	*flags |= SLAB_POISON;
1653	break;
1654	case 'u':
1655	*flags |= SLAB_STORE_USER;
1656	break;
1657	case 't':
1658	*flags |= SLAB_TRACE;
1659	break;
1660	case 'a':
1661	*flags |= SLAB_FAILSLAB;
1662	break;
1663	case 'o':
1664	/*
1665	* Avoid enabling debugging on caches if its minimum
1666	* order would increase as a result.
1667	*/
1668	higher_order_disable = true;
1669	break;
1670	default:
1671	if (init)
1672	pr_err("slab_debug option '%c' unknown. skipped\n", *str);
1673	}
1674	}
1675	check_slabs:
1676	if (*str == ',')
1677	*slabs = ++str;
1678	else
1679	*slabs = NULL;
1680
1681	/* Skip over the slab list */
1682	while (*str && *str != ';')
1683	str++;
1684
1685	/* Skip any completely empty blocks */
1686	while (*str && *str == ';')
1687	str++;
1688
1689	if (init && higher_order_disable)
1690	disable_higher_order_debug = 1;
1691
1692	if (*str)
1693	return str;
1694	else
1695	return NULL;
1696	}
1697
1698	static int __init setup_slub_debug(char *str)
1699	{
1700	slab_flags_t flags;
1701	slab_flags_t global_flags;
1702	char *saved_str;
1703	char *slab_list;
1704	bool global_slub_debug_changed = false;
1705	bool slab_list_specified = false;
1706
1707	global_flags = DEBUG_DEFAULT_FLAGS;
1708	if (*str++ != '=' || !*str)
1709	/*
1710	* No options specified. Switch on full debugging.
1711	*/
1712	goto out;
1713
1714	saved_str = str;
1715	while (str) {
1716	str = parse_slub_debug_flags(str, &flags, &slab_list, true);
1717
1718	if (!slab_list) {
1719	global_flags = flags;
1720	global_slub_debug_changed = true;
1721	} else {
1722	slab_list_specified = true;
1723	if (flags & SLAB_STORE_USER)
1724	stack_depot_request_early_init();
1725	}
1726	}
1727
1728	/*
1729	* For backwards compatibility, a single list of flags with list of
1730	* slabs means debugging is only changed for those slabs, so the global
1731	* slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
1732	* on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
1733	* long as there is no option specifying flags without a slab list.
1734	*/
1735	if (slab_list_specified) {
1736	if (!global_slub_debug_changed)
1737	global_flags = slub_debug;
1738	slub_debug_string = saved_str;
1739	}
1740	out:
1741	slub_debug = global_flags;
1742	if (slub_debug & SLAB_STORE_USER)
1743	stack_depot_request_early_init();
1744	if (slub_debug != 0 || slub_debug_string)
1745	static_branch_enable(&slub_debug_enabled);
1746	else
1747	static_branch_disable(&slub_debug_enabled);
1748	if ((static_branch_unlikely(&init_on_alloc) ||
1749	static_branch_unlikely(&init_on_free)) &&
1750	(slub_debug & SLAB_POISON))
1751	pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
1752	return 1;
1753	}
1754
1755	__setup("slab_debug", setup_slub_debug);
1756	__setup_param("slub_debug", slub_debug, setup_slub_debug, 0);
1757
1758	/*
1759	* kmem_cache_flags - apply debugging options to the cache
1760	* @flags: flags to set
1761	* @name: name of the cache
1762	*
1763	* Debug option(s) are applied to @flags. In addition to the debug
1764	* option(s), if a slab name (or multiple) is specified i.e.
1765	* slab_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1766	* then only the select slabs will receive the debug option(s).
1767	*/
1768	slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
1769	{
1770	char *iter;
1771	size_t len;
1772	char *next_block;
1773	slab_flags_t block_flags;
1774	slab_flags_t slub_debug_local = slub_debug;
1775
1776	if (flags & SLAB_NO_USER_FLAGS)
1777	return flags;
1778
1779	/*
1780	* If the slab cache is for debugging (e.g. kmemleak) then
1781	* don't store user (stack trace) information by default,
1782	* but let the user enable it via the command line below.
1783	*/
1784	if (flags & SLAB_NOLEAKTRACE)
1785	slub_debug_local &= ~SLAB_STORE_USER;
1786
1787	len = strlen(name);
1788	next_block = slub_debug_string;
1789	/* Go through all blocks of debug options, see if any matches our slab's name */
1790	while (next_block) {
1791	next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
1792	if (!iter)
1793	continue;
1794	/* Found a block that has a slab list, search it */
1795	while (*iter) {
1796	char *end, *glob;
1797	size_t cmplen;
1798
1799	end = strchrnul(iter, ',');
1800	if (next_block && next_block < end)
1801	end = next_block - 1;
1802
1803	glob = strnchr(iter, end - iter, '*');
1804	if (glob)
1805	cmplen = glob - iter;
1806	else
1807	cmplen = max_t(size_t, len, (end - iter));
1808
1809	if (!strncmp(name, iter, cmplen)) {
1810	flags |= block_flags;
1811	return flags;
1812	}
1813
1814	if (!*end || *end == ';')
1815	break;
1816	iter = end + 1;
1817	}
1818	}
1819
1820	return flags | slub_debug_local;
1821	}
1822	#else /* !CONFIG_SLUB_DEBUG */
1823	static inline void setup_object_debug(struct kmem_cache *s, void *object) {}
1824	static inline
1825	void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
1826
1827	static inline bool alloc_debug_processing(struct kmem_cache *s,
1828	struct slab *slab, void *object, int orig_size) { return true; }
1829
1830	static inline bool free_debug_processing(struct kmem_cache *s,
1831	struct slab *slab, void *head, void *tail, int *bulk_cnt,
1832	unsigned long addr, depot_stack_handle_t handle) { return true; }
1833
1834	static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
1835	static inline int check_object(struct kmem_cache *s, struct slab *slab,
1836	void *object, u8 val) { return 1; }
1837	static inline depot_stack_handle_t set_track_prepare(void) { return 0; }
1838	static inline void set_track(struct kmem_cache *s, void *object,
1839	enum track_item alloc, unsigned long addr) {}
1840	static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1841	struct slab *slab) {}
1842	static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1843	struct slab *slab) {}
1844	slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
1845	{
1846	return flags;
1847	}
1848	#define slub_debug 0
1849
1850	#define disable_higher_order_debug 0
1851
1852	static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1853	{ return 0; }
1854	static inline void inc_slabs_node(struct kmem_cache *s, int node,
1855	int objects) {}
1856	static inline void dec_slabs_node(struct kmem_cache *s, int node,
1857	int objects) {}
1858
1859	#ifndef CONFIG_SLUB_TINY
1860	static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1861	void **freelist, void *nextfree)
1862	{
1863	return false;
1864	}
1865	#endif
1866	#endif /* CONFIG_SLUB_DEBUG */
1867
1868	static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
1869	{
1870	return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1871	NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
1872	}
1873
1874	#ifdef CONFIG_MEMCG_KMEM
1875	static inline void memcg_free_slab_cgroups(struct slab *slab)
1876	{
1877	kfree(objp: slab_objcgs(slab));
1878	slab->memcg_data = 0;
1879	}
1880
1881	static inline size_t obj_full_size(struct kmem_cache *s)
1882	{
1883	/*
1884	* For each accounted object there is an extra space which is used
1885	* to store obj_cgroup membership. Charge it too.
1886	*/
1887	return s->size + sizeof(struct obj_cgroup *);
1888	}
1889
1890	/*
1891	* Returns false if the allocation should fail.
1892	*/
1893	static bool __memcg_slab_pre_alloc_hook(struct kmem_cache *s,
1894	struct list_lru *lru,
1895	struct obj_cgroup **objcgp,
1896	size_t objects, gfp_t flags)
1897	{
1898	/*
1899	* The obtained objcg pointer is safe to use within the current scope,
1900	* defined by current task or set_active_memcg() pair.
1901	* obj_cgroup_get() is used to get a permanent reference.
1902	*/
1903	struct obj_cgroup *objcg = current_obj_cgroup();
1904	if (!objcg)
1905	return true;
1906
1907	if (lru) {
1908	int ret;
1909	struct mem_cgroup *memcg;
1910
1911	memcg = get_mem_cgroup_from_objcg(objcg);
1912	ret = memcg_list_lru_alloc(memcg, lru, gfp: flags);
1913	css_put(css: &memcg->css);
1914
1915	if (ret)
1916	return false;
1917	}
1918
1919	if (obj_cgroup_charge(objcg, gfp: flags, size: objects * obj_full_size(s)))
1920	return false;
1921
1922	*objcgp = objcg;
1923	return true;
1924	}
1925
1926	/*
1927	* Returns false if the allocation should fail.
1928	*/
1929	static __fastpath_inline
1930	bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
1931	struct obj_cgroup **objcgp, size_t objects,
1932	gfp_t flags)
1933	{
1934	if (!memcg_kmem_online())
1935	return true;
1936
1937	if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
1938	return true;
1939
1940	return likely(__memcg_slab_pre_alloc_hook(s, lru, objcgp, objects,
1941	flags));
1942	}
1943
1944	static void __memcg_slab_post_alloc_hook(struct kmem_cache *s,
1945	struct obj_cgroup *objcg,
1946	gfp_t flags, size_t size,
1947	void **p)
1948	{
1949	struct slab *slab;
1950	unsigned long off;
1951	size_t i;
1952
1953	flags &= gfp_allowed_mask;
1954
1955	for (i = 0; i < size; i++) {
1956	if (likely(p[i])) {
1957	slab = virt_to_slab(addr: p[i]);
1958
1959	if (!slab_objcgs(slab) &&
1960	memcg_alloc_slab_cgroups(slab, s, gfp: flags, new_slab: false)) {
1961	obj_cgroup_uncharge(objcg, size: obj_full_size(s));
1962	continue;
1963	}
1964
1965	off = obj_to_index(cache: s, slab, obj: p[i]);
1966	obj_cgroup_get(objcg);
1967	slab_objcgs(slab)[off] = objcg;
1968	mod_objcg_state(objcg, pgdat: slab_pgdat(slab),
1969	idx: cache_vmstat_idx(s), nr: obj_full_size(s));
1970	} else {
1971	obj_cgroup_uncharge(objcg, size: obj_full_size(s));
1972	}
1973	}
1974	}
1975
1976	static __fastpath_inline
1977	void memcg_slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg,
1978	gfp_t flags, size_t size, void **p)
1979	{
1980	if (likely(!memcg_kmem_online() || !objcg))
1981	return;
1982
1983	return __memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
1984	}
1985
1986	static void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
1987	void **p, int objects,
1988	struct obj_cgroup **objcgs)
1989	{
1990	for (int i = 0; i < objects; i++) {
1991	struct obj_cgroup *objcg;
1992	unsigned int off;
1993
1994	off = obj_to_index(cache: s, slab, obj: p[i]);
1995	objcg = objcgs[off];
1996	if (!objcg)
1997	continue;
1998
1999	objcgs[off] = NULL;
2000	obj_cgroup_uncharge(objcg, size: obj_full_size(s));
2001	mod_objcg_state(objcg, pgdat: slab_pgdat(slab), idx: cache_vmstat_idx(s),
2002	nr: -obj_full_size(s));
2003	obj_cgroup_put(objcg);
2004	}
2005	}
2006
2007	static __fastpath_inline
2008	void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2009	int objects)
2010	{
2011	struct obj_cgroup **objcgs;
2012
2013	if (!memcg_kmem_online())
2014	return;
2015
2016	objcgs = slab_objcgs(slab);
2017	if (likely(!objcgs))
2018	return;
2019
2020	__memcg_slab_free_hook(s, slab, p, objects, objcgs);
2021	}
2022
2023	static inline
2024	void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects,
2025	struct obj_cgroup *objcg)
2026	{
2027	if (objcg)
2028	obj_cgroup_uncharge(objcg, size: objects * obj_full_size(s));
2029	}
2030	#else /* CONFIG_MEMCG_KMEM */
2031	static inline void memcg_free_slab_cgroups(struct slab *slab)
2032	{
2033	}
2034
2035	static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
2036	struct list_lru *lru,
2037	struct obj_cgroup **objcgp,
2038	size_t objects, gfp_t flags)
2039	{
2040	return true;
2041	}
2042
2043	static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
2044	struct obj_cgroup *objcg,
2045	gfp_t flags, size_t size,
2046	void **p)
2047	{
2048	}
2049
2050	static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
2051	void **p, int objects)
2052	{
2053	}
2054
2055	static inline
2056	void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects,
2057	struct obj_cgroup *objcg)
2058	{
2059	}
2060	#endif /* CONFIG_MEMCG_KMEM */
2061
2062	/*
2063	* Hooks for other subsystems that check memory allocations. In a typical
2064	* production configuration these hooks all should produce no code at all.
2065	*
2066	* Returns true if freeing of the object can proceed, false if its reuse
2067	* was delayed by KASAN quarantine, or it was returned to KFENCE.
2068	*/
2069	static __always_inline
2070	bool slab_free_hook(struct kmem_cache *s, void *x, bool init)
2071	{
2072	kmemleak_free_recursive(ptr: x, flags: s->flags);
2073	kmsan_slab_free(s, object: x);
2074
2075	debug_check_no_locks_freed(from: x, len: s->object_size);
2076
2077	if (!(s->flags & SLAB_DEBUG_OBJECTS))
2078	debug_check_no_obj_freed(address: x, size: s->object_size);
2079
2080	/* Use KCSAN to help debug racy use-after-free. */
2081	if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
2082	__kcsan_check_access(ptr: x, size: s->object_size,
2083	KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
2084
2085	if (kfence_free(addr: x))
2086	return false;
2087
2088	/*
2089	* As memory initialization might be integrated into KASAN,
2090	* kasan_slab_free and initialization memset's must be
2091	* kept together to avoid discrepancies in behavior.
2092	*
2093	* The initialization memset's clear the object and the metadata,
2094	* but don't touch the SLAB redzone.
2095	*/
2096	if (unlikely(init)) {
2097	int rsize;
2098
2099	if (!kasan_has_integrated_init())
2100	memset(kasan_reset_tag(x), 0, s->object_size);
2101	rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
2102	memset((char *)kasan_reset_tag(x) + s->inuse, 0,
2103	s->size - s->inuse - rsize);
2104	}
2105	/* KASAN might put x into memory quarantine, delaying its reuse. */
2106	return !kasan_slab_free(s, object: x, init);
2107	}
2108
2109	static inline bool slab_free_freelist_hook(struct kmem_cache *s,
2110	void **head, void **tail,
2111	int *cnt)
2112	{
2113
2114	void *object;
2115	void *next = *head;
2116	void *old_tail = *tail;
2117	bool init;
2118
2119	if (is_kfence_address(addr: next)) {
2120	slab_free_hook(s, x: next, init: false);
2121	return false;
2122	}
2123
2124	/* Head and tail of the reconstructed freelist */
2125	*head = NULL;
2126	*tail = NULL;
2127
2128	init = slab_want_init_on_free(c: s);
2129
2130	do {
2131	object = next;
2132	next = get_freepointer(s, object);
2133
2134	/* If object's reuse doesn't have to be delayed */
2135	if (likely(slab_free_hook(s, object, init))) {
2136	/* Move object to the new freelist */
2137	set_freepointer(s, object, fp: *head);
2138	*head = object;
2139	if (!*tail)
2140	*tail = object;
2141	} else {
2142	/*
2143	* Adjust the reconstructed freelist depth
2144	* accordingly if object's reuse is delayed.
2145	*/
2146	--(*cnt);
2147	}
2148	} while (object != old_tail);
2149
2150	return *head != NULL;
2151	}
2152
2153	static void *setup_object(struct kmem_cache *s, void *object)
2154	{
2155	setup_object_debug(s, object);
2156	object = kasan_init_slab_obj(cache: s, object);
2157	if (unlikely(s->ctor)) {
2158	kasan_unpoison_new_object(cache: s, object);
2159	s->ctor(object);
2160	kasan_poison_new_object(cache: s, object);
2161	}
2162	return object;
2163	}
2164
2165	/*
2166	* Slab allocation and freeing
2167	*/
2168	static inline struct slab *alloc_slab_page(gfp_t flags, int node,
2169	struct kmem_cache_order_objects oo)
2170	{
2171	struct folio *folio;
2172	struct slab *slab;
2173	unsigned int order = oo_order(x: oo);
2174
2175	folio = (struct folio *)alloc_pages_node(nid: node, gfp_mask: flags, order);
2176	if (!folio)
2177	return NULL;
2178
2179	slab = folio_slab(folio);
2180	__folio_set_slab(folio);
2181	/* Make the flag visible before any changes to folio->mapping */
2182	smp_wmb();
2183	if (folio_is_pfmemalloc(folio))
2184	slab_set_pfmemalloc(slab);
2185
2186	return slab;
2187	}
2188
2189	#ifdef CONFIG_SLAB_FREELIST_RANDOM
2190	/* Pre-initialize the random sequence cache */
2191	static int init_cache_random_seq(struct kmem_cache *s)
2192	{
2193	unsigned int count = oo_objects(s->oo);
2194	int err;
2195
2196	/* Bailout if already initialised */
2197	if (s->random_seq)
2198	return 0;
2199
2200	err = cache_random_seq_create(s, count, GFP_KERNEL);
2201	if (err) {
2202	pr_err("SLUB: Unable to initialize free list for %s\n",
2203	s->name);
2204	return err;
2205	}
2206
2207	/* Transform to an offset on the set of pages */
2208	if (s->random_seq) {
2209	unsigned int i;
2210
2211	for (i = 0; i < count; i++)
2212	s->random_seq[i] *= s->size;
2213	}
2214	return 0;
2215	}
2216
2217	/* Initialize each random sequence freelist per cache */
2218	static void __init init_freelist_randomization(void)
2219	{
2220	struct kmem_cache *s;
2221
2222	mutex_lock(&slab_mutex);
2223
2224	list_for_each_entry(s, &slab_caches, list)
2225	init_cache_random_seq(s);
2226
2227	mutex_unlock(&slab_mutex);
2228	}
2229
2230	/* Get the next entry on the pre-computed freelist randomized */
2231	static void *next_freelist_entry(struct kmem_cache *s,
2232	unsigned long *pos, void *start,
2233	unsigned long page_limit,
2234	unsigned long freelist_count)
2235	{
2236	unsigned int idx;
2237
2238	/*
2239	* If the target page allocation failed, the number of objects on the
2240	* page might be smaller than the usual size defined by the cache.
2241	*/
2242	do {
2243	idx = s->random_seq[*pos];
2244	*pos += 1;
2245	if (*pos >= freelist_count)
2246	*pos = 0;
2247	} while (unlikely(idx >= page_limit));
2248
2249	return (char *)start + idx;
2250	}
2251
2252	/* Shuffle the single linked freelist based on a random pre-computed sequence */
2253	static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2254	{
2255	void *start;
2256	void *cur;
2257	void *next;
2258	unsigned long idx, pos, page_limit, freelist_count;
2259
2260	if (slab->objects < 2 || !s->random_seq)
2261	return false;
2262
2263	freelist_count = oo_objects(s->oo);
2264	pos = get_random_u32_below(freelist_count);
2265
2266	page_limit = slab->objects * s->size;
2267	start = fixup_red_left(s, slab_address(slab));
2268
2269	/* First entry is used as the base of the freelist */
2270	cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count);
2271	cur = setup_object(s, cur);
2272	slab->freelist = cur;
2273
2274	for (idx = 1; idx < slab->objects; idx++) {
2275	next = next_freelist_entry(s, &pos, start, page_limit,
2276	freelist_count);
2277	next = setup_object(s, next);
2278	set_freepointer(s, cur, next);
2279	cur = next;
2280	}
2281	set_freepointer(s, cur, NULL);
2282
2283	return true;
2284	}
2285	#else
2286	static inline int init_cache_random_seq(struct kmem_cache *s)
2287	{
2288	return 0;
2289	}
2290	static inline void init_freelist_randomization(void) { }
2291	static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2292	{
2293	return false;
2294	}
2295	#endif /* CONFIG_SLAB_FREELIST_RANDOM */
2296
2297	static __always_inline void account_slab(struct slab *slab, int order,
2298	struct kmem_cache *s, gfp_t gfp)
2299	{
2300	if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT))
2301	memcg_alloc_slab_cgroups(slab, s, gfp, new_slab: true);
2302
2303	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
2304	PAGE_SIZE << order);
2305	}
2306
2307	static __always_inline void unaccount_slab(struct slab *slab, int order,
2308	struct kmem_cache *s)
2309	{
2310	if (memcg_kmem_online())
2311	memcg_free_slab_cgroups(slab);
2312
2313	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
2314	-(PAGE_SIZE << order));
2315	}
2316
2317	static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
2318	{
2319	struct slab *slab;
2320	struct kmem_cache_order_objects oo = s->oo;
2321	gfp_t alloc_gfp;
2322	void *start, *p, *next;
2323	int idx;
2324	bool shuffle;
2325
2326	flags &= gfp_allowed_mask;
2327
2328	flags |= s->allocflags;
2329
2330	/*
2331	* Let the initial higher-order allocation fail under memory pressure
2332	* so we fall-back to the minimum order allocation.
2333	*/
2334	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
2335	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(x: oo) > oo_order(x: s->min))
2336	alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
2337
2338	slab = alloc_slab_page(flags: alloc_gfp, node, oo);
2339	if (unlikely(!slab)) {
2340	oo = s->min;
2341	alloc_gfp = flags;
2342	/*
2343	* Allocation may have failed due to fragmentation.
2344	* Try a lower order alloc if possible
2345	*/
2346	slab = alloc_slab_page(flags: alloc_gfp, node, oo);
2347	if (unlikely(!slab))
2348	return NULL;
2349	stat(s, si: ORDER_FALLBACK);
2350	}
2351
2352	slab->objects = oo_objects(x: oo);
2353	slab->inuse = 0;
2354	slab->frozen = 0;
2355
2356	account_slab(slab, order: oo_order(x: oo), s, gfp: flags);
2357
2358	slab->slab_cache = s;
2359
2360	kasan_poison_slab(slab);
2361
2362	start = slab_address(slab);
2363
2364	setup_slab_debug(s, slab, addr: start);
2365
2366	shuffle = shuffle_freelist(s, slab);
2367
2368	if (!shuffle) {
2369	start = fixup_red_left(s, p: start);
2370	start = setup_object(s, object: start);
2371	slab->freelist = start;
2372	for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
2373	next = p + s->size;
2374	next = setup_object(s, object: next);
2375	set_freepointer(s, object: p, fp: next);
2376	p = next;
2377	}
2378	set_freepointer(s, object: p, NULL);
2379	}
2380
2381	return slab;
2382	}
2383
2384	static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
2385	{
2386	if (unlikely(flags & GFP_SLAB_BUG_MASK))
2387	flags = kmalloc_fix_flags(flags);
2388
2389	WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
2390
2391	return allocate_slab(s,
2392	flags: flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
2393	}
2394
2395	static void __free_slab(struct kmem_cache *s, struct slab *slab)
2396	{
2397	struct folio *folio = slab_folio(slab);
2398	int order = folio_order(folio);
2399	int pages = 1 << order;
2400
2401	__slab_clear_pfmemalloc(slab);
2402	folio->mapping = NULL;
2403	/* Make the mapping reset visible before clearing the flag */
2404	smp_wmb();
2405	__folio_clear_slab(folio);
2406	mm_account_reclaimed_pages(pages);
2407	unaccount_slab(slab, order, s);
2408	__free_pages(page: &folio->page, order);
2409	}
2410
2411	static void rcu_free_slab(struct rcu_head *h)
2412	{
2413	struct slab *slab = container_of(h, struct slab, rcu_head);
2414
2415	__free_slab(s: slab->slab_cache, slab);
2416	}
2417
2418	static void free_slab(struct kmem_cache *s, struct slab *slab)
2419	{
2420	if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
2421	void *p;
2422
2423	slab_pad_check(s, slab);
2424	for_each_object(p, s, slab_address(slab), slab->objects)
2425	check_object(s, slab, object: p, SLUB_RED_INACTIVE);
2426	}
2427
2428	if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
2429	call_rcu(head: &slab->rcu_head, func: rcu_free_slab);
2430	else
2431	__free_slab(s, slab);
2432	}
2433
2434	static void discard_slab(struct kmem_cache *s, struct slab *slab)
2435	{
2436	dec_slabs_node(s, node: slab_nid(slab), objects: slab->objects);
2437	free_slab(s, slab);
2438	}
2439
2440	/*
2441	* SLUB reuses PG_workingset bit to keep track of whether it's on
2442	* the per-node partial list.
2443	*/
2444	static inline bool slab_test_node_partial(const struct slab *slab)
2445	{
2446	return folio_test_workingset(folio: (struct folio *)slab_folio(slab));
2447	}
2448
2449	static inline void slab_set_node_partial(struct slab *slab)
2450	{
2451	set_bit(nr: PG_workingset, addr: folio_flags(slab_folio(slab), n: 0));
2452	}
2453
2454	static inline void slab_clear_node_partial(struct slab *slab)
2455	{
2456	clear_bit(nr: PG_workingset, addr: folio_flags(slab_folio(slab), n: 0));
2457	}
2458
2459	/*
2460	* Management of partially allocated slabs.
2461	*/
2462	static inline void
2463	__add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
2464	{
2465	n->nr_partial++;
2466	if (tail == DEACTIVATE_TO_TAIL)
2467	list_add_tail(new: &slab->slab_list, head: &n->partial);
2468	else
2469	list_add(new: &slab->slab_list, head: &n->partial);
2470	slab_set_node_partial(slab);
2471	}
2472
2473	static inline void add_partial(struct kmem_cache_node *n,
2474	struct slab *slab, int tail)
2475	{
2476	lockdep_assert_held(&n->list_lock);
2477	__add_partial(n, slab, tail);
2478	}
2479
2480	static inline void remove_partial(struct kmem_cache_node *n,
2481	struct slab *slab)
2482	{
2483	lockdep_assert_held(&n->list_lock);
2484	list_del(entry: &slab->slab_list);
2485	slab_clear_node_partial(slab);
2486	n->nr_partial--;
2487	}
2488
2489	/*
2490	* Called only for kmem_cache_debug() caches instead of remove_partial(), with a
2491	* slab from the n->partial list. Remove only a single object from the slab, do
2492	* the alloc_debug_processing() checks and leave the slab on the list, or move
2493	* it to full list if it was the last free object.
2494	*/
2495	static void *alloc_single_from_partial(struct kmem_cache *s,
2496	struct kmem_cache_node *n, struct slab *slab, int orig_size)
2497	{
2498	void *object;
2499
2500	lockdep_assert_held(&n->list_lock);
2501
2502	object = slab->freelist;
2503	slab->freelist = get_freepointer(s, object);
2504	slab->inuse++;
2505
2506	if (!alloc_debug_processing(s, slab, object, orig_size)) {
2507	remove_partial(n, slab);
2508	return NULL;
2509	}
2510
2511	if (slab->inuse == slab->objects) {
2512	remove_partial(n, slab);
2513	add_full(s, n, slab);
2514	}
2515
2516	return object;
2517	}
2518
2519	/*
2520	* Called only for kmem_cache_debug() caches to allocate from a freshly
2521	* allocated slab. Allocate a single object instead of whole freelist
2522	* and put the slab to the partial (or full) list.
2523	*/
2524	static void *alloc_single_from_new_slab(struct kmem_cache *s,
2525	struct slab *slab, int orig_size)
2526	{
2527	int nid = slab_nid(slab);
2528	struct kmem_cache_node *n = get_node(s, node: nid);
2529	unsigned long flags;
2530	void *object;
2531
2532
2533	object = slab->freelist;
2534	slab->freelist = get_freepointer(s, object);
2535	slab->inuse = 1;
2536
2537	if (!alloc_debug_processing(s, slab, object, orig_size))
2538	/*
2539	* It's not really expected that this would fail on a
2540	* freshly allocated slab, but a concurrent memory
2541	* corruption in theory could cause that.
2542	*/
2543	return NULL;
2544
2545	spin_lock_irqsave(&n->list_lock, flags);
2546
2547	if (slab->inuse == slab->objects)
2548	add_full(s, n, slab);
2549	else
2550	add_partial(n, slab, tail: DEACTIVATE_TO_HEAD);
2551
2552	inc_slabs_node(s, node: nid, objects: slab->objects);
2553	spin_unlock_irqrestore(lock: &n->list_lock, flags);
2554
2555	return object;
2556	}
2557
2558	#ifdef CONFIG_SLUB_CPU_PARTIAL
2559	static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
2560	#else
2561	static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
2562	int drain) { }
2563	#endif
2564	static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
2565
2566	/*
2567	* Try to allocate a partial slab from a specific node.
2568	*/
2569	static struct slab *get_partial_node(struct kmem_cache *s,
2570	struct kmem_cache_node *n,
2571	struct partial_context *pc)
2572	{
2573	struct slab *slab, *slab2, *partial = NULL;
2574	unsigned long flags;
2575	unsigned int partial_slabs = 0;
2576
2577	/*
2578	* Racy check. If we mistakenly see no partial slabs then we
2579	* just allocate an empty slab. If we mistakenly try to get a
2580	* partial slab and there is none available then get_partial()
2581	* will return NULL.
2582	*/
2583	if (!n || !n->nr_partial)
2584	return NULL;
2585
2586	spin_lock_irqsave(&n->list_lock, flags);
2587	list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
2588	if (!pfmemalloc_match(slab, gfpflags: pc->flags))
2589	continue;
2590
2591	if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
2592	void *object = alloc_single_from_partial(s, n, slab,
2593	orig_size: pc->orig_size);
2594	if (object) {
2595	partial = slab;
2596	pc->object = object;
2597	break;
2598	}
2599	continue;
2600	}
2601
2602	remove_partial(n, slab);
2603
2604	if (!partial) {
2605	partial = slab;
2606	stat(s, si: ALLOC_FROM_PARTIAL);
2607	} else {
2608	put_cpu_partial(s, slab, drain: 0);
2609	stat(s, si: CPU_PARTIAL_NODE);
2610	partial_slabs++;
2611	}
2612	#ifdef CONFIG_SLUB_CPU_PARTIAL
2613	if (!kmem_cache_has_cpu_partial(s)
2614	|| partial_slabs > s->cpu_partial_slabs / 2)
2615	break;
2616	#else
2617	break;
2618	#endif
2619
2620	}
2621	spin_unlock_irqrestore(lock: &n->list_lock, flags);
2622	return partial;
2623	}
2624
2625	/*
2626	* Get a slab from somewhere. Search in increasing NUMA distances.
2627	*/
2628	static struct slab *get_any_partial(struct kmem_cache *s,
2629	struct partial_context *pc)
2630	{
2631	#ifdef CONFIG_NUMA
2632	struct zonelist *zonelist;
2633	struct zoneref *z;
2634	struct zone *zone;
2635	enum zone_type highest_zoneidx = gfp_zone(flags: pc->flags);
2636	struct slab *slab;
2637	unsigned int cpuset_mems_cookie;
2638
2639	/*
2640	* The defrag ratio allows a configuration of the tradeoffs between
2641	* inter node defragmentation and node local allocations. A lower
2642	* defrag_ratio increases the tendency to do local allocations
2643	* instead of attempting to obtain partial slabs from other nodes.
2644	*
2645	* If the defrag_ratio is set to 0 then kmalloc() always
2646	* returns node local objects. If the ratio is higher then kmalloc()
2647	* may return off node objects because partial slabs are obtained
2648	* from other nodes and filled up.
2649	*
2650	* If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
2651	* (which makes defrag_ratio = 1000) then every (well almost)
2652	* allocation will first attempt to defrag slab caches on other nodes.
2653	* This means scanning over all nodes to look for partial slabs which
2654	* may be expensive if we do it every time we are trying to find a slab
2655	* with available objects.
2656	*/
2657	if (!s->remote_node_defrag_ratio ||
2658	get_cycles() % 1024 > s->remote_node_defrag_ratio)
2659	return NULL;
2660
2661	do {
2662	cpuset_mems_cookie = read_mems_allowed_begin();
2663	zonelist = node_zonelist(nid: mempolicy_slab_node(), flags: pc->flags);
2664	for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
2665	struct kmem_cache_node *n;
2666
2667	n = get_node(s, node: zone_to_nid(zone));
2668
2669	if (n && cpuset_zone_allowed(z: zone, gfp_mask: pc->flags) &&
2670	n->nr_partial > s->min_partial) {
2671	slab = get_partial_node(s, n, pc);
2672	if (slab) {
2673	/*
2674	* Don't check read_mems_allowed_retry()
2675	* here - if mems_allowed was updated in
2676	* parallel, that was a harmless race
2677	* between allocation and the cpuset
2678	* update
2679	*/
2680	return slab;
2681	}
2682	}
2683	}
2684	} while (read_mems_allowed_retry(seq: cpuset_mems_cookie));
2685	#endif /* CONFIG_NUMA */
2686	return NULL;
2687	}
2688
2689	/*
2690	* Get a partial slab, lock it and return it.
2691	*/
2692	static struct slab *get_partial(struct kmem_cache *s, int node,
2693	struct partial_context *pc)
2694	{
2695	struct slab *slab;
2696	int searchnode = node;
2697
2698	if (node == NUMA_NO_NODE)
2699	searchnode = numa_mem_id();
2700
2701	slab = get_partial_node(s, n: get_node(s, node: searchnode), pc);
2702	if (slab || node != NUMA_NO_NODE)
2703	return slab;
2704
2705	return get_any_partial(s, pc);
2706	}
2707
2708	#ifndef CONFIG_SLUB_TINY
2709
2710	#ifdef CONFIG_PREEMPTION
2711	/*
2712	* Calculate the next globally unique transaction for disambiguation
2713	* during cmpxchg. The transactions start with the cpu number and are then
2714	* incremented by CONFIG_NR_CPUS.
2715	*/
2716	#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
2717	#else
2718	/*
2719	* No preemption supported therefore also no need to check for
2720	* different cpus.
2721	*/
2722	#define TID_STEP 1
2723	#endif /* CONFIG_PREEMPTION */
2724
2725	static inline unsigned long next_tid(unsigned long tid)
2726	{
2727	return tid + TID_STEP;
2728	}
2729
2730	#ifdef SLUB_DEBUG_CMPXCHG
2731	static inline unsigned int tid_to_cpu(unsigned long tid)
2732	{
2733	return tid % TID_STEP;
2734	}
2735
2736	static inline unsigned long tid_to_event(unsigned long tid)
2737	{
2738	return tid / TID_STEP;
2739	}
2740	#endif
2741
2742	static inline unsigned int init_tid(int cpu)
2743	{
2744	return cpu;
2745	}
2746
2747	static inline void note_cmpxchg_failure(const char *n,
2748	const struct kmem_cache *s, unsigned long tid)
2749	{
2750	#ifdef SLUB_DEBUG_CMPXCHG
2751	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
2752
2753	pr_info("%s %s: cmpxchg redo ", n, s->name);
2754
2755	#ifdef CONFIG_PREEMPTION
2756	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
2757	pr_warn("due to cpu change %d -> %d\n",
2758	tid_to_cpu(tid), tid_to_cpu(actual_tid));
2759	else
2760	#endif
2761	if (tid_to_event(tid) != tid_to_event(actual_tid))
2762	pr_warn("due to cpu running other code. Event %ld->%ld\n",
2763	tid_to_event(tid), tid_to_event(actual_tid));
2764	else
2765	pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
2766	actual_tid, tid, next_tid(tid));
2767	#endif
2768	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
2769	}
2770
2771	static void init_kmem_cache_cpus(struct kmem_cache *s)
2772	{
2773	int cpu;
2774	struct kmem_cache_cpu *c;
2775
2776	for_each_possible_cpu(cpu) {
2777	c = per_cpu_ptr(s->cpu_slab, cpu);
2778	local_lock_init(&c->lock);
2779	c->tid = init_tid(cpu);
2780	}
2781	}
2782
2783	/*
2784	* Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
2785	* unfreezes the slabs and puts it on the proper list.
2786	* Assumes the slab has been already safely taken away from kmem_cache_cpu
2787	* by the caller.
2788	*/
2789	static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
2790	void *freelist)
2791	{
2792	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
2793	int free_delta = 0;
2794	void *nextfree, *freelist_iter, *freelist_tail;
2795	int tail = DEACTIVATE_TO_HEAD;
2796	unsigned long flags = 0;
2797	struct slab new;
2798	struct slab old;
2799
2800	if (slab->freelist) {
2801	stat(s, DEACTIVATE_REMOTE_FREES);
2802	tail = DEACTIVATE_TO_TAIL;
2803	}
2804
2805	/*
2806	* Stage one: Count the objects on cpu's freelist as free_delta and
2807	* remember the last object in freelist_tail for later splicing.
2808	*/
2809	freelist_tail = NULL;
2810	freelist_iter = freelist;
2811	while (freelist_iter) {
2812	nextfree = get_freepointer(s, freelist_iter);
2813
2814	/*
2815	* If 'nextfree' is invalid, it is possible that the object at
2816	* 'freelist_iter' is already corrupted. So isolate all objects
2817	* starting at 'freelist_iter' by skipping them.
2818	*/
2819	if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
2820	break;
2821
2822	freelist_tail = freelist_iter;
2823	free_delta++;
2824
2825	freelist_iter = nextfree;
2826	}
2827
2828	/*
2829	* Stage two: Unfreeze the slab while splicing the per-cpu
2830	* freelist to the head of slab's freelist.
2831	*/
2832	do {
2833	old.freelist = READ_ONCE(slab->freelist);
2834	old.counters = READ_ONCE(slab->counters);
2835	VM_BUG_ON(!old.frozen);
2836
2837	/* Determine target state of the slab */
2838	new.counters = old.counters;
2839	new.frozen = 0;
2840	if (freelist_tail) {
2841	new.inuse -= free_delta;
2842	set_freepointer(s, freelist_tail, old.freelist);
2843	new.freelist = freelist;
2844	} else {
2845	new.freelist = old.freelist;
2846	}
2847	} while (!slab_update_freelist(s, slab,
2848	old.freelist, old.counters,
2849	new.freelist, new.counters,
2850	"unfreezing slab"));
2851
2852	/*
2853	* Stage three: Manipulate the slab list based on the updated state.
2854	*/
2855	if (!new.inuse && n->nr_partial >= s->min_partial) {
2856	stat(s, DEACTIVATE_EMPTY);
2857	discard_slab(s, slab);
2858	stat(s, FREE_SLAB);
2859	} else if (new.freelist) {
2860	spin_lock_irqsave(&n->list_lock, flags);
2861	add_partial(n, slab, tail);
2862	spin_unlock_irqrestore(&n->list_lock, flags);
2863	stat(s, tail);
2864	} else {
2865	stat(s, DEACTIVATE_FULL);
2866	}
2867	}
2868
2869	#ifdef CONFIG_SLUB_CPU_PARTIAL
2870	static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
2871	{
2872	struct kmem_cache_node *n = NULL, *n2 = NULL;
2873	struct slab *slab, *slab_to_discard = NULL;
2874	unsigned long flags = 0;
2875
2876	while (partial_slab) {
2877	slab = partial_slab;
2878	partial_slab = slab->next;
2879
2880	n2 = get_node(s, slab_nid(slab));
2881	if (n != n2) {
2882	if (n)
2883	spin_unlock_irqrestore(&n->list_lock, flags);
2884
2885	n = n2;
2886	spin_lock_irqsave(&n->list_lock, flags);
2887	}
2888
2889	if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
2890	slab->next = slab_to_discard;
2891	slab_to_discard = slab;
2892	} else {
2893	add_partial(n, slab, DEACTIVATE_TO_TAIL);
2894	stat(s, FREE_ADD_PARTIAL);
2895	}
2896	}
2897
2898	if (n)
2899	spin_unlock_irqrestore(&n->list_lock, flags);
2900
2901	while (slab_to_discard) {
2902	slab = slab_to_discard;
2903	slab_to_discard = slab_to_discard->next;
2904
2905	stat(s, DEACTIVATE_EMPTY);
2906	discard_slab(s, slab);
2907	stat(s, FREE_SLAB);
2908	}
2909	}
2910
2911	/*
2912	* Put all the cpu partial slabs to the node partial list.
2913	*/
2914	static void put_partials(struct kmem_cache *s)
2915	{
2916	struct slab *partial_slab;
2917	unsigned long flags;
2918
2919	local_lock_irqsave(&s->cpu_slab->lock, flags);
2920	partial_slab = this_cpu_read(s->cpu_slab->partial);
2921	this_cpu_write(s->cpu_slab->partial, NULL);
2922	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2923
2924	if (partial_slab)
2925	__put_partials(s, partial_slab);
2926	}
2927
2928	static void put_partials_cpu(struct kmem_cache *s,
2929	struct kmem_cache_cpu *c)
2930	{
2931	struct slab *partial_slab;
2932
2933	partial_slab = slub_percpu_partial(c);
2934	c->partial = NULL;
2935
2936	if (partial_slab)
2937	__put_partials(s, partial_slab);
2938	}
2939
2940	/*
2941	* Put a slab into a partial slab slot if available.
2942	*
2943	* If we did not find a slot then simply move all the partials to the
2944	* per node partial list.
2945	*/
2946	static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
2947	{
2948	struct slab *oldslab;
2949	struct slab *slab_to_put = NULL;
2950	unsigned long flags;
2951	int slabs = 0;
2952
2953	local_lock_irqsave(&s->cpu_slab->lock, flags);
2954
2955	oldslab = this_cpu_read(s->cpu_slab->partial);
2956
2957	if (oldslab) {
2958	if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
2959	/*
2960	* Partial array is full. Move the existing set to the
2961	* per node partial list. Postpone the actual unfreezing
2962	* outside of the critical section.
2963	*/
2964	slab_to_put = oldslab;
2965	oldslab = NULL;
2966	} else {
2967	slabs = oldslab->slabs;
2968	}
2969	}
2970
2971	slabs++;
2972
2973	slab->slabs = slabs;
2974	slab->next = oldslab;
2975
2976	this_cpu_write(s->cpu_slab->partial, slab);
2977
2978	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2979
2980	if (slab_to_put) {
2981	__put_partials(s, slab_to_put);
2982	stat(s, CPU_PARTIAL_DRAIN);
2983	}
2984	}
2985
2986	#else /* CONFIG_SLUB_CPU_PARTIAL */
2987
2988	static inline void put_partials(struct kmem_cache *s) { }
2989	static inline void put_partials_cpu(struct kmem_cache *s,
2990	struct kmem_cache_cpu *c) { }
2991
2992	#endif /* CONFIG_SLUB_CPU_PARTIAL */
2993
2994	static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
2995	{
2996	unsigned long flags;
2997	struct slab *slab;
2998	void *freelist;
2999
3000	local_lock_irqsave(&s->cpu_slab->lock, flags);
3001
3002	slab = c->slab;
3003	freelist = c->freelist;
3004
3005	c->slab = NULL;
3006	c->freelist = NULL;
3007	c->tid = next_tid(c->tid);
3008
3009	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3010
3011	if (slab) {
3012	deactivate_slab(s, slab, freelist);
3013	stat(s, CPUSLAB_FLUSH);
3014	}
3015	}
3016
3017	static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
3018	{
3019	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
3020	void *freelist = c->freelist;
3021	struct slab *slab = c->slab;
3022
3023	c->slab = NULL;
3024	c->freelist = NULL;
3025	c->tid = next_tid(c->tid);
3026
3027	if (slab) {
3028	deactivate_slab(s, slab, freelist);
3029	stat(s, CPUSLAB_FLUSH);
3030	}
3031
3032	put_partials_cpu(s, c);
3033	}
3034
3035	struct slub_flush_work {
3036	struct work_struct work;
3037	struct kmem_cache *s;
3038	bool skip;
3039	};
3040
3041	/*
3042	* Flush cpu slab.
3043	*
3044	* Called from CPU work handler with migration disabled.
3045	*/
3046	static void flush_cpu_slab(struct work_struct *w)
3047	{
3048	struct kmem_cache *s;
3049	struct kmem_cache_cpu *c;
3050	struct slub_flush_work *sfw;
3051
3052	sfw = container_of(w, struct slub_flush_work, work);
3053
3054	s = sfw->s;
3055	c = this_cpu_ptr(s->cpu_slab);
3056
3057	if (c->slab)
3058	flush_slab(s, c);
3059
3060	put_partials(s);
3061	}
3062
3063	static bool has_cpu_slab(int cpu, struct kmem_cache *s)
3064	{
3065	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
3066
3067	return c->slab || slub_percpu_partial(c);
3068	}
3069
3070	static DEFINE_MUTEX(flush_lock);
3071	static DEFINE_PER_CPU(struct slub_flush_work, slub_flush);
3072
3073	static void flush_all_cpus_locked(struct kmem_cache *s)
3074	{
3075	struct slub_flush_work *sfw;
3076	unsigned int cpu;
3077
3078	lockdep_assert_cpus_held();
3079	mutex_lock(&flush_lock);
3080
3081	for_each_online_cpu(cpu) {
3082	sfw = &per_cpu(slub_flush, cpu);
3083	if (!has_cpu_slab(cpu, s)) {
3084	sfw->skip = true;
3085	continue;
3086	}
3087	INIT_WORK(&sfw->work, flush_cpu_slab);
3088	sfw->skip = false;
3089	sfw->s = s;
3090	queue_work_on(cpu, flushwq, &sfw->work);
3091	}
3092
3093	for_each_online_cpu(cpu) {
3094	sfw = &per_cpu(slub_flush, cpu);
3095	if (sfw->skip)
3096	continue;
3097	flush_work(&sfw->work);
3098	}
3099
3100	mutex_unlock(&flush_lock);
3101	}
3102
3103	static void flush_all(struct kmem_cache *s)
3104	{
3105	cpus_read_lock();
3106	flush_all_cpus_locked(s);
3107	cpus_read_unlock();
3108	}
3109
3110	/*
3111	* Use the cpu notifier to insure that the cpu slabs are flushed when
3112	* necessary.
3113	*/
3114	static int slub_cpu_dead(unsigned int cpu)
3115	{
3116	struct kmem_cache *s;
3117
3118	mutex_lock(&slab_mutex);
3119	list_for_each_entry(s, &slab_caches, list)
3120	__flush_cpu_slab(s, cpu);
3121	mutex_unlock(&slab_mutex);
3122	return 0;
3123	}
3124
3125	#else /* CONFIG_SLUB_TINY */
3126	static inline void flush_all_cpus_locked(struct kmem_cache *s) { }
3127	static inline void flush_all(struct kmem_cache *s) { }
3128	static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { }
3129	static inline int slub_cpu_dead(unsigned int cpu) { return 0; }
3130	#endif /* CONFIG_SLUB_TINY */
3131
3132	/*
3133	* Check if the objects in a per cpu structure fit numa
3134	* locality expectations.
3135	*/
3136	static inline int node_match(struct slab *slab, int node)
3137	{
3138	#ifdef CONFIG_NUMA
3139	if (node != NUMA_NO_NODE && slab_nid(slab) != node)
3140	return 0;
3141	#endif
3142	return 1;
3143	}
3144
3145	#ifdef CONFIG_SLUB_DEBUG
3146	static int count_free(struct slab *slab)
3147	{
3148	return slab->objects - slab->inuse;
3149	}
3150
3151	static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
3152	{
3153	return atomic_long_read(&n->total_objects);
3154	}
3155
3156	/* Supports checking bulk free of a constructed freelist */
3157	static inline bool free_debug_processing(struct kmem_cache *s,
3158	struct slab *slab, void *head, void *tail, int *bulk_cnt,
3159	unsigned long addr, depot_stack_handle_t handle)
3160	{
3161	bool checks_ok = false;
3162	void *object = head;
3163	int cnt = 0;
3164
3165	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
3166	if (!check_slab(s, slab))
3167	goto out;
3168	}
3169
3170	if (slab->inuse < *bulk_cnt) {
3171	slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",
3172	slab->inuse, *bulk_cnt);
3173	goto out;
3174	}
3175
3176	next_object:
3177
3178	if (++cnt > *bulk_cnt)
3179	goto out_cnt;
3180
3181	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
3182	if (!free_consistency_checks(s, slab, object, addr))
3183	goto out;
3184	}
3185
3186	if (s->flags & SLAB_STORE_USER)
3187	set_track_update(s, object, TRACK_FREE, addr, handle);
3188	trace(s, slab, object, 0);
3189	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
3190	init_object(s, object, SLUB_RED_INACTIVE);
3191
3192	/* Reached end of constructed freelist yet? */
3193	if (object != tail) {
3194	object = get_freepointer(s, object);
3195	goto next_object;
3196	}
3197	checks_ok = true;
3198
3199	out_cnt:
3200	if (cnt != *bulk_cnt) {
3201	slab_err(s, slab, "Bulk free expected %d objects but found %d\n",
3202	*bulk_cnt, cnt);
3203	*bulk_cnt = cnt;
3204	}
3205
3206	out:
3207
3208	if (!checks_ok)
3209	slab_fix(s, "Object at 0x%p not freed", object);
3210
3211	return checks_ok;
3212	}
3213	#endif /* CONFIG_SLUB_DEBUG */
3214
3215	#if defined(CONFIG_SLUB_DEBUG) || defined(SLAB_SUPPORTS_SYSFS)
3216	static unsigned long count_partial(struct kmem_cache_node *n,
3217	int (*get_count)(struct slab *))
3218	{
3219	unsigned long flags;
3220	unsigned long x = 0;
3221	struct slab *slab;
3222
3223	spin_lock_irqsave(&n->list_lock, flags);
3224	list_for_each_entry(slab, &n->partial, slab_list)
3225	x += get_count(slab);
3226	spin_unlock_irqrestore(&n->list_lock, flags);
3227	return x;
3228	}
3229	#endif /* CONFIG_SLUB_DEBUG || SLAB_SUPPORTS_SYSFS */
3230
3231	#ifdef CONFIG_SLUB_DEBUG
3232	static noinline void
3233	slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
3234	{
3235	static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
3236	DEFAULT_RATELIMIT_BURST);
3237	int node;
3238	struct kmem_cache_node *n;
3239
3240	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
3241	return;
3242
3243	pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
3244	nid, gfpflags, &gfpflags);
3245	pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
3246	s->name, s->object_size, s->size, oo_order(s->oo),
3247	oo_order(s->min));
3248
3249	if (oo_order(s->min) > get_order(s->object_size))
3250	pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n",
3251	s->name);
3252
3253	for_each_kmem_cache_node(s, node, n) {
3254	unsigned long nr_slabs;
3255	unsigned long nr_objs;
3256	unsigned long nr_free;
3257
3258	nr_free = count_partial(n, count_free);
3259	nr_slabs = node_nr_slabs(n);
3260	nr_objs = node_nr_objs(n);
3261
3262	pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
3263	node, nr_slabs, nr_objs, nr_free);
3264	}
3265	}
3266	#else /* CONFIG_SLUB_DEBUG */
3267	static inline void
3268	slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { }
3269	#endif
3270
3271	static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
3272	{
3273	if (unlikely(slab_test_pfmemalloc(slab)))
3274	return gfp_pfmemalloc_allowed(gfp_mask: gfpflags);
3275
3276	return true;
3277	}
3278
3279	#ifndef CONFIG_SLUB_TINY
3280	static inline bool
3281	__update_cpu_freelist_fast(struct kmem_cache *s,
3282	void *freelist_old, void *freelist_new,
3283	unsigned long tid)
3284	{
3285	freelist_aba_t old = { .freelist = freelist_old, .counter = tid };
3286	freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };
3287
3288	return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
3289	&old.full, new.full);
3290	}
3291
3292	/*
3293	* Check the slab->freelist and either transfer the freelist to the
3294	* per cpu freelist or deactivate the slab.
3295	*
3296	* The slab is still frozen if the return value is not NULL.
3297	*
3298	* If this function returns NULL then the slab has been unfrozen.
3299	*/
3300	static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
3301	{
3302	struct slab new;
3303	unsigned long counters;
3304	void *freelist;
3305
3306	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
3307
3308	do {
3309	freelist = slab->freelist;
3310	counters = slab->counters;
3311
3312	new.counters = counters;
3313
3314	new.inuse = slab->objects;
3315	new.frozen = freelist != NULL;
3316
3317	} while (!__slab_update_freelist(s, slab,
3318	freelist, counters,
3319	NULL, new.counters,
3320	"get_freelist"));
3321
3322	return freelist;
3323	}
3324
3325	/*
3326	* Freeze the partial slab and return the pointer to the freelist.
3327	*/
3328	static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab)
3329	{
3330	struct slab new;
3331	unsigned long counters;
3332	void *freelist;
3333
3334	do {
3335	freelist = slab->freelist;
3336	counters = slab->counters;
3337
3338	new.counters = counters;
3339	VM_BUG_ON(new.frozen);
3340
3341	new.inuse = slab->objects;
3342	new.frozen = 1;
3343
3344	} while (!slab_update_freelist(s, slab,
3345	freelist, counters,
3346	NULL, new.counters,
3347	"freeze_slab"));
3348
3349	return freelist;
3350	}
3351
3352	/*
3353	* Slow path. The lockless freelist is empty or we need to perform
3354	* debugging duties.
3355	*
3356	* Processing is still very fast if new objects have been freed to the
3357	* regular freelist. In that case we simply take over the regular freelist
3358	* as the lockless freelist and zap the regular freelist.
3359	*
3360	* If that is not working then we fall back to the partial lists. We take the
3361	* first element of the freelist as the object to allocate now and move the
3362	* rest of the freelist to the lockless freelist.
3363	*
3364	* And if we were unable to get a new slab from the partial slab lists then
3365	* we need to allocate a new slab. This is the slowest path since it involves
3366	* a call to the page allocator and the setup of a new slab.
3367	*
3368	* Version of __slab_alloc to use when we know that preemption is
3369	* already disabled (which is the case for bulk allocation).
3370	*/
3371	static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3372	unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size)
3373	{
3374	void *freelist;
3375	struct slab *slab;
3376	unsigned long flags;
3377	struct partial_context pc;
3378
3379	stat(s, ALLOC_SLOWPATH);
3380
3381	reread_slab:
3382
3383	slab = READ_ONCE(c->slab);
3384	if (!slab) {
3385	/*
3386	* if the node is not online or has no normal memory, just
3387	* ignore the node constraint
3388	*/
3389	if (unlikely(node != NUMA_NO_NODE &&
3390	!node_isset(node, slab_nodes)))
3391	node = NUMA_NO_NODE;
3392	goto new_slab;
3393	}
3394
3395	if (unlikely(!node_match(slab, node))) {
3396	/*
3397	* same as above but node_match() being false already
3398	* implies node != NUMA_NO_NODE
3399	*/
3400	if (!node_isset(node, slab_nodes)) {
3401	node = NUMA_NO_NODE;
3402	} else {
3403	stat(s, ALLOC_NODE_MISMATCH);
3404	goto deactivate_slab;
3405	}
3406	}
3407
3408	/*
3409	* By rights, we should be searching for a slab page that was
3410	* PFMEMALLOC but right now, we are losing the pfmemalloc
3411	* information when the page leaves the per-cpu allocator
3412	*/
3413	if (unlikely(!pfmemalloc_match(slab, gfpflags)))
3414	goto deactivate_slab;
3415
3416	/* must check again c->slab in case we got preempted and it changed */
3417	local_lock_irqsave(&s->cpu_slab->lock, flags);
3418	if (unlikely(slab != c->slab)) {
3419	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3420	goto reread_slab;
3421	}
3422	freelist = c->freelist;
3423	if (freelist)
3424	goto load_freelist;
3425
3426	freelist = get_freelist(s, slab);
3427
3428	if (!freelist) {
3429	c->slab = NULL;
3430	c->tid = next_tid(c->tid);
3431	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3432	stat(s, DEACTIVATE_BYPASS);
3433	goto new_slab;
3434	}
3435
3436	stat(s, ALLOC_REFILL);
3437
3438	load_freelist:
3439
3440	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
3441
3442	/*
3443	* freelist is pointing to the list of objects to be used.
3444	* slab is pointing to the slab from which the objects are obtained.
3445	* That slab must be frozen for per cpu allocations to work.
3446	*/
3447	VM_BUG_ON(!c->slab->frozen);
3448	c->freelist = get_freepointer(s, freelist);
3449	c->tid = next_tid(c->tid);
3450	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3451	return freelist;
3452
3453	deactivate_slab:
3454
3455	local_lock_irqsave(&s->cpu_slab->lock, flags);
3456	if (slab != c->slab) {
3457	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3458	goto reread_slab;
3459	}
3460	freelist = c->freelist;
3461	c->slab = NULL;
3462	c->freelist = NULL;
3463	c->tid = next_tid(c->tid);
3464	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3465	deactivate_slab(s, slab, freelist);
3466
3467	new_slab:
3468
3469	#ifdef CONFIG_SLUB_CPU_PARTIAL
3470	while (slub_percpu_partial(c)) {
3471	local_lock_irqsave(&s->cpu_slab->lock, flags);
3472	if (unlikely(c->slab)) {
3473	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3474	goto reread_slab;
3475	}
3476	if (unlikely(!slub_percpu_partial(c))) {
3477	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3478	/* we were preempted and partial list got empty */
3479	goto new_objects;
3480	}
3481
3482	slab = slub_percpu_partial(c);
3483	slub_set_percpu_partial(c, slab);
3484
3485	if (likely(node_match(slab, node) &&
3486	pfmemalloc_match(slab, gfpflags))) {
3487	c->slab = slab;
3488	freelist = get_freelist(s, slab);
3489	VM_BUG_ON(!freelist);
3490	stat(s, CPU_PARTIAL_ALLOC);
3491	goto load_freelist;
3492	}
3493
3494	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3495
3496	slab->next = NULL;
3497	__put_partials(s, slab);
3498	}
3499	#endif
3500
3501	new_objects:
3502
3503	pc.flags = gfpflags;
3504	pc.orig_size = orig_size;
3505	slab = get_partial(s, node, &pc);
3506	if (slab) {
3507	if (kmem_cache_debug(s)) {
3508	freelist = pc.object;
3509	/*
3510	* For debug caches here we had to go through
3511	* alloc_single_from_partial() so just store the
3512	* tracking info and return the object.
3513	*/
3514	if (s->flags & SLAB_STORE_USER)
3515	set_track(s, freelist, TRACK_ALLOC, addr);
3516
3517	return freelist;
3518	}
3519
3520	freelist = freeze_slab(s, slab);
3521	goto retry_load_slab;
3522	}
3523
3524	slub_put_cpu_ptr(s->cpu_slab);
3525	slab = new_slab(s, gfpflags, node);
3526	c = slub_get_cpu_ptr(s->cpu_slab);
3527
3528	if (unlikely(!slab)) {
3529	slab_out_of_memory(s, gfpflags, node);
3530	return NULL;
3531	}
3532
3533	stat(s, ALLOC_SLAB);
3534
3535	if (kmem_cache_debug(s)) {
3536	freelist = alloc_single_from_new_slab(s, slab, orig_size);
3537
3538	if (unlikely(!freelist))
3539	goto new_objects;
3540
3541	if (s->flags & SLAB_STORE_USER)
3542	set_track(s, freelist, TRACK_ALLOC, addr);
3543
3544	return freelist;
3545	}
3546
3547	/*
3548	* No other reference to the slab yet so we can
3549	* muck around with it freely without cmpxchg
3550	*/
3551	freelist = slab->freelist;
3552	slab->freelist = NULL;
3553	slab->inuse = slab->objects;
3554	slab->frozen = 1;
3555
3556	inc_slabs_node(s, slab_nid(slab), slab->objects);
3557
3558	if (unlikely(!pfmemalloc_match(slab, gfpflags))) {
3559	/*
3560	* For !pfmemalloc_match() case we don't load freelist so that
3561	* we don't make further mismatched allocations easier.
3562	*/
3563	deactivate_slab(s, slab, get_freepointer(s, freelist));
3564	return freelist;
3565	}
3566
3567	retry_load_slab:
3568
3569	local_lock_irqsave(&s->cpu_slab->lock, flags);
3570	if (unlikely(c->slab)) {
3571	void *flush_freelist = c->freelist;
3572	struct slab *flush_slab = c->slab;
3573
3574	c->slab = NULL;
3575	c->freelist = NULL;
3576	c->tid = next_tid(c->tid);
3577
3578	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3579
3580	deactivate_slab(s, flush_slab, flush_freelist);
3581
3582	stat(s, CPUSLAB_FLUSH);
3583
3584	goto retry_load_slab;
3585	}
3586	c->slab = slab;
3587
3588	goto load_freelist;
3589	}
3590
3591	/*
3592	* A wrapper for ___slab_alloc() for contexts where preemption is not yet
3593	* disabled. Compensates for possible cpu changes by refetching the per cpu area
3594	* pointer.
3595	*/
3596	static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3597	unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size)
3598	{
3599	void *p;
3600
3601	#ifdef CONFIG_PREEMPT_COUNT
3602	/*
3603	* We may have been preempted and rescheduled on a different
3604	* cpu before disabling preemption. Need to reload cpu area
3605	* pointer.
3606	*/
3607	c = slub_get_cpu_ptr(s->cpu_slab);
3608	#endif
3609
3610	p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size);
3611	#ifdef CONFIG_PREEMPT_COUNT
3612	slub_put_cpu_ptr(s->cpu_slab);
3613	#endif
3614	return p;
3615	}
3616
3617	static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
3618	gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3619	{
3620	struct kmem_cache_cpu *c;
3621	struct slab *slab;
3622	unsigned long tid;
3623	void *object;
3624
3625	redo:
3626	/*
3627	* Must read kmem_cache cpu data via this cpu ptr. Preemption is
3628	* enabled. We may switch back and forth between cpus while
3629	* reading from one cpu area. That does not matter as long
3630	* as we end up on the original cpu again when doing the cmpxchg.
3631	*
3632	* We must guarantee that tid and kmem_cache_cpu are retrieved on the
3633	* same cpu. We read first the kmem_cache_cpu pointer and use it to read
3634	* the tid. If we are preempted and switched to another cpu between the
3635	* two reads, it's OK as the two are still associated with the same cpu
3636	* and cmpxchg later will validate the cpu.
3637	*/
3638	c = raw_cpu_ptr(s->cpu_slab);
3639	tid = READ_ONCE(c->tid);
3640
3641	/*
3642	* Irqless object alloc/free algorithm used here depends on sequence
3643	* of fetching cpu_slab's data. tid should be fetched before anything
3644	* on c to guarantee that object and slab associated with previous tid
3645	* won't be used with current tid. If we fetch tid first, object and
3646	* slab could be one associated with next tid and our alloc/free
3647	* request will be failed. In this case, we will retry. So, no problem.
3648	*/
3649	barrier();
3650
3651	/*
3652	* The transaction ids are globally unique per cpu and per operation on
3653	* a per cpu queue. Thus they can be guarantee that the cmpxchg_double
3654	* occurs on the right processor and that there was no operation on the
3655	* linked list in between.
3656	*/
3657
3658	object = c->freelist;
3659	slab = c->slab;
3660
3661	if (!USE_LOCKLESS_FAST_PATH() ||
3662	unlikely(!object || !slab || !node_match(slab, node))) {
3663	object = __slab_alloc(s, gfpflags, node, addr, c, orig_size);
3664	} else {
3665	void *next_object = get_freepointer_safe(s, object);
3666
3667	/*
3668	* The cmpxchg will only match if there was no additional
3669	* operation and if we are on the right processor.
3670	*
3671	* The cmpxchg does the following atomically (without lock
3672	* semantics!)
3673	* 1. Relocate first pointer to the current per cpu area.
3674	* 2. Verify that tid and freelist have not been changed
3675	* 3. If they were not changed replace tid and freelist
3676	*
3677	* Since this is without lock semantics the protection is only
3678	* against code executing on this cpu *not* from access by
3679	* other cpus.
3680	*/
3681	if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) {
3682	note_cmpxchg_failure("slab_alloc", s, tid);
3683	goto redo;
3684	}
3685	prefetch_freepointer(s, next_object);
3686	stat(s, ALLOC_FASTPATH);
3687	}
3688
3689	return object;
3690	}
3691	#else /* CONFIG_SLUB_TINY */
3692	static void *__slab_alloc_node(struct kmem_cache *s,
3693	gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3694	{
3695	struct partial_context pc;
3696	struct slab *slab;
3697	void *object;
3698
3699	pc.flags = gfpflags;
3700	pc.orig_size = orig_size;
3701	slab = get_partial(s, node, pc: &pc);
3702
3703	if (slab)
3704	return pc.object;
3705
3706	slab = new_slab(s, flags: gfpflags, node);
3707	if (unlikely(!slab)) {
3708	slab_out_of_memory(s, gfpflags, nid: node);
3709	return NULL;
3710	}
3711
3712	object = alloc_single_from_new_slab(s, slab, orig_size);
3713
3714	return object;
3715	}
3716	#endif /* CONFIG_SLUB_TINY */
3717
3718	/*
3719	* If the object has been wiped upon free, make sure it's fully initialized by
3720	* zeroing out freelist pointer.
3721	*/
3722	static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
3723	void *obj)
3724	{
3725	if (unlikely(slab_want_init_on_free(s)) && obj)
3726	memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
3727	0, sizeof(void *));
3728	}
3729
3730	noinline int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
3731	{
3732	if (__should_failslab(s, gfpflags))
3733	return -ENOMEM;
3734	return 0;
3735	}
3736	ALLOW_ERROR_INJECTION(should_failslab, ERRNO);
3737
3738	static __fastpath_inline
3739	struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
3740	struct list_lru *lru,
3741	struct obj_cgroup **objcgp,
3742	size_t size, gfp_t flags)
3743	{
3744	flags &= gfp_allowed_mask;
3745
3746	might_alloc(gfp_mask: flags);
3747
3748	if (unlikely(should_failslab(s, flags)))
3749	return NULL;
3750
3751	if (unlikely(!memcg_slab_pre_alloc_hook(s, lru, objcgp, size, flags)))
3752	return NULL;
3753
3754	return s;
3755	}
3756
3757	static __fastpath_inline
3758	void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg,
3759	gfp_t flags, size_t size, void **p, bool init,
3760	unsigned int orig_size)
3761	{
3762	unsigned int zero_size = s->object_size;
3763	bool kasan_init = init;
3764	size_t i;
3765	gfp_t init_flags = flags & gfp_allowed_mask;
3766
3767	/*
3768	* For kmalloc object, the allocated memory size(object_size) is likely
3769	* larger than the requested size(orig_size). If redzone check is
3770	* enabled for the extra space, don't zero it, as it will be redzoned
3771	* soon. The redzone operation for this extra space could be seen as a
3772	* replacement of current poisoning under certain debug option, and
3773	* won't break other sanity checks.
3774	*/
3775	if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) &&
3776	(s->flags & SLAB_KMALLOC))
3777	zero_size = orig_size;
3778
3779	/*
3780	* When slab_debug is enabled, avoid memory initialization integrated
3781	* into KASAN and instead zero out the memory via the memset below with
3782	* the proper size. Otherwise, KASAN might overwrite SLUB redzones and
3783	* cause false-positive reports. This does not lead to a performance
3784	* penalty on production builds, as slab_debug is not intended to be
3785	* enabled there.
3786	*/
3787	if (__slub_debug_enabled())
3788	kasan_init = false;
3789
3790	/*
3791	* As memory initialization might be integrated into KASAN,
3792	* kasan_slab_alloc and initialization memset must be
3793	* kept together to avoid discrepancies in behavior.
3794	*
3795	* As p[i] might get tagged, memset and kmemleak hook come after KASAN.
3796	*/
3797	for (i = 0; i < size; i++) {
3798	p[i] = kasan_slab_alloc(s, object: p[i], flags: init_flags, init: kasan_init);
3799	if (p[i] && init && (!kasan_init ||
3800	!kasan_has_integrated_init()))
3801	memset(p[i], 0, zero_size);
3802	kmemleak_alloc_recursive(ptr: p[i], size: s->object_size, min_count: 1,
3803	flags: s->flags, gfp: init_flags);
3804	kmsan_slab_alloc(s, object: p[i], flags: init_flags);
3805	}
3806
3807	memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
3808	}
3809
3810	/*
3811	* Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
3812	* have the fastpath folded into their functions. So no function call
3813	* overhead for requests that can be satisfied on the fastpath.
3814	*
3815	* The fastpath works by first checking if the lockless freelist can be used.
3816	* If not then __slab_alloc is called for slow processing.
3817	*
3818	* Otherwise we can simply pick the next object from the lockless free list.
3819	*/
3820	static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
3821	gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3822	{
3823	void *object;
3824	struct obj_cgroup *objcg = NULL;
3825	bool init = false;
3826
3827	s = slab_pre_alloc_hook(s, lru, objcgp: &objcg, size: 1, flags: gfpflags);
3828	if (unlikely(!s))
3829	return NULL;
3830
3831	object = kfence_alloc(s, size: orig_size, flags: gfpflags);
3832	if (unlikely(object))
3833	goto out;
3834
3835	object = __slab_alloc_node(s, gfpflags, node, addr, orig_size);
3836
3837	maybe_wipe_obj_freeptr(s, obj: object);
3838	init = slab_want_init_on_alloc(flags: gfpflags, c: s);
3839
3840	out:
3841	/*
3842	* When init equals 'true', like for kzalloc() family, only
3843	* @orig_size bytes might be zeroed instead of s->object_size
3844	*/
3845	slab_post_alloc_hook(s, objcg, flags: gfpflags, size: 1, p: &object, init, orig_size);
3846
3847	return object;
3848	}
3849
3850	void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
3851	{
3852	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_,
3853	orig_size: s->object_size);
3854
3855	trace_kmem_cache_alloc(_RET_IP_, ptr: ret, s, gfp_flags: gfpflags, NUMA_NO_NODE);
3856
3857	return ret;
3858	}
3859	EXPORT_SYMBOL(kmem_cache_alloc);
3860
3861	void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
3862	gfp_t gfpflags)
3863	{
3864	void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_,
3865	orig_size: s->object_size);
3866
3867	trace_kmem_cache_alloc(_RET_IP_, ptr: ret, s, gfp_flags: gfpflags, NUMA_NO_NODE);
3868
3869	return ret;
3870	}
3871	EXPORT_SYMBOL(kmem_cache_alloc_lru);
3872
3873	/**
3874	* kmem_cache_alloc_node - Allocate an object on the specified node
3875	* @s: The cache to allocate from.
3876	* @gfpflags: See kmalloc().
3877	* @node: node number of the target node.
3878	*
3879	* Identical to kmem_cache_alloc but it will allocate memory on the given
3880	* node, which can improve the performance for cpu bound structures.
3881	*
3882	* Fallback to other node is possible if __GFP_THISNODE is not set.
3883	*
3884	* Return: pointer to the new object or %NULL in case of error
3885	*/
3886	void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
3887	{
3888	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, orig_size: s->object_size);
3889
3890	trace_kmem_cache_alloc(_RET_IP_, ptr: ret, s, gfp_flags: gfpflags, node);
3891
3892	return ret;
3893	}
3894	EXPORT_SYMBOL(kmem_cache_alloc_node);
3895
3896	/*
3897	* To avoid unnecessary overhead, we pass through large allocation requests
3898	* directly to the page allocator. We use __GFP_COMP, because we will need to
3899	* know the allocation order to free the pages properly in kfree.
3900	*/
3901	static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
3902	{
3903	struct folio *folio;
3904	void *ptr = NULL;
3905	unsigned int order = get_order(size);
3906
3907	if (unlikely(flags & GFP_SLAB_BUG_MASK))
3908	flags = kmalloc_fix_flags(flags);
3909
3910	flags |= __GFP_COMP;
3911	folio = (struct folio *)alloc_pages_node(nid: node, gfp_mask: flags, order);
3912	if (folio) {
3913	ptr = folio_address(folio);
3914	lruvec_stat_mod_folio(folio, idx: NR_SLAB_UNRECLAIMABLE_B,
3915	PAGE_SIZE << order);
3916	}
3917
3918	ptr = kasan_kmalloc_large(ptr, size, flags);
3919	/* As ptr might get tagged, call kmemleak hook after KASAN. */
3920	kmemleak_alloc(ptr, size, min_count: 1, gfp: flags);
3921	kmsan_kmalloc_large(ptr, size, flags);
3922
3923	return ptr;
3924	}
3925
3926	void *kmalloc_large(size_t size, gfp_t flags)
3927	{
3928	void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);
3929
3930	trace_kmalloc(_RET_IP_, ptr: ret, bytes_req: size, PAGE_SIZE << get_order(size),
3931	gfp_flags: flags, NUMA_NO_NODE);
3932	return ret;
3933	}
3934	EXPORT_SYMBOL(kmalloc_large);
3935
3936	void *kmalloc_large_node(size_t size, gfp_t flags, int node)
3937	{
3938	void *ret = __kmalloc_large_node(size, flags, node);
3939
3940	trace_kmalloc(_RET_IP_, ptr: ret, bytes_req: size, PAGE_SIZE << get_order(size),
3941	gfp_flags: flags, node);
3942	return ret;
3943	}
3944	EXPORT_SYMBOL(kmalloc_large_node);
3945
3946	static __always_inline
3947	void *__do_kmalloc_node(size_t size, gfp_t flags, int node,
3948	unsigned long caller)
3949	{
3950	struct kmem_cache *s;
3951	void *ret;
3952
3953	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
3954	ret = __kmalloc_large_node(size, flags, node);
3955	trace_kmalloc(call_site: caller, ptr: ret, bytes_req: size,
3956	PAGE_SIZE << get_order(size), gfp_flags: flags, node);
3957	return ret;
3958	}
3959
3960	if (unlikely(!size))
3961	return ZERO_SIZE_PTR;
3962
3963	s = kmalloc_slab(size, flags, caller);
3964
3965	ret = slab_alloc_node(s, NULL, gfpflags: flags, node, addr: caller, orig_size: size);
3966	ret = kasan_kmalloc(s, object: ret, size, flags);
3967	trace_kmalloc(call_site: caller, ptr: ret, bytes_req: size, bytes_alloc: s->size, gfp_flags: flags, node);
3968	return ret;
3969	}
3970
3971	void *__kmalloc_node(size_t size, gfp_t flags, int node)
3972	{
3973	return __do_kmalloc_node(size, flags, node, _RET_IP_);
3974	}
3975	EXPORT_SYMBOL(__kmalloc_node);
3976
3977	void *__kmalloc(size_t size, gfp_t flags)
3978	{
3979	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
3980	}
3981	EXPORT_SYMBOL(__kmalloc);
3982
3983	void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3984	int node, unsigned long caller)
3985	{
3986	return __do_kmalloc_node(size, flags, node, caller);
3987	}
3988	EXPORT_SYMBOL(__kmalloc_node_track_caller);
3989
3990	void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
3991	{
3992	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,
3993	_RET_IP_, orig_size: size);
3994
3995	trace_kmalloc(_RET_IP_, ptr: ret, bytes_req: size, bytes_alloc: s->size, gfp_flags: gfpflags, NUMA_NO_NODE);
3996
3997	ret = kasan_kmalloc(s, object: ret, size, flags: gfpflags);
3998	return ret;
3999	}
4000	EXPORT_SYMBOL(kmalloc_trace);
4001
4002	void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
4003	int node, size_t size)
4004	{
4005	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, orig_size: size);
4006
4007	trace_kmalloc(_RET_IP_, ptr: ret, bytes_req: size, bytes_alloc: s->size, gfp_flags: gfpflags, node);
4008
4009	ret = kasan_kmalloc(s, object: ret, size, flags: gfpflags);
4010	return ret;
4011	}
4012	EXPORT_SYMBOL(kmalloc_node_trace);
4013
4014	static noinline void free_to_partial_list(
4015	struct kmem_cache *s, struct slab *slab,
4016	void *head, void *tail, int bulk_cnt,
4017	unsigned long addr)
4018	{
4019	struct kmem_cache_node *n = get_node(s, node: slab_nid(slab));
4020	struct slab *slab_free = NULL;
4021	int cnt = bulk_cnt;
4022	unsigned long flags;
4023	depot_stack_handle_t handle = 0;
4024
4025	if (s->flags & SLAB_STORE_USER)
4026	handle = set_track_prepare();
4027
4028	spin_lock_irqsave(&n->list_lock, flags);
4029
4030	if (free_debug_processing(s, slab, head, tail, bulk_cnt: &cnt, addr, handle)) {
4031	void *prior = slab->freelist;
4032
4033	/* Perform the actual freeing while we still hold the locks */
4034	slab->inuse -= cnt;
4035	set_freepointer(s, object: tail, fp: prior);
4036	slab->freelist = head;
4037
4038	/*
4039	* If the slab is empty, and node's partial list is full,
4040	* it should be discarded anyway no matter it's on full or
4041	* partial list.
4042	*/
4043	if (slab->inuse == 0 && n->nr_partial >= s->min_partial)
4044	slab_free = slab;
4045
4046	if (!prior) {
4047	/* was on full list */
4048	remove_full(s, n, slab);
4049	if (!slab_free) {
4050	add_partial(n, slab, tail: DEACTIVATE_TO_TAIL);
4051	stat(s, si: FREE_ADD_PARTIAL);
4052	}
4053	} else if (slab_free) {
4054	remove_partial(n, slab);
4055	stat(s, si: FREE_REMOVE_PARTIAL);
4056	}
4057	}
4058
4059	if (slab_free) {
4060	/*
4061	* Update the counters while still holding n->list_lock to
4062	* prevent spurious validation warnings
4063	*/
4064	dec_slabs_node(s, node: slab_nid(slab: slab_free), objects: slab_free->objects);
4065	}
4066
4067	spin_unlock_irqrestore(lock: &n->list_lock, flags);
4068
4069	if (slab_free) {
4070	stat(s, si: FREE_SLAB);
4071	free_slab(s, slab: slab_free);
4072	}
4073	}
4074
4075	/*
4076	* Slow path handling. This may still be called frequently since objects
4077	* have a longer lifetime than the cpu slabs in most processing loads.
4078	*
4079	* So we still attempt to reduce cache line usage. Just take the slab
4080	* lock and free the item. If there is no additional partial slab
4081	* handling required then we can return immediately.
4082	*/
4083	static void __slab_free(struct kmem_cache *s, struct slab *slab,
4084	void *head, void *tail, int cnt,
4085	unsigned long addr)
4086
4087	{
4088	void *prior;
4089	int was_frozen;
4090	struct slab new;
4091	unsigned long counters;
4092	struct kmem_cache_node *n = NULL;
4093	unsigned long flags;
4094	bool on_node_partial;
4095
4096	stat(s, si: FREE_SLOWPATH);
4097
4098	if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
4099	free_to_partial_list(s, slab, head, tail, bulk_cnt: cnt, addr);
4100	return;
4101	}
4102
4103	do {
4104	if (unlikely(n)) {
4105	spin_unlock_irqrestore(lock: &n->list_lock, flags);
4106	n = NULL;
4107	}
4108	prior = slab->freelist;
4109	counters = slab->counters;
4110	set_freepointer(s, object: tail, fp: prior);
4111	new.counters = counters;
4112	was_frozen = new.frozen;
4113	new.inuse -= cnt;
4114	if ((!new.inuse || !prior) && !was_frozen) {
4115	/* Needs to be taken off a list */
4116	if (!kmem_cache_has_cpu_partial(s) || prior) {
4117
4118	n = get_node(s, node: slab_nid(slab));
4119	/*
4120	* Speculatively acquire the list_lock.
4121	* If the cmpxchg does not succeed then we may
4122	* drop the list_lock without any processing.
4123	*
4124	* Otherwise the list_lock will synchronize with
4125	* other processors updating the list of slabs.
4126	*/
4127	spin_lock_irqsave(&n->list_lock, flags);
4128
4129	on_node_partial = slab_test_node_partial(slab);
4130	}
4131	}
4132
4133	} while (!slab_update_freelist(s, slab,
4134	freelist_old: prior, counters_old: counters,
4135	freelist_new: head, counters_new: new.counters,
4136	n: "__slab_free"));
4137
4138	if (likely(!n)) {
4139
4140	if (likely(was_frozen)) {
4141	/*
4142	* The list lock was not taken therefore no list
4143	* activity can be necessary.
4144	*/
4145	stat(s, si: FREE_FROZEN);
4146	} else if (kmem_cache_has_cpu_partial(s) && !prior) {
4147	/*
4148	* If we started with a full slab then put it onto the
4149	* per cpu partial list.
4150	*/
4151	put_cpu_partial(s, slab, drain: 1);
4152	stat(s, si: CPU_PARTIAL_FREE);
4153	}
4154
4155	return;
4156	}
4157
4158	/*
4159	* This slab was partially empty but not on the per-node partial list,
4160	* in which case we shouldn't manipulate its list, just return.
4161	*/
4162	if (prior && !on_node_partial) {
4163	spin_unlock_irqrestore(lock: &n->list_lock, flags);
4164	return;
4165	}
4166
4167	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
4168	goto slab_empty;
4169
4170	/*
4171	* Objects left in the slab. If it was not on the partial list before
4172	* then add it.
4173	*/
4174	if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
4175	add_partial(n, slab, tail: DEACTIVATE_TO_TAIL);
4176	stat(s, si: FREE_ADD_PARTIAL);
4177	}
4178	spin_unlock_irqrestore(lock: &n->list_lock, flags);
4179	return;
4180
4181	slab_empty:
4182	if (prior) {
4183	/*
4184	* Slab on the partial list.
4185	*/
4186	remove_partial(n, slab);
4187	stat(s, si: FREE_REMOVE_PARTIAL);
4188	}
4189
4190	spin_unlock_irqrestore(lock: &n->list_lock, flags);
4191	stat(s, si: FREE_SLAB);
4192	discard_slab(s, slab);
4193	}
4194
4195	#ifndef CONFIG_SLUB_TINY
4196	/*
4197	* Fastpath with forced inlining to produce a kfree and kmem_cache_free that
4198	* can perform fastpath freeing without additional function calls.
4199	*
4200	* The fastpath is only possible if we are freeing to the current cpu slab
4201	* of this processor. This typically the case if we have just allocated
4202	* the item before.
4203	*
4204	* If fastpath is not possible then fall back to __slab_free where we deal
4205	* with all sorts of special processing.
4206	*
4207	* Bulk free of a freelist with several objects (all pointing to the
4208	* same slab) possible by specifying head and tail ptr, plus objects
4209	* count (cnt). Bulk free indicated by tail pointer being set.
4210	*/
4211	static __always_inline void do_slab_free(struct kmem_cache *s,
4212	struct slab *slab, void *head, void *tail,
4213	int cnt, unsigned long addr)
4214	{
4215	struct kmem_cache_cpu *c;
4216	unsigned long tid;
4217	void **freelist;
4218
4219	redo:
4220	/*
4221	* Determine the currently cpus per cpu slab.
4222	* The cpu may change afterward. However that does not matter since
4223	* data is retrieved via this pointer. If we are on the same cpu
4224	* during the cmpxchg then the free will succeed.
4225	*/
4226	c = raw_cpu_ptr(s->cpu_slab);
4227	tid = READ_ONCE(c->tid);
4228
4229	/* Same with comment on barrier() in slab_alloc_node() */
4230	barrier();
4231
4232	if (unlikely(slab != c->slab)) {
4233	__slab_free(s, slab, head, tail, cnt, addr);
4234	return;
4235	}
4236
4237	if (USE_LOCKLESS_FAST_PATH()) {
4238	freelist = READ_ONCE(c->freelist);
4239
4240	set_freepointer(s, tail, freelist);
4241
4242	if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {
4243	note_cmpxchg_failure("slab_free", s, tid);
4244	goto redo;
4245	}
4246	} else {
4247	/* Update the free list under the local lock */
4248	local_lock(&s->cpu_slab->lock);
4249	c = this_cpu_ptr(s->cpu_slab);
4250	if (unlikely(slab != c->slab)) {
4251	local_unlock(&s->cpu_slab->lock);
4252	goto redo;
4253	}
4254	tid = c->tid;
4255	freelist = c->freelist;
4256
4257	set_freepointer(s, tail, freelist);
4258	c->freelist = head;
4259	c->tid = next_tid(tid);
4260
4261	local_unlock(&s->cpu_slab->lock);
4262	}
4263	stat_add(s, FREE_FASTPATH, cnt);
4264	}
4265	#else /* CONFIG_SLUB_TINY */
4266	static void do_slab_free(struct kmem_cache *s,
4267	struct slab *slab, void *head, void *tail,
4268	int cnt, unsigned long addr)
4269	{
4270	__slab_free(s, slab, head, tail, cnt, addr);
4271	}
4272	#endif /* CONFIG_SLUB_TINY */
4273
4274	static __fastpath_inline
4275	void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
4276	unsigned long addr)
4277	{
4278	memcg_slab_free_hook(s, slab, p: &object, objects: 1);
4279
4280	if (likely(slab_free_hook(s, object, slab_want_init_on_free(s))))
4281	do_slab_free(s, slab, head: object, tail: object, cnt: 1, addr);
4282	}
4283
4284	static __fastpath_inline
4285	void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,
4286	void *tail, void **p, int cnt, unsigned long addr)
4287	{
4288	memcg_slab_free_hook(s, slab, p, objects: cnt);
4289	/*
4290	* With KASAN enabled slab_free_freelist_hook modifies the freelist
4291	* to remove objects, whose reuse must be delayed.
4292	*/
4293	if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt)))
4294	do_slab_free(s, slab, head, tail, cnt, addr);
4295	}
4296
4297	#ifdef CONFIG_KASAN_GENERIC
4298	void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
4299	{
4300	do_slab_free(cache, virt_to_slab(x), x, x, 1, addr);
4301	}
4302	#endif
4303
4304	static inline struct kmem_cache *virt_to_cache(const void *obj)
4305	{
4306	struct slab *slab;
4307
4308	slab = virt_to_slab(addr: obj);
4309	if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__))
4310	return NULL;
4311	return slab->slab_cache;
4312	}
4313
4314	static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
4315	{
4316	struct kmem_cache *cachep;
4317
4318	if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
4319	!kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
4320	return s;
4321
4322	cachep = virt_to_cache(obj: x);
4323	if (WARN(cachep && cachep != s,
4324	"%s: Wrong slab cache. %s but object is from %s\n",
4325	__func__, s->name, cachep->name))
4326	print_tracking(s: cachep, object: x);
4327	return cachep;
4328	}
4329
4330	/**
4331	* kmem_cache_free - Deallocate an object
4332	* @s: The cache the allocation was from.
4333	* @x: The previously allocated object.
4334	*
4335	* Free an object which was previously allocated from this
4336	* cache.
4337	*/
4338	void kmem_cache_free(struct kmem_cache *s, void *x)
4339	{
4340	s = cache_from_obj(s, x);
4341	if (!s)
4342	return;
4343	trace_kmem_cache_free(_RET_IP_, ptr: x, s);
4344	slab_free(s, slab: virt_to_slab(addr: x), object: x, _RET_IP_);
4345	}
4346	EXPORT_SYMBOL(kmem_cache_free);
4347
4348	static void free_large_kmalloc(struct folio *folio, void *object)
4349	{
4350	unsigned int order = folio_order(folio);
4351
4352	if (WARN_ON_ONCE(order == 0))
4353	pr_warn_once("object pointer: 0x%p\n", object);
4354
4355	kmemleak_free(ptr: object);
4356	kasan_kfree_large(ptr: object);
4357	kmsan_kfree_large(ptr: object);
4358
4359	lruvec_stat_mod_folio(folio, idx: NR_SLAB_UNRECLAIMABLE_B,
4360	val: -(PAGE_SIZE << order));
4361	folio_put(folio);
4362	}
4363
4364	/**
4365	* kfree - free previously allocated memory
4366	* @object: pointer returned by kmalloc() or kmem_cache_alloc()
4367	*
4368	* If @object is NULL, no operation is performed.
4369	*/
4370	void kfree(const void *object)
4371	{
4372	struct folio *folio;
4373	struct slab *slab;
4374	struct kmem_cache *s;
4375	void *x = (void *)object;
4376
4377	trace_kfree(_RET_IP_, ptr: object);
4378
4379	if (unlikely(ZERO_OR_NULL_PTR(object)))
4380	return;
4381
4382	folio = virt_to_folio(x: object);
4383	if (unlikely(!folio_test_slab(folio))) {
4384	free_large_kmalloc(folio, object: (void *)object);
4385	return;
4386	}
4387
4388	slab = folio_slab(folio);
4389	s = slab->slab_cache;
4390	slab_free(s, slab, object: x, _RET_IP_);
4391	}
4392	EXPORT_SYMBOL(kfree);
4393
4394	struct detached_freelist {
4395	struct slab *slab;
4396	void *tail;
4397	void *freelist;
4398	int cnt;
4399	struct kmem_cache *s;
4400	};
4401
4402	/*
4403	* This function progressively scans the array with free objects (with
4404	* a limited look ahead) and extract objects belonging to the same
4405	* slab. It builds a detached freelist directly within the given
4406	* slab/objects. This can happen without any need for
4407	* synchronization, because the objects are owned by running process.
4408	* The freelist is build up as a single linked list in the objects.
4409	* The idea is, that this detached freelist can then be bulk
4410	* transferred to the real freelist(s), but only requiring a single
4411	* synchronization primitive. Look ahead in the array is limited due
4412	* to performance reasons.
4413	*/
4414	static inline
4415	int build_detached_freelist(struct kmem_cache *s, size_t size,
4416	void **p, struct detached_freelist *df)
4417	{
4418	int lookahead = 3;
4419	void *object;
4420	struct folio *folio;
4421	size_t same;
4422
4423	object = p[--size];
4424	folio = virt_to_folio(x: object);
4425	if (!s) {
4426	/* Handle kalloc'ed objects */
4427	if (unlikely(!folio_test_slab(folio))) {
4428	free_large_kmalloc(folio, object);
4429	df->slab = NULL;
4430	return size;
4431	}
4432	/* Derive kmem_cache from object */
4433	df->slab = folio_slab(folio);
4434	df->s = df->slab->slab_cache;
4435	} else {
4436	df->slab = folio_slab(folio);
4437	df->s = cache_from_obj(s, x: object); /* Support for memcg */
4438	}
4439
4440	/* Start new detached freelist */
4441	df->tail = object;
4442	df->freelist = object;
4443	df->cnt = 1;
4444
4445	if (is_kfence_address(addr: object))
4446	return size;
4447
4448	set_freepointer(s: df->s, object, NULL);
4449
4450	same = size;
4451	while (size) {
4452	object = p[--size];
4453	/* df->slab is always set at this point */
4454	if (df->slab == virt_to_slab(addr: object)) {
4455	/* Opportunity build freelist */
4456	set_freepointer(s: df->s, object, fp: df->freelist);
4457	df->freelist = object;
4458	df->cnt++;
4459	same--;
4460	if (size != same)
4461	swap(p[size], p[same]);
4462	continue;
4463	}
4464
4465	/* Limit look ahead search */
4466	if (!--lookahead)
4467	break;
4468	}
4469
4470	return same;
4471	}
4472
4473	/*
4474	* Internal bulk free of objects that were not initialised by the post alloc
4475	* hooks and thus should not be processed by the free hooks
4476	*/
4477	static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
4478	{
4479	if (!size)
4480	return;
4481
4482	do {
4483	struct detached_freelist df;
4484
4485	size = build_detached_freelist(s, size, p, df: &df);
4486	if (!df.slab)
4487	continue;
4488
4489	do_slab_free(s: df.s, slab: df.slab, head: df.freelist, tail: df.tail, cnt: df.cnt,
4490	_RET_IP_);
4491	} while (likely(size));
4492	}
4493
4494	/* Note that interrupts must be enabled when calling this function. */
4495	void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
4496	{
4497	if (!size)
4498	return;
4499
4500	do {
4501	struct detached_freelist df;
4502
4503	size = build_detached_freelist(s, size, p, df: &df);
4504	if (!df.slab)
4505	continue;
4506
4507	slab_free_bulk(s: df.s, slab: df.slab, head: df.freelist, tail: df.tail, p: &p[size],
4508	cnt: df.cnt, _RET_IP_);
4509	} while (likely(size));
4510	}
4511	EXPORT_SYMBOL(kmem_cache_free_bulk);
4512
4513	#ifndef CONFIG_SLUB_TINY
4514	static inline
4515	int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
4516	void **p)
4517	{
4518	struct kmem_cache_cpu *c;
4519	unsigned long irqflags;
4520	int i;
4521
4522	/*
4523	* Drain objects in the per cpu slab, while disabling local
4524	* IRQs, which protects against PREEMPT and interrupts
4525	* handlers invoking normal fastpath.
4526	*/
4527	c = slub_get_cpu_ptr(s->cpu_slab);
4528	local_lock_irqsave(&s->cpu_slab->lock, irqflags);
4529
4530	for (i = 0; i < size; i++) {
4531	void *object = kfence_alloc(s, s->object_size, flags);
4532
4533	if (unlikely(object)) {
4534	p[i] = object;
4535	continue;
4536	}
4537
4538	object = c->freelist;
4539	if (unlikely(!object)) {
4540	/*
4541	* We may have removed an object from c->freelist using
4542	* the fastpath in the previous iteration; in that case,
4543	* c->tid has not been bumped yet.
4544	* Since ___slab_alloc() may reenable interrupts while
4545	* allocating memory, we should bump c->tid now.
4546	*/
4547	c->tid = next_tid(c->tid);
4548
4549	local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
4550
4551	/*
4552	* Invoking slow path likely have side-effect
4553	* of re-populating per CPU c->freelist
4554	*/
4555	p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
4556	_RET_IP_, c, s->object_size);
4557	if (unlikely(!p[i]))
4558	goto error;
4559
4560	c = this_cpu_ptr(s->cpu_slab);
4561	maybe_wipe_obj_freeptr(s, p[i]);
4562
4563	local_lock_irqsave(&s->cpu_slab->lock, irqflags);
4564
4565	continue; /* goto for-loop */
4566	}
4567	c->freelist = get_freepointer(s, object);
4568	p[i] = object;
4569	maybe_wipe_obj_freeptr(s, p[i]);
4570	stat(s, ALLOC_FASTPATH);
4571	}
4572	c->tid = next_tid(c->tid);
4573	local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
4574	slub_put_cpu_ptr(s->cpu_slab);
4575
4576	return i;
4577
4578	error:
4579	slub_put_cpu_ptr(s->cpu_slab);
4580	__kmem_cache_free_bulk(s, i, p);
4581	return 0;
4582
4583	}
4584	#else /* CONFIG_SLUB_TINY */
4585	static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
4586	size_t size, void **p)
4587	{
4588	int i;
4589
4590	for (i = 0; i < size; i++) {
4591	void *object = kfence_alloc(s, size: s->object_size, flags);
4592
4593	if (unlikely(object)) {
4594	p[i] = object;
4595	continue;
4596	}
4597
4598	p[i] = __slab_alloc_node(s, gfpflags: flags, NUMA_NO_NODE,
4599	_RET_IP_, orig_size: s->object_size);
4600	if (unlikely(!p[i]))
4601	goto error;
4602
4603	maybe_wipe_obj_freeptr(s, obj: p[i]);
4604	}
4605
4606	return i;
4607
4608	error:
4609	__kmem_cache_free_bulk(s, size: i, p);
4610	return 0;
4611	}
4612	#endif /* CONFIG_SLUB_TINY */
4613
4614	/* Note that interrupts must be enabled when calling this function. */
4615	int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
4616	void **p)
4617	{
4618	int i;
4619	struct obj_cgroup *objcg = NULL;
4620
4621	if (!size)
4622	return 0;
4623
4624	/* memcg and kmem_cache debug support */
4625	s = slab_pre_alloc_hook(s, NULL, objcgp: &objcg, size, flags);
4626	if (unlikely(!s))
4627	return 0;
4628
4629	i = __kmem_cache_alloc_bulk(s, flags, size, p);
4630
4631	/*
4632	* memcg and kmem_cache debug support and memory initialization.
4633	* Done outside of the IRQ disabled fastpath loop.
4634	*/
4635	if (likely(i != 0)) {
4636	slab_post_alloc_hook(s, objcg, flags, size, p,
4637	init: slab_want_init_on_alloc(flags, c: s), orig_size: s->object_size);
4638	} else {
4639	memcg_slab_alloc_error_hook(s, objects: size, objcg);
4640	}
4641
4642	return i;
4643	}
4644	EXPORT_SYMBOL(kmem_cache_alloc_bulk);
4645
4646
4647	/*
4648	* Object placement in a slab is made very easy because we always start at
4649	* offset 0. If we tune the size of the object to the alignment then we can
4650	* get the required alignment by putting one properly sized object after
4651	* another.
4652	*
4653	* Notice that the allocation order determines the sizes of the per cpu
4654	* caches. Each processor has always one slab available for allocations.
4655	* Increasing the allocation order reduces the number of times that slabs
4656	* must be moved on and off the partial lists and is therefore a factor in
4657	* locking overhead.
4658	*/
4659
4660	/*
4661	* Minimum / Maximum order of slab pages. This influences locking overhead
4662	* and slab fragmentation. A higher order reduces the number of partial slabs
4663	* and increases the number of allocations possible without having to
4664	* take the list_lock.
4665	*/
4666	static unsigned int slub_min_order;
4667	static unsigned int slub_max_order =
4668	IS_ENABLED(CONFIG_SLUB_TINY) ? 1 : PAGE_ALLOC_COSTLY_ORDER;
4669	static unsigned int slub_min_objects;
4670
4671	/*
4672	* Calculate the order of allocation given an slab object size.
4673	*
4674	* The order of allocation has significant impact on performance and other
4675	* system components. Generally order 0 allocations should be preferred since
4676	* order 0 does not cause fragmentation in the page allocator. Larger objects
4677	* be problematic to put into order 0 slabs because there may be too much
4678	* unused space left. We go to a higher order if more than 1/16th of the slab
4679	* would be wasted.
4680	*
4681	* In order to reach satisfactory performance we must ensure that a minimum
4682	* number of objects is in one slab. Otherwise we may generate too much
4683	* activity on the partial lists which requires taking the list_lock. This is
4684	* less a concern for large slabs though which are rarely used.
4685	*
4686	* slab_max_order specifies the order where we begin to stop considering the
4687	* number of objects in a slab as critical. If we reach slab_max_order then
4688	* we try to keep the page order as low as possible. So we accept more waste
4689	* of space in favor of a small page order.
4690	*
4691	* Higher order allocations also allow the placement of more objects in a
4692	* slab and thereby reduce object handling overhead. If the user has
4693	* requested a higher minimum order then we start with that one instead of
4694	* the smallest order which will fit the object.
4695	*/
4696	static inline unsigned int calc_slab_order(unsigned int size,
4697	unsigned int min_order, unsigned int max_order,
4698	unsigned int fract_leftover)
4699	{
4700	unsigned int order;
4701
4702	for (order = min_order; order <= max_order; order++) {
4703
4704	unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
4705	unsigned int rem;
4706
4707	rem = slab_size % size;
4708
4709	if (rem <= slab_size / fract_leftover)
4710	break;
4711	}
4712
4713	return order;
4714	}
4715
4716	static inline int calculate_order(unsigned int size)
4717	{
4718	unsigned int order;
4719	unsigned int min_objects;
4720	unsigned int max_objects;
4721	unsigned int min_order;
4722
4723	min_objects = slub_min_objects;
4724	if (!min_objects) {
4725	/*
4726	* Some architectures will only update present cpus when
4727	* onlining them, so don't trust the number if it's just 1. But
4728	* we also don't want to use nr_cpu_ids always, as on some other
4729	* architectures, there can be many possible cpus, but never
4730	* onlined. Here we compromise between trying to avoid too high
4731	* order on systems that appear larger than they are, and too
4732	* low order on systems that appear smaller than they are.
4733	*/
4734	unsigned int nr_cpus = num_present_cpus();
4735	if (nr_cpus <= 1)
4736	nr_cpus = nr_cpu_ids;
4737	min_objects = 4 * (fls(x: nr_cpus) + 1);
4738	}
4739	/* min_objects can't be 0 because get_order(0) is undefined */
4740	max_objects = max(order_objects(slub_max_order, size), 1U);
4741	min_objects = min(min_objects, max_objects);
4742
4743	min_order = max_t(unsigned int, slub_min_order,
4744	get_order(min_objects * size));
4745	if (order_objects(order: min_order, size) > MAX_OBJS_PER_PAGE)
4746	return get_order(size: size * MAX_OBJS_PER_PAGE) - 1;
4747
4748	/*
4749	* Attempt to find best configuration for a slab. This works by first
4750	* attempting to generate a layout with the best possible configuration
4751	* and backing off gradually.
4752	*
4753	* We start with accepting at most 1/16 waste and try to find the
4754	* smallest order from min_objects-derived/slab_min_order up to
4755	* slab_max_order that will satisfy the constraint. Note that increasing
4756	* the order can only result in same or less fractional waste, not more.
4757	*
4758	* If that fails, we increase the acceptable fraction of waste and try
4759	* again. The last iteration with fraction of 1/2 would effectively
4760	* accept any waste and give us the order determined by min_objects, as
4761	* long as at least single object fits within slab_max_order.
4762	*/
4763	for (unsigned int fraction = 16; fraction > 1; fraction /= 2) {
4764	order = calc_slab_order(size, min_order, max_order: slub_max_order,
4765	fract_leftover: fraction);
4766	if (order <= slub_max_order)
4767	return order;
4768	}
4769
4770	/*
4771	* Doh this slab cannot be placed using slab_max_order.
4772	*/
4773	order = get_order(size);
4774	if (order <= MAX_PAGE_ORDER)
4775	return order;
4776	return -ENOSYS;
4777	}
4778
4779	static void
4780	init_kmem_cache_node(struct kmem_cache_node *n)
4781	{
4782	n->nr_partial = 0;
4783	spin_lock_init(&n->list_lock);
4784	INIT_LIST_HEAD(list: &n->partial);
4785	#ifdef CONFIG_SLUB_DEBUG
4786	atomic_long_set(&n->nr_slabs, 0);
4787	atomic_long_set(&n->total_objects, 0);
4788	INIT_LIST_HEAD(&n->full);
4789	#endif
4790	}
4791
4792	#ifndef CONFIG_SLUB_TINY
4793	static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
4794	{
4795	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
4796	NR_KMALLOC_TYPES * KMALLOC_SHIFT_HIGH *
4797	sizeof(struct kmem_cache_cpu));
4798
4799	/*
4800	* Must align to double word boundary for the double cmpxchg
4801	* instructions to work; see __pcpu_double_call_return_bool().
4802	*/
4803	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
4804	2 * sizeof(void *));
4805
4806	if (!s->cpu_slab)
4807	return 0;
4808
4809	init_kmem_cache_cpus(s);
4810
4811	return 1;
4812	}
4813	#else
4814	static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
4815	{
4816	return 1;
4817	}
4818	#endif /* CONFIG_SLUB_TINY */
4819
4820	static struct kmem_cache *kmem_cache_node;
4821
4822	/*
4823	* No kmalloc_node yet so do it by hand. We know that this is the first
4824	* slab on the node for this slabcache. There are no concurrent accesses
4825	* possible.
4826	*
4827	* Note that this function only works on the kmem_cache_node
4828	* when allocating for the kmem_cache_node. This is used for bootstrapping
4829	* memory on a fresh node that has no slab structures yet.
4830	*/
4831	static void early_kmem_cache_node_alloc(int node)
4832	{
4833	struct slab *slab;
4834	struct kmem_cache_node *n;
4835
4836	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
4837
4838	slab = new_slab(s: kmem_cache_node, GFP_NOWAIT, node);
4839
4840	BUG_ON(!slab);
4841	if (slab_nid(slab) != node) {
4842	pr_err("SLUB: Unable to allocate memory from node %d\n", node);
4843	pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
4844	}
4845
4846	n = slab->freelist;
4847	BUG_ON(!n);
4848	#ifdef CONFIG_SLUB_DEBUG
4849	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
4850	init_tracking(kmem_cache_node, n);
4851	#endif
4852	n = kasan_slab_alloc(s: kmem_cache_node, object: n, GFP_KERNEL, init: false);
4853	slab->freelist = get_freepointer(s: kmem_cache_node, object: n);
4854	slab->inuse = 1;
4855	kmem_cache_node->node[node] = n;
4856	init_kmem_cache_node(n);
4857	inc_slabs_node(s: kmem_cache_node, node, objects: slab->objects);
4858
4859	/*
4860	* No locks need to be taken here as it has just been
4861	* initialized and there is no concurrent access.
4862	*/
4863	__add_partial(n, slab, tail: DEACTIVATE_TO_HEAD);
4864	}
4865
4866	static void free_kmem_cache_nodes(struct kmem_cache *s)
4867	{
4868	int node;
4869	struct kmem_cache_node *n;
4870
4871	for_each_kmem_cache_node(s, node, n) {
4872	s->node[node] = NULL;
4873	kmem_cache_free(kmem_cache_node, n);
4874	}
4875	}
4876
4877	void __kmem_cache_release(struct kmem_cache *s)
4878	{
4879	cache_random_seq_destroy(cachep: s);
4880	#ifndef CONFIG_SLUB_TINY
4881	free_percpu(s->cpu_slab);
4882	#endif
4883	free_kmem_cache_nodes(s);
4884	}
4885
4886	static int init_kmem_cache_nodes(struct kmem_cache *s)
4887	{
4888	int node;
4889
4890	for_each_node_mask(node, slab_nodes) {
4891	struct kmem_cache_node *n;
4892
4893	if (slab_state == DOWN) {
4894	early_kmem_cache_node_alloc(node);
4895	continue;
4896	}
4897	n = kmem_cache_alloc_node(kmem_cache_node,
4898	GFP_KERNEL, node);
4899
4900	if (!n) {
4901	free_kmem_cache_nodes(s);
4902	return 0;
4903	}
4904
4905	init_kmem_cache_node(n);
4906	s->node[node] = n;
4907	}
4908	return 1;
4909	}
4910
4911	static void set_cpu_partial(struct kmem_cache *s)
4912	{
4913	#ifdef CONFIG_SLUB_CPU_PARTIAL
4914	unsigned int nr_objects;
4915
4916	/*
4917	* cpu_partial determined the maximum number of objects kept in the
4918	* per cpu partial lists of a processor.
4919	*
4920	* Per cpu partial lists mainly contain slabs that just have one
4921	* object freed. If they are used for allocation then they can be
4922	* filled up again with minimal effort. The slab will never hit the
4923	* per node partial lists and therefore no locking will be required.
4924	*
4925	* For backwards compatibility reasons, this is determined as number
4926	* of objects, even though we now limit maximum number of pages, see
4927	* slub_set_cpu_partial()
4928	*/
4929	if (!kmem_cache_has_cpu_partial(s))
4930	nr_objects = 0;
4931	else if (s->size >= PAGE_SIZE)
4932	nr_objects = 6;
4933	else if (s->size >= 1024)
4934	nr_objects = 24;
4935	else if (s->size >= 256)
4936	nr_objects = 52;
4937	else
4938	nr_objects = 120;
4939
4940	slub_set_cpu_partial(s, nr_objects);
4941	#endif
4942	}
4943
4944	/*
4945	* calculate_sizes() determines the order and the distribution of data within
4946	* a slab object.
4947	*/
4948	static int calculate_sizes(struct kmem_cache *s)
4949	{
4950	slab_flags_t flags = s->flags;
4951	unsigned int size = s->object_size;
4952	unsigned int order;
4953
4954	/*
4955	* Round up object size to the next word boundary. We can only
4956	* place the free pointer at word boundaries and this determines
4957	* the possible location of the free pointer.
4958	*/
4959	size = ALIGN(size, sizeof(void *));
4960
4961	#ifdef CONFIG_SLUB_DEBUG
4962	/*
4963	* Determine if we can poison the object itself. If the user of
4964	* the slab may touch the object after free or before allocation
4965	* then we should never poison the object itself.
4966	*/
4967	if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
4968	!s->ctor)
4969	s->flags |= __OBJECT_POISON;
4970	else
4971	s->flags &= ~__OBJECT_POISON;
4972
4973
4974	/*
4975	* If we are Redzoning then check if there is some space between the
4976	* end of the object and the free pointer. If not then add an
4977	* additional word to have some bytes to store Redzone information.
4978	*/
4979	if ((flags & SLAB_RED_ZONE) && size == s->object_size)
4980	size += sizeof(void *);
4981	#endif
4982
4983	/*
4984	* With that we have determined the number of bytes in actual use
4985	* by the object and redzoning.
4986	*/
4987	s->inuse = size;
4988
4989	if (slub_debug_orig_size(s) ||
4990	(flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
4991	((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
4992	s->ctor) {
4993	/*
4994	* Relocate free pointer after the object if it is not
4995	* permitted to overwrite the first word of the object on
4996	* kmem_cache_free.
4997	*
4998	* This is the case if we do RCU, have a constructor or
4999	* destructor, are poisoning the objects, or are
5000	* redzoning an object smaller than sizeof(void *).
5001	*
5002	* The assumption that s->offset >= s->inuse means free
5003	* pointer is outside of the object is used in the
5004	* freeptr_outside_object() function. If that is no
5005	* longer true, the function needs to be modified.
5006	*/
5007	s->offset = size;
5008	size += sizeof(void *);
5009	} else {
5010	/*
5011	* Store freelist pointer near middle of object to keep
5012	* it away from the edges of the object to avoid small
5013	* sized over/underflows from neighboring allocations.
5014	*/
5015	s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
5016	}
5017
5018	#ifdef CONFIG_SLUB_DEBUG
5019	if (flags & SLAB_STORE_USER) {
5020	/*
5021	* Need to store information about allocs and frees after
5022	* the object.
5023	*/
5024	size += 2 * sizeof(struct track);
5025
5026	/* Save the original kmalloc request size */
5027	if (flags & SLAB_KMALLOC)
5028	size += sizeof(unsigned int);
5029	}
5030	#endif
5031
5032	kasan_cache_create(cache: s, size: &size, flags: &s->flags);
5033	#ifdef CONFIG_SLUB_DEBUG
5034	if (flags & SLAB_RED_ZONE) {
5035	/*
5036	* Add some empty padding so that we can catch
5037	* overwrites from earlier objects rather than let
5038	* tracking information or the free pointer be
5039	* corrupted if a user writes before the start
5040	* of the object.
5041	*/
5042	size += sizeof(void *);
5043
5044	s->red_left_pad = sizeof(void *);
5045	s->red_left_pad = ALIGN(s->red_left_pad, s->align);
5046	size += s->red_left_pad;
5047	}
5048	#endif
5049
5050	/*
5051	* SLUB stores one object immediately after another beginning from
5052	* offset 0. In order to align the objects we have to simply size
5053	* each object to conform to the alignment.
5054	*/
5055	size = ALIGN(size, s->align);
5056	s->size = size;
5057	s->reciprocal_size = reciprocal_value(d: size);
5058	order = calculate_order(size);
5059
5060	if ((int)order < 0)
5061	return 0;
5062
5063	s->allocflags = 0;
5064	if (order)
5065	s->allocflags |= __GFP_COMP;
5066
5067	if (s->flags & SLAB_CACHE_DMA)
5068	s->allocflags |= GFP_DMA;
5069
5070	if (s->flags & SLAB_CACHE_DMA32)
5071	s->allocflags |= GFP_DMA32;
5072
5073	if (s->flags & SLAB_RECLAIM_ACCOUNT)
5074	s->allocflags |= __GFP_RECLAIMABLE;
5075
5076	/*
5077	* Determine the number of objects per slab
5078	*/
5079	s->oo = oo_make(order, size);
5080	s->min = oo_make(order: get_order(size), size);
5081
5082	return !!oo_objects(x: s->oo);
5083	}
5084
5085	static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
5086	{
5087	s->flags = kmem_cache_flags(flags, name: s->name);
5088	#ifdef CONFIG_SLAB_FREELIST_HARDENED
5089	s->random = get_random_long();
5090	#endif
5091
5092	if (!calculate_sizes(s))
5093	goto error;
5094	if (disable_higher_order_debug) {
5095	/*
5096	* Disable debugging flags that store metadata if the min slab
5097	* order increased.
5098	*/
5099	if (get_order(size: s->size) > get_order(size: s->object_size)) {
5100	s->flags &= ~DEBUG_METADATA_FLAGS;
5101	s->offset = 0;
5102	if (!calculate_sizes(s))
5103	goto error;
5104	}
5105	}
5106
5107	#ifdef system_has_freelist_aba
5108	if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) {
5109	/* Enable fast mode */
5110	s->flags |= __CMPXCHG_DOUBLE;
5111	}
5112	#endif
5113
5114	/*
5115	* The larger the object size is, the more slabs we want on the partial
5116	* list to avoid pounding the page allocator excessively.
5117	*/
5118	s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
5119	s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
5120
5121	set_cpu_partial(s);
5122
5123	#ifdef CONFIG_NUMA
5124	s->remote_node_defrag_ratio = 1000;
5125	#endif
5126
5127	/* Initialize the pre-computed randomized freelist if slab is up */
5128	if (slab_state >= UP) {
5129	if (init_cache_random_seq(s))
5130	goto error;
5131	}
5132
5133	if (!init_kmem_cache_nodes(s))
5134	goto error;
5135
5136	if (alloc_kmem_cache_cpus(s))
5137	return 0;
5138
5139	error:
5140	__kmem_cache_release(s);
5141	return -EINVAL;
5142	}
5143
5144	static void list_slab_objects(struct kmem_cache *s, struct slab *slab,
5145	const char *text)
5146	{
5147	#ifdef CONFIG_SLUB_DEBUG
5148	void *addr = slab_address(slab);
5149	void *p;
5150
5151	slab_err(s, slab, text, s->name);
5152
5153	spin_lock(&object_map_lock);
5154	__fill_map(object_map, s, slab);
5155
5156	for_each_object(p, s, addr, slab->objects) {
5157
5158	if (!test_bit(__obj_to_index(s, addr, p), object_map)) {
5159	pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
5160	print_tracking(s, p);
5161	}
5162	}
5163	spin_unlock(&object_map_lock);
5164	#endif
5165	}
5166
5167	/*
5168	* Attempt to free all partial slabs on a node.
5169	* This is called from __kmem_cache_shutdown(). We must take list_lock
5170	* because sysfs file might still access partial list after the shutdowning.
5171	*/
5172	static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
5173	{
5174	LIST_HEAD(discard);
5175	struct slab *slab, *h;
5176
5177	BUG_ON(irqs_disabled());
5178	spin_lock_irq(lock: &n->list_lock);
5179	list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
5180	if (!slab->inuse) {
5181	remove_partial(n, slab);
5182	list_add(new: &slab->slab_list, head: &discard);
5183	} else {
5184	list_slab_objects(s, slab,
5185	text: "Objects remaining in %s on __kmem_cache_shutdown()");
5186	}
5187	}
5188	spin_unlock_irq(lock: &n->list_lock);
5189
5190	list_for_each_entry_safe(slab, h, &discard, slab_list)
5191	discard_slab(s, slab);
5192	}
5193
5194	bool __kmem_cache_empty(struct kmem_cache *s)
5195	{
5196	int node;
5197	struct kmem_cache_node *n;
5198
5199	for_each_kmem_cache_node(s, node, n)
5200	if (n->nr_partial || node_nr_slabs(n))
5201	return false;
5202	return true;
5203	}
5204
5205	/*
5206	* Release all resources used by a slab cache.
5207	*/
5208	int __kmem_cache_shutdown(struct kmem_cache *s)
5209	{
5210	int node;
5211	struct kmem_cache_node *n;
5212
5213	flush_all_cpus_locked(s);
5214	/* Attempt to free all objects */
5215	for_each_kmem_cache_node(s, node, n) {
5216	free_partial(s, n);
5217	if (n->nr_partial || node_nr_slabs(n))
5218	return 1;
5219	}
5220	return 0;
5221	}
5222
5223	#ifdef CONFIG_PRINTK
5224	void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
5225	{
5226	void *base;
5227	int __maybe_unused i;
5228	unsigned int objnr;
5229	void *objp;
5230	void *objp0;
5231	struct kmem_cache *s = slab->slab_cache;
5232	struct track __maybe_unused *trackp;
5233
5234	kpp->kp_ptr = object;
5235	kpp->kp_slab = slab;
5236	kpp->kp_slab_cache = s;
5237	base = slab_address(slab);
5238	objp0 = kasan_reset_tag(addr: object);
5239	#ifdef CONFIG_SLUB_DEBUG
5240	objp = restore_red_left(s, objp0);
5241	#else
5242	objp = objp0;
5243	#endif
5244	objnr = obj_to_index(cache: s, slab, obj: objp);
5245	kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
5246	objp = base + s->size * objnr;
5247	kpp->kp_objp = objp;
5248	if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
5249	|| (objp - base) % s->size) ||
5250	!(s->flags & SLAB_STORE_USER))
5251	return;
5252	#ifdef CONFIG_SLUB_DEBUG
5253	objp = fixup_red_left(s, objp);
5254	trackp = get_track(s, objp, TRACK_ALLOC);
5255	kpp->kp_ret = (void *)trackp->addr;
5256	#ifdef CONFIG_STACKDEPOT
5257	{
5258	depot_stack_handle_t handle;
5259	unsigned long *entries;
5260	unsigned int nr_entries;
5261
5262	handle = READ_ONCE(trackp->handle);
5263	if (handle) {
5264	nr_entries = stack_depot_fetch(handle, &entries);
5265	for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
5266	kpp->kp_stack[i] = (void *)entries[i];
5267	}
5268
5269	trackp = get_track(s, objp, TRACK_FREE);
5270	handle = READ_ONCE(trackp->handle);
5271	if (handle) {
5272	nr_entries = stack_depot_fetch(handle, &entries);
5273	for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
5274	kpp->kp_free_stack[i] = (void *)entries[i];
5275	}
5276	}
5277	#endif
5278	#endif
5279	}
5280	#endif
5281
5282	/********************************************************************
5283	* Kmalloc subsystem
5284	*******************************************************************/
5285
5286	static int __init setup_slub_min_order(char *str)
5287	{
5288	get_option(str: &str, pint: (int *)&slub_min_order);
5289
5290	if (slub_min_order > slub_max_order)
5291	slub_max_order = slub_min_order;
5292
5293	return 1;
5294	}
5295
5296	__setup("slab_min_order=", setup_slub_min_order);
5297	__setup_param("slub_min_order=", slub_min_order, setup_slub_min_order, 0);
5298
5299
5300	static int __init setup_slub_max_order(char *str)
5301	{
5302	get_option(str: &str, pint: (int *)&slub_max_order);
5303	slub_max_order = min_t(unsigned int, slub_max_order, MAX_PAGE_ORDER);
5304
5305	if (slub_min_order > slub_max_order)
5306	slub_min_order = slub_max_order;
5307
5308	return 1;
5309	}
5310
5311	__setup("slab_max_order=", setup_slub_max_order);
5312	__setup_param("slub_max_order=", slub_max_order, setup_slub_max_order, 0);
5313
5314	static int __init setup_slub_min_objects(char *str)
5315	{
5316	get_option(str: &str, pint: (int *)&slub_min_objects);
5317
5318	return 1;
5319	}
5320
5321	__setup("slab_min_objects=", setup_slub_min_objects);
5322	__setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0);
5323
5324	#ifdef CONFIG_HARDENED_USERCOPY
5325	/*
5326	* Rejects incorrectly sized objects and objects that are to be copied
5327	* to/from userspace but do not fall entirely within the containing slab
5328	* cache's usercopy region.
5329	*
5330	* Returns NULL if check passes, otherwise const char * to name of cache
5331	* to indicate an error.
5332	*/
5333	void __check_heap_object(const void *ptr, unsigned long n,
5334	const struct slab *slab, bool to_user)
5335	{
5336	struct kmem_cache *s;
5337	unsigned int offset;
5338	bool is_kfence = is_kfence_address(addr: ptr);
5339
5340	ptr = kasan_reset_tag(addr: ptr);
5341
5342	/* Find object and usable object size. */
5343	s = slab->slab_cache;
5344
5345	/* Reject impossible pointers. */
5346	if (ptr < slab_address(slab))
5347	usercopy_abort(name: "SLUB object not in SLUB page?!", NULL,
5348	to_user, offset: 0, len: n);
5349
5350	/* Find offset within object. */
5351	if (is_kfence)
5352	offset = ptr - kfence_object_start(addr: ptr);
5353	else
5354	offset = (ptr - slab_address(slab)) % s->size;
5355
5356	/* Adjust for redzone and reject if within the redzone. */
5357	if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
5358	if (offset < s->red_left_pad)
5359	usercopy_abort(name: "SLUB object in left red zone",
5360	detail: s->name, to_user, offset, len: n);
5361	offset -= s->red_left_pad;
5362	}
5363
5364	/* Allow address range falling entirely within usercopy region. */
5365	if (offset >= s->useroffset &&
5366	offset - s->useroffset <= s->usersize &&
5367	n <= s->useroffset - offset + s->usersize)
5368	return;
5369
5370	usercopy_abort(name: "SLUB object", detail: s->name, to_user, offset, len: n);
5371	}
5372	#endif /* CONFIG_HARDENED_USERCOPY */
5373
5374	#define SHRINK_PROMOTE_MAX 32
5375
5376	/*
5377	* kmem_cache_shrink discards empty slabs and promotes the slabs filled
5378	* up most to the head of the partial lists. New allocations will then
5379	* fill those up and thus they can be removed from the partial lists.
5380	*
5381	* The slabs with the least items are placed last. This results in them
5382	* being allocated from last increasing the chance that the last objects
5383	* are freed in them.
5384	*/
5385	static int __kmem_cache_do_shrink(struct kmem_cache *s)
5386	{
5387	int node;
5388	int i;
5389	struct kmem_cache_node *n;
5390	struct slab *slab;
5391	struct slab *t;
5392	struct list_head discard;
5393	struct list_head promote[SHRINK_PROMOTE_MAX];
5394	unsigned long flags;
5395	int ret = 0;
5396
5397	for_each_kmem_cache_node(s, node, n) {
5398	INIT_LIST_HEAD(list: &discard);
5399	for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
5400	INIT_LIST_HEAD(list: promote + i);
5401
5402	spin_lock_irqsave(&n->list_lock, flags);
5403
5404	/*
5405	* Build lists of slabs to discard or promote.
5406	*
5407	* Note that concurrent frees may occur while we hold the
5408	* list_lock. slab->inuse here is the upper limit.
5409	*/
5410	list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
5411	int free = slab->objects - slab->inuse;
5412
5413	/* Do not reread slab->inuse */
5414	barrier();
5415
5416	/* We do not keep full slabs on the list */
5417	BUG_ON(free <= 0);
5418
5419	if (free == slab->objects) {
5420	list_move(list: &slab->slab_list, head: &discard);
5421	slab_clear_node_partial(slab);
5422	n->nr_partial--;
5423	dec_slabs_node(s, node, objects: slab->objects);
5424	} else if (free <= SHRINK_PROMOTE_MAX)
5425	list_move(list: &slab->slab_list, head: promote + free - 1);
5426	}
5427
5428	/*
5429	* Promote the slabs filled up most to the head of the
5430	* partial list.
5431	*/
5432	for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
5433	list_splice(list: promote + i, head: &n->partial);
5434
5435	spin_unlock_irqrestore(lock: &n->list_lock, flags);
5436
5437	/* Release empty slabs */
5438	list_for_each_entry_safe(slab, t, &discard, slab_list)
5439	free_slab(s, slab);
5440
5441	if (node_nr_slabs(n))
5442	ret = 1;
5443	}
5444
5445	return ret;
5446	}
5447
5448	int __kmem_cache_shrink(struct kmem_cache *s)
5449	{
5450	flush_all(s);
5451	return __kmem_cache_do_shrink(s);
5452	}
5453
5454	static int slab_mem_going_offline_callback(void *arg)
5455	{
5456	struct kmem_cache *s;
5457
5458	mutex_lock(&slab_mutex);
5459	list_for_each_entry(s, &slab_caches, list) {
5460	flush_all_cpus_locked(s);
5461	__kmem_cache_do_shrink(s);
5462	}
5463	mutex_unlock(lock: &slab_mutex);
5464
5465	return 0;
5466	}
5467
5468	static void slab_mem_offline_callback(void *arg)
5469	{
5470	struct memory_notify *marg = arg;
5471	int offline_node;
5472
5473	offline_node = marg->status_change_nid_normal;
5474
5475	/*
5476	* If the node still has available memory. we need kmem_cache_node
5477	* for it yet.
5478	*/
5479	if (offline_node < 0)
5480	return;
5481
5482	mutex_lock(&slab_mutex);
5483	node_clear(offline_node, slab_nodes);
5484	/*
5485	* We no longer free kmem_cache_node structures here, as it would be
5486	* racy with all get_node() users, and infeasible to protect them with
5487	* slab_mutex.
5488	*/
5489	mutex_unlock(lock: &slab_mutex);
5490	}
5491
5492	static int slab_mem_going_online_callback(void *arg)
5493	{
5494	struct kmem_cache_node *n;
5495	struct kmem_cache *s;
5496	struct memory_notify *marg = arg;
5497	int nid = marg->status_change_nid_normal;
5498	int ret = 0;
5499
5500	/*
5501	* If the node's memory is already available, then kmem_cache_node is
5502	* already created. Nothing to do.
5503	*/
5504	if (nid < 0)
5505	return 0;
5506
5507	/*
5508	* We are bringing a node online. No memory is available yet. We must
5509	* allocate a kmem_cache_node structure in order to bring the node
5510	* online.
5511	*/
5512	mutex_lock(&slab_mutex);
5513	list_for_each_entry(s, &slab_caches, list) {
5514	/*
5515	* The structure may already exist if the node was previously
5516	* onlined and offlined.
5517	*/
5518	if (get_node(s, node: nid))
5519	continue;
5520	/*
5521	* XXX: kmem_cache_alloc_node will fallback to other nodes
5522	* since memory is not yet available from the node that
5523	* is brought up.
5524	*/
5525	n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
5526	if (!n) {
5527	ret = -ENOMEM;
5528	goto out;
5529	}
5530	init_kmem_cache_node(n);
5531	s->node[nid] = n;
5532	}
5533	/*
5534	* Any cache created after this point will also have kmem_cache_node
5535	* initialized for the new node.
5536	*/
5537	node_set(nid, slab_nodes);
5538	out:
5539	mutex_unlock(lock: &slab_mutex);
5540	return ret;
5541	}
5542
5543	static int slab_memory_callback(struct notifier_block *self,
5544	unsigned long action, void *arg)
5545	{
5546	int ret = 0;
5547
5548	switch (action) {
5549	case MEM_GOING_ONLINE:
5550	ret = slab_mem_going_online_callback(arg);
5551	break;
5552	case MEM_GOING_OFFLINE:
5553	ret = slab_mem_going_offline_callback(arg);
5554	break;
5555	case MEM_OFFLINE:
5556	case MEM_CANCEL_ONLINE:
5557	slab_mem_offline_callback(arg);
5558	break;
5559	case MEM_ONLINE:
5560	case MEM_CANCEL_OFFLINE:
5561	break;
5562	}
5563	if (ret)
5564	ret = notifier_from_errno(err: ret);
5565	else
5566	ret = NOTIFY_OK;
5567	return ret;
5568	}
5569
5570	/********************************************************************
5571	* Basic setup of slabs
5572	*******************************************************************/
5573
5574	/*
5575	* Used for early kmem_cache structures that were allocated using
5576	* the page allocator. Allocate them properly then fix up the pointers
5577	* that may be pointing to the wrong kmem_cache structure.
5578	*/
5579
5580	static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
5581	{
5582	int node;
5583	struct kmem_cache *s = kmem_cache_zalloc(k: kmem_cache, GFP_NOWAIT);
5584	struct kmem_cache_node *n;
5585
5586	memcpy(s, static_cache, kmem_cache->object_size);
5587
5588	/*
5589	* This runs very early, and only the boot processor is supposed to be
5590	* up. Even if it weren't true, IRQs are not up so we couldn't fire
5591	* IPIs around.
5592	*/
5593	__flush_cpu_slab(s, smp_processor_id());
5594	for_each_kmem_cache_node(s, node, n) {
5595	struct slab *p;
5596
5597	list_for_each_entry(p, &n->partial, slab_list)
5598	p->slab_cache = s;
5599
5600	#ifdef CONFIG_SLUB_DEBUG
5601	list_for_each_entry(p, &n->full, slab_list)
5602	p->slab_cache = s;
5603	#endif
5604	}
5605	list_add(new: &s->list, head: &slab_caches);
5606	return s;
5607	}
5608
5609	void __init kmem_cache_init(void)
5610	{
5611	static __initdata struct kmem_cache boot_kmem_cache,
5612	boot_kmem_cache_node;
5613	int node;
5614
5615	if (debug_guardpage_minorder())
5616	slub_max_order = 0;
5617
5618	/* Print slub debugging pointers without hashing */
5619	if (__slub_debug_enabled())
5620	no_hash_pointers_enable(NULL);
5621
5622	kmem_cache_node = &boot_kmem_cache_node;
5623	kmem_cache = &boot_kmem_cache;
5624
5625	/*
5626	* Initialize the nodemask for which we will allocate per node
5627	* structures. Here we don't need taking slab_mutex yet.
5628	*/
5629	for_each_node_state(node, N_NORMAL_MEMORY)
5630	node_set(node, slab_nodes);
5631
5632	create_boot_cache(kmem_cache_node, name: "kmem_cache_node",
5633	size: sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, useroffset: 0, usersize: 0);
5634
5635	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
5636
5637	/* Able to allocate the per node structures */
5638	slab_state = PARTIAL;
5639
5640	create_boot_cache(kmem_cache, name: "kmem_cache",
5641	offsetof(struct kmem_cache, node) +
5642	nr_node_ids * sizeof(struct kmem_cache_node *),
5643	SLAB_HWCACHE_ALIGN, useroffset: 0, usersize: 0);
5644
5645	kmem_cache = bootstrap(static_cache: &boot_kmem_cache);
5646	kmem_cache_node = bootstrap(static_cache: &boot_kmem_cache_node);
5647
5648	/* Now we can use the kmem_cache to allocate kmalloc slabs */
5649	setup_kmalloc_cache_index_table();
5650	create_kmalloc_caches();
5651
5652	/* Setup random freelists for each cache */
5653	init_freelist_randomization();
5654
5655	cpuhp_setup_state_nocalls(state: CPUHP_SLUB_DEAD, name: "slub:dead", NULL,
5656	teardown: slub_cpu_dead);
5657
5658	pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
5659	cache_line_size(),
5660	slub_min_order, slub_max_order, slub_min_objects,
5661	nr_cpu_ids, nr_node_ids);
5662	}
5663
5664	void __init kmem_cache_init_late(void)
5665	{
5666	#ifndef CONFIG_SLUB_TINY
5667	flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0);
5668	WARN_ON(!flushwq);
5669	#endif
5670	}
5671
5672	struct kmem_cache *
5673	__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
5674	slab_flags_t flags, void (*ctor)(void *))
5675	{
5676	struct kmem_cache *s;
5677
5678	s = find_mergeable(size, align, flags, name, ctor);
5679	if (s) {
5680	if (sysfs_slab_alias(s, p: name))
5681	return NULL;
5682
5683	s->refcount++;
5684
5685	/*
5686	* Adjust the object sizes so that we clear
5687	* the complete object on kzalloc.
5688	*/
5689	s->object_size = max(s->object_size, size);
5690	s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
5691	}
5692
5693	return s;
5694	}
5695
5696	int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
5697	{
5698	int err;
5699
5700	err = kmem_cache_open(s, flags);
5701	if (err)
5702	return err;
5703
5704	/* Mutex is not taken during early boot */
5705	if (slab_state <= UP)
5706	return 0;
5707
5708	err = sysfs_slab_add(s);
5709	if (err) {
5710	__kmem_cache_release(s);
5711	return err;
5712	}
5713
5714	if (s->flags & SLAB_STORE_USER)
5715	debugfs_slab_add(s);
5716
5717	return 0;
5718	}
5719
5720	#ifdef SLAB_SUPPORTS_SYSFS
5721	static int count_inuse(struct slab *slab)
5722	{
5723	return slab->inuse;
5724	}
5725
5726	static int count_total(struct slab *slab)
5727	{
5728	return slab->objects;
5729	}
5730	#endif
5731
5732	#ifdef CONFIG_SLUB_DEBUG
5733	static void validate_slab(struct kmem_cache *s, struct slab *slab,
5734	unsigned long *obj_map)
5735	{
5736	void *p;
5737	void *addr = slab_address(slab);
5738
5739	if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
5740	return;
5741
5742	/* Now we know that a valid freelist exists */
5743	__fill_map(obj_map, s, slab);
5744	for_each_object(p, s, addr, slab->objects) {
5745	u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
5746	SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
5747
5748	if (!check_object(s, slab, p, val))
5749	break;
5750	}
5751	}
5752
5753	static int validate_slab_node(struct kmem_cache *s,
5754	struct kmem_cache_node *n, unsigned long *obj_map)
5755	{
5756	unsigned long count = 0;
5757	struct slab *slab;
5758	unsigned long flags;
5759
5760	spin_lock_irqsave(&n->list_lock, flags);
5761
5762	list_for_each_entry(slab, &n->partial, slab_list) {
5763	validate_slab(s, slab, obj_map);
5764	count++;
5765	}
5766	if (count != n->nr_partial) {
5767	pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
5768	s->name, count, n->nr_partial);
5769	slab_add_kunit_errors();
5770	}
5771
5772	if (!(s->flags & SLAB_STORE_USER))
5773	goto out;
5774
5775	list_for_each_entry(slab, &n->full, slab_list) {
5776	validate_slab(s, slab, obj_map);
5777	count++;
5778	}
5779	if (count != node_nr_slabs(n)) {
5780	pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
5781	s->name, count, node_nr_slabs(n));
5782	slab_add_kunit_errors();
5783	}
5784
5785	out:
5786	spin_unlock_irqrestore(&n->list_lock, flags);
5787	return count;
5788	}
5789
5790	long validate_slab_cache(struct kmem_cache *s)
5791	{
5792	int node;
5793	unsigned long count = 0;
5794	struct kmem_cache_node *n;
5795	unsigned long *obj_map;
5796
5797	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
5798	if (!obj_map)
5799	return -ENOMEM;
5800
5801	flush_all(s);
5802	for_each_kmem_cache_node(s, node, n)
5803	count += validate_slab_node(s, n, obj_map);
5804
5805	bitmap_free(obj_map);
5806
5807	return count;
5808	}
5809	EXPORT_SYMBOL(validate_slab_cache);
5810
5811	#ifdef CONFIG_DEBUG_FS
5812	/*
5813	* Generate lists of code addresses where slabcache objects are allocated
5814	* and freed.
5815	*/
5816
5817	struct location {
5818	depot_stack_handle_t handle;
5819	unsigned long count;
5820	unsigned long addr;
5821	unsigned long waste;
5822	long long sum_time;
5823	long min_time;
5824	long max_time;
5825	long min_pid;
5826	long max_pid;
5827	DECLARE_BITMAP(cpus, NR_CPUS);
5828	nodemask_t nodes;
5829	};
5830
5831	struct loc_track {
5832	unsigned long max;
5833	unsigned long count;
5834	struct location *loc;
5835	loff_t idx;
5836	};
5837
5838	static struct dentry *slab_debugfs_root;
5839
5840	static void free_loc_track(struct loc_track *t)
5841	{
5842	if (t->max)
5843	free_pages((unsigned long)t->loc,
5844	get_order(sizeof(struct location) * t->max));
5845	}
5846
5847	static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
5848	{
5849	struct location *l;
5850	int order;
5851
5852	order = get_order(sizeof(struct location) * max);
5853
5854	l = (void *)__get_free_pages(flags, order);
5855	if (!l)
5856	return 0;
5857
5858	if (t->count) {
5859	memcpy(l, t->loc, sizeof(struct location) * t->count);
5860	free_loc_track(t);
5861	}
5862	t->max = max;
5863	t->loc = l;
5864	return 1;
5865	}
5866
5867	static int add_location(struct loc_track *t, struct kmem_cache *s,
5868	const struct track *track,
5869	unsigned int orig_size)
5870	{
5871	long start, end, pos;
5872	struct location *l;
5873	unsigned long caddr, chandle, cwaste;
5874	unsigned long age = jiffies - track->when;
5875	depot_stack_handle_t handle = 0;
5876	unsigned int waste = s->object_size - orig_size;
5877
5878	#ifdef CONFIG_STACKDEPOT
5879	handle = READ_ONCE(track->handle);
5880	#endif
5881	start = -1;
5882	end = t->count;
5883
5884	for ( ; ; ) {
5885	pos = start + (end - start + 1) / 2;
5886
5887	/*
5888	* There is nothing at "end". If we end up there
5889	* we need to add something to before end.
5890	*/
5891	if (pos == end)
5892	break;
5893
5894	l = &t->loc[pos];
5895	caddr = l->addr;
5896	chandle = l->handle;
5897	cwaste = l->waste;
5898	if ((track->addr == caddr) && (handle == chandle) &&
5899	(waste == cwaste)) {
5900
5901	l->count++;
5902	if (track->when) {
5903	l->sum_time += age;
5904	if (age < l->min_time)
5905	l->min_time = age;
5906	if (age > l->max_time)
5907	l->max_time = age;
5908
5909	if (track->pid < l->min_pid)
5910	l->min_pid = track->pid;
5911	if (track->pid > l->max_pid)
5912	l->max_pid = track->pid;
5913
5914	cpumask_set_cpu(track->cpu,
5915	to_cpumask(l->cpus));
5916	}
5917	node_set(page_to_nid(virt_to_page(track)), l->nodes);
5918	return 1;
5919	}
5920
5921	if (track->addr < caddr)
5922	end = pos;
5923	else if (track->addr == caddr && handle < chandle)
5924	end = pos;
5925	else if (track->addr == caddr && handle == chandle &&
5926	waste < cwaste)
5927	end = pos;
5928	else
5929	start = pos;
5930	}
5931
5932	/*
5933	* Not found. Insert new tracking element.
5934	*/
5935	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
5936	return 0;
5937
5938	l = t->loc + pos;
5939	if (pos < t->count)
5940	memmove(l + 1, l,
5941	(t->count - pos) * sizeof(struct location));
5942	t->count++;
5943	l->count = 1;
5944	l->addr = track->addr;
5945	l->sum_time = age;
5946	l->min_time = age;
5947	l->max_time = age;
5948	l->min_pid = track->pid;
5949	l->max_pid = track->pid;
5950	l->handle = handle;
5951	l->waste = waste;
5952	cpumask_clear(to_cpumask(l->cpus));
5953	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
5954	nodes_clear(l->nodes);
5955	node_set(page_to_nid(virt_to_page(track)), l->nodes);
5956	return 1;
5957	}
5958
5959	static void process_slab(struct loc_track *t, struct kmem_cache *s,
5960	struct slab *slab, enum track_item alloc,
5961	unsigned long *obj_map)
5962	{
5963	void *addr = slab_address(slab);
5964	bool is_alloc = (alloc == TRACK_ALLOC);
5965	void *p;
5966
5967	__fill_map(obj_map, s, slab);
5968
5969	for_each_object(p, s, addr, slab->objects)
5970	if (!test_bit(__obj_to_index(s, addr, p), obj_map))
5971	add_location(t, s, get_track(s, p, alloc),
5972	is_alloc ? get_orig_size(s, p) :
5973	s->object_size);
5974	}
5975	#endif /* CONFIG_DEBUG_FS */
5976	#endif /* CONFIG_SLUB_DEBUG */
5977
5978	#ifdef SLAB_SUPPORTS_SYSFS
5979	enum slab_stat_type {
5980	SL_ALL, /* All slabs */
5981	SL_PARTIAL, /* Only partially allocated slabs */
5982	SL_CPU, /* Only slabs used for cpu caches */
5983	SL_OBJECTS, /* Determine allocated objects not slabs */
5984	SL_TOTAL /* Determine object capacity not slabs */
5985	};
5986
5987	#define SO_ALL (1 << SL_ALL)
5988	#define SO_PARTIAL (1 << SL_PARTIAL)
5989	#define SO_CPU (1 << SL_CPU)
5990	#define SO_OBJECTS (1 << SL_OBJECTS)
5991	#define SO_TOTAL (1 << SL_TOTAL)
5992
5993	static ssize_t show_slab_objects(struct kmem_cache *s,
5994	char *buf, unsigned long flags)
5995	{
5996	unsigned long total = 0;
5997	int node;
5998	int x;
5999	unsigned long *nodes;
6000	int len = 0;
6001
6002	nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
6003	if (!nodes)
6004	return -ENOMEM;
6005
6006	if (flags & SO_CPU) {
6007	int cpu;
6008
6009	for_each_possible_cpu(cpu) {
6010	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
6011	cpu);
6012	int node;
6013	struct slab *slab;
6014
6015	slab = READ_ONCE(c->slab);
6016	if (!slab)
6017	continue;
6018
6019	node = slab_nid(slab);
6020	if (flags & SO_TOTAL)
6021	x = slab->objects;
6022	else if (flags & SO_OBJECTS)
6023	x = slab->inuse;
6024	else
6025	x = 1;
6026
6027	total += x;
6028	nodes[node] += x;
6029
6030	#ifdef CONFIG_SLUB_CPU_PARTIAL
6031	slab = slub_percpu_partial_read_once(c);
6032	if (slab) {
6033	node = slab_nid(slab);
6034	if (flags & SO_TOTAL)
6035	WARN_ON_ONCE(1);
6036	else if (flags & SO_OBJECTS)
6037	WARN_ON_ONCE(1);
6038	else
6039	x = slab->slabs;
6040	total += x;
6041	nodes[node] += x;
6042	}
6043	#endif
6044	}
6045	}
6046
6047	/*
6048	* It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
6049	* already held which will conflict with an existing lock order:
6050	*
6051	* mem_hotplug_lock->slab_mutex->kernfs_mutex
6052	*
6053	* We don't really need mem_hotplug_lock (to hold off
6054	* slab_mem_going_offline_callback) here because slab's memory hot
6055	* unplug code doesn't destroy the kmem_cache->node[] data.
6056	*/
6057
6058	#ifdef CONFIG_SLUB_DEBUG
6059	if (flags & SO_ALL) {
6060	struct kmem_cache_node *n;
6061
6062	for_each_kmem_cache_node(s, node, n) {
6063
6064	if (flags & SO_TOTAL)
6065	x = node_nr_objs(n);
6066	else if (flags & SO_OBJECTS)
6067	x = node_nr_objs(n) - count_partial(n, count_free);
6068	else
6069	x = node_nr_slabs(n);
6070	total += x;
6071	nodes[node] += x;
6072	}
6073
6074	} else
6075	#endif
6076	if (flags & SO_PARTIAL) {
6077	struct kmem_cache_node *n;
6078
6079	for_each_kmem_cache_node(s, node, n) {
6080	if (flags & SO_TOTAL)
6081	x = count_partial(n, count_total);
6082	else if (flags & SO_OBJECTS)
6083	x = count_partial(n, count_inuse);
6084	else
6085	x = n->nr_partial;
6086	total += x;
6087	nodes[node] += x;
6088	}
6089	}
6090
6091	len += sysfs_emit_at(buf, len, "%lu", total);
6092	#ifdef CONFIG_NUMA
6093	for (node = 0; node < nr_node_ids; node++) {
6094	if (nodes[node])
6095	len += sysfs_emit_at(buf, len, " N%d=%lu",
6096	node, nodes[node]);
6097	}
6098	#endif
6099	len += sysfs_emit_at(buf, len, "\n");
6100	kfree(nodes);
6101
6102	return len;
6103	}
6104
6105	#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
6106	#define to_slab(n) container_of(n, struct kmem_cache, kobj)
6107
6108	struct slab_attribute {
6109	struct attribute attr;
6110	ssize_t (*show)(struct kmem_cache *s, char *buf);
6111	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
6112	};
6113
6114	#define SLAB_ATTR_RO(_name) \
6115	static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400)
6116
6117	#define SLAB_ATTR(_name) \
6118	static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600)
6119
6120	static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
6121	{
6122	return sysfs_emit(buf, "%u\n", s->size);
6123	}
6124	SLAB_ATTR_RO(slab_size);
6125
6126	static ssize_t align_show(struct kmem_cache *s, char *buf)
6127	{
6128	return sysfs_emit(buf, "%u\n", s->align);
6129	}
6130	SLAB_ATTR_RO(align);
6131
6132	static ssize_t object_size_show(struct kmem_cache *s, char *buf)
6133	{
6134	return sysfs_emit(buf, "%u\n", s->object_size);
6135	}
6136	SLAB_ATTR_RO(object_size);
6137
6138	static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
6139	{
6140	return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
6141	}
6142	SLAB_ATTR_RO(objs_per_slab);
6143
6144	static ssize_t order_show(struct kmem_cache *s, char *buf)
6145	{
6146	return sysfs_emit(buf, "%u\n", oo_order(s->oo));
6147	}
6148	SLAB_ATTR_RO(order);
6149
6150	static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
6151	{
6152	return sysfs_emit(buf, "%lu\n", s->min_partial);
6153	}
6154
6155	static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
6156	size_t length)
6157	{
6158	unsigned long min;
6159	int err;
6160
6161	err = kstrtoul(buf, 10, &min);
6162	if (err)
6163	return err;
6164
6165	s->min_partial = min;
6166	return length;
6167	}
6168	SLAB_ATTR(min_partial);
6169
6170	static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
6171	{
6172	unsigned int nr_partial = 0;
6173	#ifdef CONFIG_SLUB_CPU_PARTIAL
6174	nr_partial = s->cpu_partial;
6175	#endif
6176
6177	return sysfs_emit(buf, "%u\n", nr_partial);
6178	}
6179
6180	static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
6181	size_t length)
6182	{
6183	unsigned int objects;
6184	int err;
6185
6186	err = kstrtouint(buf, 10, &objects);
6187	if (err)
6188	return err;
6189	if (objects && !kmem_cache_has_cpu_partial(s))
6190	return -EINVAL;
6191
6192	slub_set_cpu_partial(s, objects);
6193	flush_all(s);
6194	return length;
6195	}
6196	SLAB_ATTR(cpu_partial);
6197
6198	static ssize_t ctor_show(struct kmem_cache *s, char *buf)
6199	{
6200	if (!s->ctor)
6201	return 0;
6202	return sysfs_emit(buf, "%pS\n", s->ctor);
6203	}
6204	SLAB_ATTR_RO(ctor);
6205
6206	static ssize_t aliases_show(struct kmem_cache *s, char *buf)
6207	{
6208	return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
6209	}
6210	SLAB_ATTR_RO(aliases);
6211
6212	static ssize_t partial_show(struct kmem_cache *s, char *buf)
6213	{
6214	return show_slab_objects(s, buf, SO_PARTIAL);
6215	}
6216	SLAB_ATTR_RO(partial);
6217
6218	static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
6219	{
6220	return show_slab_objects(s, buf, SO_CPU);
6221	}
6222	SLAB_ATTR_RO(cpu_slabs);
6223
6224	static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
6225	{
6226	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
6227	}
6228	SLAB_ATTR_RO(objects_partial);
6229
6230	static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
6231	{
6232	int objects = 0;
6233	int slabs = 0;
6234	int cpu __maybe_unused;
6235	int len = 0;
6236
6237	#ifdef CONFIG_SLUB_CPU_PARTIAL
6238	for_each_online_cpu(cpu) {
6239	struct slab *slab;
6240
6241	slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
6242
6243	if (slab)
6244	slabs += slab->slabs;
6245	}
6246	#endif
6247
6248	/* Approximate half-full slabs, see slub_set_cpu_partial() */
6249	objects = (slabs * oo_objects(s->oo)) / 2;
6250	len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
6251
6252	#ifdef CONFIG_SLUB_CPU_PARTIAL
6253	for_each_online_cpu(cpu) {
6254	struct slab *slab;
6255
6256	slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
6257	if (slab) {
6258	slabs = READ_ONCE(slab->slabs);
6259	objects = (slabs * oo_objects(s->oo)) / 2;
6260	len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
6261	cpu, objects, slabs);
6262	}
6263	}
6264	#endif
6265	len += sysfs_emit_at(buf, len, "\n");
6266
6267	return len;
6268	}
6269	SLAB_ATTR_RO(slabs_cpu_partial);
6270
6271	static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
6272	{
6273	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
6274	}
6275	SLAB_ATTR_RO(reclaim_account);
6276
6277	static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
6278	{
6279	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
6280	}
6281	SLAB_ATTR_RO(hwcache_align);
6282
6283	#ifdef CONFIG_ZONE_DMA
6284	static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
6285	{
6286	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
6287	}
6288	SLAB_ATTR_RO(cache_dma);
6289	#endif
6290
6291	#ifdef CONFIG_HARDENED_USERCOPY
6292	static ssize_t usersize_show(struct kmem_cache *s, char *buf)
6293	{
6294	return sysfs_emit(buf, "%u\n", s->usersize);
6295	}
6296	SLAB_ATTR_RO(usersize);
6297	#endif
6298
6299	static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
6300	{
6301	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
6302	}
6303	SLAB_ATTR_RO(destroy_by_rcu);
6304
6305	#ifdef CONFIG_SLUB_DEBUG
6306	static ssize_t slabs_show(struct kmem_cache *s, char *buf)
6307	{
6308	return show_slab_objects(s, buf, SO_ALL);
6309	}
6310	SLAB_ATTR_RO(slabs);
6311
6312	static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
6313	{
6314	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
6315	}
6316	SLAB_ATTR_RO(total_objects);
6317
6318	static ssize_t objects_show(struct kmem_cache *s, char *buf)
6319	{
6320	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
6321	}
6322	SLAB_ATTR_RO(objects);
6323
6324	static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
6325	{
6326	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
6327	}
6328	SLAB_ATTR_RO(sanity_checks);
6329
6330	static ssize_t trace_show(struct kmem_cache *s, char *buf)
6331	{
6332	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
6333	}
6334	SLAB_ATTR_RO(trace);
6335
6336	static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
6337	{
6338	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
6339	}
6340
6341	SLAB_ATTR_RO(red_zone);
6342
6343	static ssize_t poison_show(struct kmem_cache *s, char *buf)
6344	{
6345	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
6346	}
6347
6348	SLAB_ATTR_RO(poison);
6349
6350	static ssize_t store_user_show(struct kmem_cache *s, char *buf)
6351	{
6352	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
6353	}
6354
6355	SLAB_ATTR_RO(store_user);
6356
6357	static ssize_t validate_show(struct kmem_cache *s, char *buf)
6358	{
6359	return 0;
6360	}
6361
6362	static ssize_t validate_store(struct kmem_cache *s,
6363	const char *buf, size_t length)
6364	{
6365	int ret = -EINVAL;
6366
6367	if (buf[0] == '1' && kmem_cache_debug(s)) {
6368	ret = validate_slab_cache(s);
6369	if (ret >= 0)
6370	ret = length;
6371	}
6372	return ret;
6373	}
6374	SLAB_ATTR(validate);
6375
6376	#endif /* CONFIG_SLUB_DEBUG */
6377
6378	#ifdef CONFIG_FAILSLAB
6379	static ssize_t failslab_show(struct kmem_cache *s, char *buf)
6380	{
6381	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
6382	}
6383
6384	static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
6385	size_t length)
6386	{
6387	if (s->refcount > 1)
6388	return -EINVAL;
6389
6390	if (buf[0] == '1')
6391	WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB);
6392	else
6393	WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB);
6394
6395	return length;
6396	}
6397	SLAB_ATTR(failslab);
6398	#endif
6399
6400	static ssize_t shrink_show(struct kmem_cache *s, char *buf)
6401	{
6402	return 0;
6403	}
6404
6405	static ssize_t shrink_store(struct kmem_cache *s,
6406	const char *buf, size_t length)
6407	{
6408	if (buf[0] == '1')
6409	kmem_cache_shrink(s);
6410	else
6411	return -EINVAL;
6412	return length;
6413	}
6414	SLAB_ATTR(shrink);
6415
6416	#ifdef CONFIG_NUMA
6417	static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
6418	{
6419	return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
6420	}
6421
6422	static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
6423	const char *buf, size_t length)
6424	{
6425	unsigned int ratio;
6426	int err;
6427
6428	err = kstrtouint(buf, 10, &ratio);
6429	if (err)
6430	return err;
6431	if (ratio > 100)
6432	return -ERANGE;
6433
6434	s->remote_node_defrag_ratio = ratio * 10;
6435
6436	return length;
6437	}
6438	SLAB_ATTR(remote_node_defrag_ratio);
6439	#endif
6440
6441	#ifdef CONFIG_SLUB_STATS
6442	static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
6443	{
6444	unsigned long sum = 0;
6445	int cpu;
6446	int len = 0;
6447	int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
6448
6449	if (!data)
6450	return -ENOMEM;
6451
6452	for_each_online_cpu(cpu) {
6453	unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
6454
6455	data[cpu] = x;
6456	sum += x;
6457	}
6458
6459	len += sysfs_emit_at(buf, len, "%lu", sum);
6460
6461	#ifdef CONFIG_SMP
6462	for_each_online_cpu(cpu) {
6463	if (data[cpu])
6464	len += sysfs_emit_at(buf, len, " C%d=%u",
6465	cpu, data[cpu]);
6466	}
6467	#endif
6468	kfree(data);
6469	len += sysfs_emit_at(buf, len, "\n");
6470
6471	return len;
6472	}
6473
6474	static void clear_stat(struct kmem_cache *s, enum stat_item si)
6475	{
6476	int cpu;
6477
6478	for_each_online_cpu(cpu)
6479	per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
6480	}
6481
6482	#define STAT_ATTR(si, text) \
6483	static ssize_t text##_show(struct kmem_cache *s, char *buf) \
6484	{ \
6485	return show_stat(s, buf, si); \
6486	} \
6487	static ssize_t text##_store(struct kmem_cache *s, \
6488	const char *buf, size_t length) \
6489	{ \
6490	if (buf[0] != '0') \
6491	return -EINVAL; \
6492	clear_stat(s, si); \
6493	return length; \
6494	} \
6495	SLAB_ATTR(text); \
6496
6497	STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
6498	STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
6499	STAT_ATTR(FREE_FASTPATH, free_fastpath);
6500	STAT_ATTR(FREE_SLOWPATH, free_slowpath);
6501	STAT_ATTR(FREE_FROZEN, free_frozen);
6502	STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
6503	STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
6504	STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
6505	STAT_ATTR(ALLOC_SLAB, alloc_slab);
6506	STAT_ATTR(ALLOC_REFILL, alloc_refill);
6507	STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
6508	STAT_ATTR(FREE_SLAB, free_slab);
6509	STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
6510	STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
6511	STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
6512	STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
6513	STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
6514	STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
6515	STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
6516	STAT_ATTR(ORDER_FALLBACK, order_fallback);
6517	STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
6518	STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
6519	STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
6520	STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
6521	STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
6522	STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
6523	#endif /* CONFIG_SLUB_STATS */
6524
6525	#ifdef CONFIG_KFENCE
6526	static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf)
6527	{
6528	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE));
6529	}
6530
6531	static ssize_t skip_kfence_store(struct kmem_cache *s,
6532	const char *buf, size_t length)
6533	{
6534	int ret = length;
6535
6536	if (buf[0] == '0')
6537	s->flags &= ~SLAB_SKIP_KFENCE;
6538	else if (buf[0] == '1')
6539	s->flags |= SLAB_SKIP_KFENCE;
6540	else
6541	ret = -EINVAL;
6542
6543	return ret;
6544	}
6545	SLAB_ATTR(skip_kfence);
6546	#endif
6547
6548	static struct attribute *slab_attrs[] = {
6549	&slab_size_attr.attr,
6550	&object_size_attr.attr,
6551	&objs_per_slab_attr.attr,
6552	&order_attr.attr,
6553	&min_partial_attr.attr,
6554	&cpu_partial_attr.attr,
6555	&objects_partial_attr.attr,
6556	&partial_attr.attr,
6557	&cpu_slabs_attr.attr,
6558	&ctor_attr.attr,
6559	&aliases_attr.attr,
6560	&align_attr.attr,
6561	&hwcache_align_attr.attr,
6562	&reclaim_account_attr.attr,
6563	&destroy_by_rcu_attr.attr,
6564	&shrink_attr.attr,
6565	&slabs_cpu_partial_attr.attr,
6566	#ifdef CONFIG_SLUB_DEBUG
6567	&total_objects_attr.attr,
6568	&objects_attr.attr,
6569	&slabs_attr.attr,
6570	&sanity_checks_attr.attr,
6571	&trace_attr.attr,
6572	&red_zone_attr.attr,
6573	&poison_attr.attr,
6574	&store_user_attr.attr,
6575	&validate_attr.attr,
6576	#endif
6577	#ifdef CONFIG_ZONE_DMA
6578	&cache_dma_attr.attr,
6579	#endif
6580	#ifdef CONFIG_NUMA
6581	&remote_node_defrag_ratio_attr.attr,
6582	#endif
6583	#ifdef CONFIG_SLUB_STATS
6584	&alloc_fastpath_attr.attr,
6585	&alloc_slowpath_attr.attr,
6586	&free_fastpath_attr.attr,
6587	&free_slowpath_attr.attr,
6588	&free_frozen_attr.attr,
6589	&free_add_partial_attr.attr,
6590	&free_remove_partial_attr.attr,
6591	&alloc_from_partial_attr.attr,
6592	&alloc_slab_attr.attr,
6593	&alloc_refill_attr.attr,
6594	&alloc_node_mismatch_attr.attr,
6595	&free_slab_attr.attr,
6596	&cpuslab_flush_attr.attr,
6597	&deactivate_full_attr.attr,
6598	&deactivate_empty_attr.attr,
6599	&deactivate_to_head_attr.attr,
6600	&deactivate_to_tail_attr.attr,
6601	&deactivate_remote_frees_attr.attr,
6602	&deactivate_bypass_attr.attr,
6603	&order_fallback_attr.attr,
6604	&cmpxchg_double_fail_attr.attr,
6605	&cmpxchg_double_cpu_fail_attr.attr,
6606	&cpu_partial_alloc_attr.attr,
6607	&cpu_partial_free_attr.attr,
6608	&cpu_partial_node_attr.attr,
6609	&cpu_partial_drain_attr.attr,
6610	#endif
6611	#ifdef CONFIG_FAILSLAB
6612	&failslab_attr.attr,
6613	#endif
6614	#ifdef CONFIG_HARDENED_USERCOPY
6615	&usersize_attr.attr,
6616	#endif
6617	#ifdef CONFIG_KFENCE
6618	&skip_kfence_attr.attr,
6619	#endif
6620
6621	NULL
6622	};
6623
6624	static const struct attribute_group slab_attr_group = {
6625	.attrs = slab_attrs,
6626	};
6627
6628	static ssize_t slab_attr_show(struct kobject *kobj,
6629	struct attribute *attr,
6630	char *buf)
6631	{
6632	struct slab_attribute *attribute;
6633	struct kmem_cache *s;
6634
6635	attribute = to_slab_attr(attr);
6636	s = to_slab(kobj);
6637
6638	if (!attribute->show)
6639	return -EIO;
6640
6641	return attribute->show(s, buf);
6642	}
6643
6644	static ssize_t slab_attr_store(struct kobject *kobj,
6645	struct attribute *attr,
6646	const char *buf, size_t len)
6647	{
6648	struct slab_attribute *attribute;
6649	struct kmem_cache *s;
6650
6651	attribute = to_slab_attr(attr);
6652	s = to_slab(kobj);
6653
6654	if (!attribute->store)
6655	return -EIO;
6656
6657	return attribute->store(s, buf, len);
6658	}
6659
6660	static void kmem_cache_release(struct kobject *k)
6661	{
6662	slab_kmem_cache_release(to_slab(k));
6663	}
6664
6665	static const struct sysfs_ops slab_sysfs_ops = {
6666	.show = slab_attr_show,
6667	.store = slab_attr_store,
6668	};
6669
6670	static const struct kobj_type slab_ktype = {
6671	.sysfs_ops = &slab_sysfs_ops,
6672	.release = kmem_cache_release,
6673	};
6674
6675	static struct kset *slab_kset;
6676
6677	static inline struct kset *cache_kset(struct kmem_cache *s)
6678	{
6679	return slab_kset;
6680	}
6681
6682	#define ID_STR_LENGTH 32
6683
6684	/* Create a unique string id for a slab cache:
6685	*
6686	* Format :[flags-]size
6687	*/
6688	static char *create_unique_id(struct kmem_cache *s)
6689	{
6690	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
6691	char *p = name;
6692
6693	if (!name)
6694	return ERR_PTR(-ENOMEM);
6695
6696	*p++ = ':';
6697	/*
6698	* First flags affecting slabcache operations. We will only
6699	* get here for aliasable slabs so we do not need to support
6700	* too many flags. The flags here must cover all flags that
6701	* are matched during merging to guarantee that the id is
6702	* unique.
6703	*/
6704	if (s->flags & SLAB_CACHE_DMA)
6705	*p++ = 'd';
6706	if (s->flags & SLAB_CACHE_DMA32)
6707	*p++ = 'D';
6708	if (s->flags & SLAB_RECLAIM_ACCOUNT)
6709	*p++ = 'a';
6710	if (s->flags & SLAB_CONSISTENCY_CHECKS)
6711	*p++ = 'F';
6712	if (s->flags & SLAB_ACCOUNT)
6713	*p++ = 'A';
6714	if (p != name + 1)
6715	*p++ = '-';
6716	p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size);
6717
6718	if (WARN_ON(p > name + ID_STR_LENGTH - 1)) {
6719	kfree(name);
6720	return ERR_PTR(-EINVAL);
6721	}
6722	kmsan_unpoison_memory(name, p - name);
6723	return name;
6724	}
6725
6726	static int sysfs_slab_add(struct kmem_cache *s)
6727	{
6728	int err;
6729	const char *name;
6730	struct kset *kset = cache_kset(s);
6731	int unmergeable = slab_unmergeable(s);
6732
6733	if (!unmergeable && disable_higher_order_debug &&
6734	(slub_debug & DEBUG_METADATA_FLAGS))
6735	unmergeable = 1;
6736
6737	if (unmergeable) {
6738	/*
6739	* Slabcache can never be merged so we can use the name proper.
6740	* This is typically the case for debug situations. In that
6741	* case we can catch duplicate names easily.
6742	*/
6743	sysfs_remove_link(&slab_kset->kobj, s->name);
6744	name = s->name;
6745	} else {
6746	/*
6747	* Create a unique name for the slab as a target
6748	* for the symlinks.
6749	*/
6750	name = create_unique_id(s);
6751	if (IS_ERR(name))
6752	return PTR_ERR(name);
6753	}
6754
6755	s->kobj.kset = kset;
6756	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
6757	if (err)
6758	goto out;
6759
6760	err = sysfs_create_group(&s->kobj, &slab_attr_group);
6761	if (err)
6762	goto out_del_kobj;
6763
6764	if (!unmergeable) {
6765	/* Setup first alias */
6766	sysfs_slab_alias(s, s->name);
6767	}
6768	out:
6769	if (!unmergeable)
6770	kfree(name);
6771	return err;
6772	out_del_kobj:
6773	kobject_del(&s->kobj);
6774	goto out;
6775	}
6776
6777	void sysfs_slab_unlink(struct kmem_cache *s)
6778	{
6779	kobject_del(&s->kobj);
6780	}
6781
6782	void sysfs_slab_release(struct kmem_cache *s)
6783	{
6784	kobject_put(&s->kobj);
6785	}
6786
6787	/*
6788	* Need to buffer aliases during bootup until sysfs becomes
6789	* available lest we lose that information.
6790	*/
6791	struct saved_alias {
6792	struct kmem_cache *s;
6793	const char *name;
6794	struct saved_alias *next;
6795	};
6796
6797	static struct saved_alias *alias_list;
6798
6799	static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
6800	{
6801	struct saved_alias *al;
6802
6803	if (slab_state == FULL) {
6804	/*
6805	* If we have a leftover link then remove it.
6806	*/
6807	sysfs_remove_link(&slab_kset->kobj, name);
6808	return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
6809	}
6810
6811	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
6812	if (!al)
6813	return -ENOMEM;
6814
6815	al->s = s;
6816	al->name = name;
6817	al->next = alias_list;
6818	alias_list = al;
6819	kmsan_unpoison_memory(al, sizeof(*al));
6820	return 0;
6821	}
6822
6823	static int __init slab_sysfs_init(void)
6824	{
6825	struct kmem_cache *s;
6826	int err;
6827
6828	mutex_lock(&slab_mutex);
6829
6830	slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
6831	if (!slab_kset) {
6832	mutex_unlock(&slab_mutex);
6833	pr_err("Cannot register slab subsystem.\n");
6834	return -ENOMEM;
6835	}
6836
6837	slab_state = FULL;
6838
6839	list_for_each_entry(s, &slab_caches, list) {
6840	err = sysfs_slab_add(s);
6841	if (err)
6842	pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
6843	s->name);
6844	}
6845
6846	while (alias_list) {
6847	struct saved_alias *al = alias_list;
6848
6849	alias_list = alias_list->next;
6850	err = sysfs_slab_alias(al->s, al->name);
6851	if (err)
6852	pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
6853	al->name);
6854	kfree(al);
6855	}
6856
6857	mutex_unlock(&slab_mutex);
6858	return 0;
6859	}
6860	late_initcall(slab_sysfs_init);
6861	#endif /* SLAB_SUPPORTS_SYSFS */
6862
6863	#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
6864	static int slab_debugfs_show(struct seq_file *seq, void *v)
6865	{
6866	struct loc_track *t = seq->private;
6867	struct location *l;
6868	unsigned long idx;
6869
6870	idx = (unsigned long) t->idx;
6871	if (idx < t->count) {
6872	l = &t->loc[idx];
6873
6874	seq_printf(seq, "%7ld ", l->count);
6875
6876	if (l->addr)
6877	seq_printf(seq, "%pS", (void *)l->addr);
6878	else
6879	seq_puts(seq, "<not-available>");
6880
6881	if (l->waste)
6882	seq_printf(seq, " waste=%lu/%lu",
6883	l->count * l->waste, l->waste);
6884
6885	if (l->sum_time != l->min_time) {
6886	seq_printf(seq, " age=%ld/%llu/%ld",
6887	l->min_time, div_u64(l->sum_time, l->count),
6888	l->max_time);
6889	} else
6890	seq_printf(seq, " age=%ld", l->min_time);
6891
6892	if (l->min_pid != l->max_pid)
6893	seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
6894	else
6895	seq_printf(seq, " pid=%ld",
6896	l->min_pid);
6897
6898	if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
6899	seq_printf(seq, " cpus=%*pbl",
6900	cpumask_pr_args(to_cpumask(l->cpus)));
6901
6902	if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
6903	seq_printf(seq, " nodes=%*pbl",
6904	nodemask_pr_args(&l->nodes));
6905
6906	#ifdef CONFIG_STACKDEPOT
6907	{
6908	depot_stack_handle_t handle;
6909	unsigned long *entries;
6910	unsigned int nr_entries, j;
6911
6912	handle = READ_ONCE(l->handle);
6913	if (handle) {
6914	nr_entries = stack_depot_fetch(handle, &entries);
6915	seq_puts(seq, "\n");
6916	for (j = 0; j < nr_entries; j++)
6917	seq_printf(seq, " %pS\n", (void *)entries[j]);
6918	}
6919	}
6920	#endif
6921	seq_puts(seq, "\n");
6922	}
6923
6924	if (!idx && !t->count)
6925	seq_puts(seq, "No data\n");
6926
6927	return 0;
6928	}
6929
6930	static void slab_debugfs_stop(struct seq_file *seq, void *v)
6931	{
6932	}
6933
6934	static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
6935	{
6936	struct loc_track *t = seq->private;
6937
6938	t->idx = ++(*ppos);
6939	if (*ppos <= t->count)
6940	return ppos;
6941
6942	return NULL;
6943	}
6944
6945	static int cmp_loc_by_count(const void *a, const void *b, const void *data)
6946	{
6947	struct location *loc1 = (struct location *)a;
6948	struct location *loc2 = (struct location *)b;
6949
6950	if (loc1->count > loc2->count)
6951	return -1;
6952	else
6953	return 1;
6954	}
6955
6956	static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
6957	{
6958	struct loc_track *t = seq->private;
6959
6960	t->idx = *ppos;
6961	return ppos;
6962	}
6963
6964	static const struct seq_operations slab_debugfs_sops = {
6965	.start = slab_debugfs_start,
6966	.next = slab_debugfs_next,
6967	.stop = slab_debugfs_stop,
6968	.show = slab_debugfs_show,
6969	};
6970
6971	static int slab_debug_trace_open(struct inode *inode, struct file *filep)
6972	{
6973
6974	struct kmem_cache_node *n;
6975	enum track_item alloc;
6976	int node;
6977	struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
6978	sizeof(struct loc_track));
6979	struct kmem_cache *s = file_inode(filep)->i_private;
6980	unsigned long *obj_map;
6981
6982	if (!t)
6983	return -ENOMEM;
6984
6985	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
6986	if (!obj_map) {
6987	seq_release_private(inode, filep);
6988	return -ENOMEM;
6989	}
6990
6991	if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
6992	alloc = TRACK_ALLOC;
6993	else
6994	alloc = TRACK_FREE;
6995
6996	if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
6997	bitmap_free(obj_map);
6998	seq_release_private(inode, filep);
6999	return -ENOMEM;
7000	}
7001
7002	for_each_kmem_cache_node(s, node, n) {
7003	unsigned long flags;
7004	struct slab *slab;
7005
7006	if (!node_nr_slabs(n))
7007	continue;
7008
7009	spin_lock_irqsave(&n->list_lock, flags);
7010	list_for_each_entry(slab, &n->partial, slab_list)
7011	process_slab(t, s, slab, alloc, obj_map);
7012	list_for_each_entry(slab, &n->full, slab_list)
7013	process_slab(t, s, slab, alloc, obj_map);
7014	spin_unlock_irqrestore(&n->list_lock, flags);
7015	}
7016
7017	/* Sort locations by count */
7018	sort_r(t->loc, t->count, sizeof(struct location),
7019	cmp_loc_by_count, NULL, NULL);
7020
7021	bitmap_free(obj_map);
7022	return 0;
7023	}
7024
7025	static int slab_debug_trace_release(struct inode *inode, struct file *file)
7026	{
7027	struct seq_file *seq = file->private_data;
7028	struct loc_track *t = seq->private;
7029
7030	free_loc_track(t);
7031	return seq_release_private(inode, file);
7032	}
7033
7034	static const struct file_operations slab_debugfs_fops = {
7035	.open = slab_debug_trace_open,
7036	.read = seq_read,
7037	.llseek = seq_lseek,
7038	.release = slab_debug_trace_release,
7039	};
7040
7041	static void debugfs_slab_add(struct kmem_cache *s)
7042	{
7043	struct dentry *slab_cache_dir;
7044
7045	if (unlikely(!slab_debugfs_root))
7046	return;
7047
7048	slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
7049
7050	debugfs_create_file("alloc_traces", 0400,
7051	slab_cache_dir, s, &slab_debugfs_fops);
7052
7053	debugfs_create_file("free_traces", 0400,
7054	slab_cache_dir, s, &slab_debugfs_fops);
7055	}
7056
7057	void debugfs_slab_release(struct kmem_cache *s)
7058	{
7059	debugfs_lookup_and_remove(s->name, slab_debugfs_root);
7060	}
7061
7062	static int __init slab_debugfs_init(void)
7063	{
7064	struct kmem_cache *s;
7065
7066	slab_debugfs_root = debugfs_create_dir("slab", NULL);
7067
7068	list_for_each_entry(s, &slab_caches, list)
7069	if (s->flags & SLAB_STORE_USER)
7070	debugfs_slab_add(s);
7071
7072	return 0;
7073
7074	}
7075	__initcall(slab_debugfs_init);
7076	#endif
7077	/*
7078	* The /proc/slabinfo ABI
7079	*/
7080	#ifdef CONFIG_SLUB_DEBUG
7081	void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
7082	{
7083	unsigned long nr_slabs = 0;
7084	unsigned long nr_objs = 0;
7085	unsigned long nr_free = 0;
7086	int node;
7087	struct kmem_cache_node *n;
7088
7089	for_each_kmem_cache_node(s, node, n) {
7090	nr_slabs += node_nr_slabs(n);
7091	nr_objs += node_nr_objs(n);
7092	nr_free += count_partial(n, count_free);
7093	}
7094
7095	sinfo->active_objs = nr_objs - nr_free;
7096	sinfo->num_objs = nr_objs;
7097	sinfo->active_slabs = nr_slabs;
7098	sinfo->num_slabs = nr_slabs;
7099	sinfo->objects_per_slab = oo_objects(s->oo);
7100	sinfo->cache_order = oo_order(s->oo);
7101	}
7102
7103	void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
7104	{
7105	}
7106
7107	ssize_t slabinfo_write(struct file *file, const char __user *buffer,
7108	size_t count, loff_t *ppos)
7109	{
7110	return -EIO;
7111	}
7112	#endif /* CONFIG_SLUB_DEBUG */
7113