1	/*
2	* Resizable virtual memory filesystem for Linux.
3	*
4	* Copyright (C) 2000 Linus Torvalds.
5	* 2000 Transmeta Corp.
6	* 2000-2001 Christoph Rohland
7	* 2000-2001 SAP AG
8	* 2002 Red Hat Inc.
9	* Copyright (C) 2002-2011 Hugh Dickins.
10	* Copyright (C) 2011 Google Inc.
11	* Copyright (C) 2002-2005 VERITAS Software Corporation.
12	* Copyright (C) 2004 Andi Kleen, SuSE Labs
13	*
14	* Extended attribute support for tmpfs:
15	* Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16	* Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17	*
18	* tiny-shmem:
19	* Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20	*
21	* This file is released under the GPL.
22	*/
23
24	#include <linux/fs.h>
25	#include <linux/init.h>
26	#include <linux/vfs.h>
27	#include <linux/mount.h>
28	#include <linux/ramfs.h>
29	#include <linux/pagemap.h>
30	#include <linux/file.h>
31	#include <linux/fileattr.h>
32	#include <linux/mm.h>
33	#include <linux/random.h>
34	#include <linux/sched/signal.h>
35	#include <linux/export.h>
36	#include <linux/shmem_fs.h>
37	#include <linux/swap.h>
38	#include <linux/uio.h>
39	#include <linux/hugetlb.h>
40	#include <linux/fs_parser.h>
41	#include <linux/swapfile.h>
42	#include <linux/iversion.h>
43	#include <linux/unicode.h>
44	#include "swap.h"
45
46	static struct vfsmount *shm_mnt __ro_after_init;
47
48	#ifdef CONFIG_SHMEM
49	/*
50	* This virtual memory filesystem is heavily based on the ramfs. It
51	* extends ramfs by the ability to use swap and honor resource limits
52	* which makes it a completely usable filesystem.
53	*/
54
55	#include <linux/xattr.h>
56	#include <linux/exportfs.h>
57	#include <linux/posix_acl.h>
58	#include <linux/posix_acl_xattr.h>
59	#include <linux/mman.h>
60	#include <linux/string.h>
61	#include <linux/slab.h>
62	#include <linux/backing-dev.h>
63	#include <linux/writeback.h>
64	#include <linux/pagevec.h>
65	#include <linux/percpu_counter.h>
66	#include <linux/falloc.h>
67	#include <linux/splice.h>
68	#include <linux/security.h>
69	#include <linux/swapops.h>
70	#include <linux/mempolicy.h>
71	#include <linux/namei.h>
72	#include <linux/ctype.h>
73	#include <linux/migrate.h>
74	#include <linux/highmem.h>
75	#include <linux/seq_file.h>
76	#include <linux/magic.h>
77	#include <linux/syscalls.h>
78	#include <linux/fcntl.h>
79	#include <uapi/linux/memfd.h>
80	#include <linux/rmap.h>
81	#include <linux/uuid.h>
82	#include <linux/quotaops.h>
83	#include <linux/rcupdate_wait.h>
84
85	#include <linux/uaccess.h>
86
87	#include "internal.h"
88
89	#define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT)
90
91	/* Pretend that each entry is of this size in directory's i_size */
92	#define BOGO_DIRENT_SIZE 20
93
94	/* Pretend that one inode + its dentry occupy this much memory */
95	#define BOGO_INODE_SIZE 1024
96
97	/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
98	#define SHORT_SYMLINK_LEN 128
99
100	/*
101	* shmem_fallocate communicates with shmem_fault or shmem_writeout via
102	* inode->i_private (with i_rwsem making sure that it has only one user at
103	* a time): we would prefer not to enlarge the shmem inode just for that.
104	*/
105	struct shmem_falloc {
106	wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
107	pgoff_t start; /* start of range currently being fallocated */
108	pgoff_t next; /* the next page offset to be fallocated */
109	pgoff_t nr_falloced; /* how many new pages have been fallocated */
110	pgoff_t nr_unswapped; /* how often writeout refused to swap out */
111	};
112
113	struct shmem_options {
114	unsigned long long blocks;
115	unsigned long long inodes;
116	struct mempolicy *mpol;
117	kuid_t uid;
118	kgid_t gid;
119	umode_t mode;
120	bool full_inums;
121	int huge;
122	int seen;
123	bool noswap;
124	unsigned short quota_types;
125	struct shmem_quota_limits qlimits;
126	#if IS_ENABLED(CONFIG_UNICODE)
127	struct unicode_map *encoding;
128	bool strict_encoding;
129	#endif
130	#define SHMEM_SEEN_BLOCKS 1
131	#define SHMEM_SEEN_INODES 2
132	#define SHMEM_SEEN_HUGE 4
133	#define SHMEM_SEEN_INUMS 8
134	#define SHMEM_SEEN_NOSWAP 16
135	#define SHMEM_SEEN_QUOTA 32
136	};
137
138	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
139	static unsigned long huge_shmem_orders_always __read_mostly;
140	static unsigned long huge_shmem_orders_madvise __read_mostly;
141	static unsigned long huge_shmem_orders_inherit __read_mostly;
142	static unsigned long huge_shmem_orders_within_size __read_mostly;
143	static bool shmem_orders_configured __initdata;
144	#endif
145
146	#ifdef CONFIG_TMPFS
147	static unsigned long shmem_default_max_blocks(void)
148	{
149	return totalram_pages() / 2;
150	}
151
152	static unsigned long shmem_default_max_inodes(void)
153	{
154	unsigned long nr_pages = totalram_pages();
155
156	return min3(nr_pages - totalhigh_pages(), nr_pages / 2,
157	ULONG_MAX / BOGO_INODE_SIZE);
158	}
159	#endif
160
161	static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
162	struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
163	struct vm_area_struct *vma, vm_fault_t *fault_type);
164
165	static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
166	{
167	return sb->s_fs_info;
168	}
169
170	/*
171	* shmem_file_setup pre-accounts the whole fixed size of a VM object,
172	* for shared memory and for shared anonymous (/dev/zero) mappings
173	* (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
174	* consistent with the pre-accounting of private mappings ...
175	*/
176	static inline int shmem_acct_size(unsigned long flags, loff_t size)
177	{
178	return (flags & VM_NORESERVE) ?
179	0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
180	}
181
182	static inline void shmem_unacct_size(unsigned long flags, loff_t size)
183	{
184	if (!(flags & VM_NORESERVE))
185	vm_unacct_memory(VM_ACCT(size));
186	}
187
188	static inline int shmem_reacct_size(unsigned long flags,
189	loff_t oldsize, loff_t newsize)
190	{
191	if (!(flags & VM_NORESERVE)) {
192	if (VM_ACCT(newsize) > VM_ACCT(oldsize))
193	return security_vm_enough_memory_mm(current->mm,
194	VM_ACCT(newsize) - VM_ACCT(oldsize));
195	else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
196	vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
197	}
198	return 0;
199	}
200
201	/*
202	* ... whereas tmpfs objects are accounted incrementally as
203	* pages are allocated, in order to allow large sparse files.
204	* shmem_get_folio reports shmem_acct_blocks failure as -ENOSPC not -ENOMEM,
205	* so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
206	*/
207	static inline int shmem_acct_blocks(unsigned long flags, long pages)
208	{
209	if (!(flags & VM_NORESERVE))
210	return 0;
211
212	return security_vm_enough_memory_mm(current->mm,
213	pages: pages * VM_ACCT(PAGE_SIZE));
214	}
215
216	static inline void shmem_unacct_blocks(unsigned long flags, long pages)
217	{
218	if (flags & VM_NORESERVE)
219	vm_unacct_memory(pages: pages * VM_ACCT(PAGE_SIZE));
220	}
221
222	static int shmem_inode_acct_blocks(struct inode *inode, long pages)
223	{
224	struct shmem_inode_info *info = SHMEM_I(inode);
225	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
226	int err = -ENOSPC;
227
228	if (shmem_acct_blocks(flags: info->flags, pages))
229	return err;
230
231	might_sleep(); /* when quotas */
232	if (sbinfo->max_blocks) {
233	if (!percpu_counter_limited_add(fbc: &sbinfo->used_blocks,
234	limit: sbinfo->max_blocks, amount: pages))
235	goto unacct;
236
237	err = dquot_alloc_block_nodirty(inode, nr: pages);
238	if (err) {
239	percpu_counter_sub(fbc: &sbinfo->used_blocks, amount: pages);
240	goto unacct;
241	}
242	} else {
243	err = dquot_alloc_block_nodirty(inode, nr: pages);
244	if (err)
245	goto unacct;
246	}
247
248	return 0;
249
250	unacct:
251	shmem_unacct_blocks(flags: info->flags, pages);
252	return err;
253	}
254
255	static void shmem_inode_unacct_blocks(struct inode *inode, long pages)
256	{
257	struct shmem_inode_info *info = SHMEM_I(inode);
258	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
259
260	might_sleep(); /* when quotas */
261	dquot_free_block_nodirty(inode, nr: pages);
262
263	if (sbinfo->max_blocks)
264	percpu_counter_sub(fbc: &sbinfo->used_blocks, amount: pages);
265	shmem_unacct_blocks(flags: info->flags, pages);
266	}
267
268	static const struct super_operations shmem_ops;
269	static const struct address_space_operations shmem_aops;
270	static const struct file_operations shmem_file_operations;
271	static const struct inode_operations shmem_inode_operations;
272	static const struct inode_operations shmem_dir_inode_operations;
273	static const struct inode_operations shmem_special_inode_operations;
274	static const struct vm_operations_struct shmem_vm_ops;
275	static const struct vm_operations_struct shmem_anon_vm_ops;
276	static struct file_system_type shmem_fs_type;
277
278	bool shmem_mapping(struct address_space *mapping)
279	{
280	return mapping->a_ops == &shmem_aops;
281	}
282	EXPORT_SYMBOL_GPL(shmem_mapping);
283
284	bool vma_is_anon_shmem(struct vm_area_struct *vma)
285	{
286	return vma->vm_ops == &shmem_anon_vm_ops;
287	}
288
289	bool vma_is_shmem(struct vm_area_struct *vma)
290	{
291	return vma_is_anon_shmem(vma) || vma->vm_ops == &shmem_vm_ops;
292	}
293
294	static LIST_HEAD(shmem_swaplist);
295	static DEFINE_MUTEX(shmem_swaplist_mutex);
296
297	#ifdef CONFIG_TMPFS_QUOTA
298
299	static int shmem_enable_quotas(struct super_block *sb,
300	unsigned short quota_types)
301	{
302	int type, err = 0;
303
304	sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE | DQUOT_NOLIST_DIRTY;
305	for (type = 0; type < SHMEM_MAXQUOTAS; type++) {
306	if (!(quota_types & (1 << type)))
307	continue;
308	err = dquot_load_quota_sb(sb, type, QFMT_SHMEM,
309	DQUOT_USAGE_ENABLED |
310	DQUOT_LIMITS_ENABLED);
311	if (err)
312	goto out_err;
313	}
314	return 0;
315
316	out_err:
317	pr_warn("tmpfs: failed to enable quota tracking (type=%d, err=%d)\n",
318	type, err);
319	for (type--; type >= 0; type--)
320	dquot_quota_off(sb, type);
321	return err;
322	}
323
324	static void shmem_disable_quotas(struct super_block *sb)
325	{
326	int type;
327
328	for (type = 0; type < SHMEM_MAXQUOTAS; type++)
329	dquot_quota_off(sb, type);
330	}
331
332	static struct dquot __rcu **shmem_get_dquots(struct inode *inode)
333	{
334	return SHMEM_I(inode)->i_dquot;
335	}
336	#endif /* CONFIG_TMPFS_QUOTA */
337
338	/*
339	* shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
340	* produces a novel ino for the newly allocated inode.
341	*
342	* It may also be called when making a hard link to permit the space needed by
343	* each dentry. However, in that case, no new inode number is needed since that
344	* internally draws from another pool of inode numbers (currently global
345	* get_next_ino()). This case is indicated by passing NULL as inop.
346	*/
347	#define SHMEM_INO_BATCH 1024
348	static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
349	{
350	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
351	ino_t ino;
352
353	if (!(sb->s_flags & SB_KERNMOUNT)) {
354	raw_spin_lock(&sbinfo->stat_lock);
355	if (sbinfo->max_inodes) {
356	if (sbinfo->free_ispace < BOGO_INODE_SIZE) {
357	raw_spin_unlock(&sbinfo->stat_lock);
358	return -ENOSPC;
359	}
360	sbinfo->free_ispace -= BOGO_INODE_SIZE;
361	}
362	if (inop) {
363	ino = sbinfo->next_ino++;
364	if (unlikely(is_zero_ino(ino)))
365	ino = sbinfo->next_ino++;
366	if (unlikely(!sbinfo->full_inums &&
367	ino > UINT_MAX)) {
368	/*
369	* Emulate get_next_ino uint wraparound for
370	* compatibility
371	*/
372	if (IS_ENABLED(CONFIG_64BIT))
373	pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
374	__func__, MINOR(sb->s_dev));
375	sbinfo->next_ino = 1;
376	ino = sbinfo->next_ino++;
377	}
378	*inop = ino;
379	}
380	raw_spin_unlock(&sbinfo->stat_lock);
381	} else if (inop) {
382	/*
383	* __shmem_file_setup, one of our callers, is lock-free: it
384	* doesn't hold stat_lock in shmem_reserve_inode since
385	* max_inodes is always 0, and is called from potentially
386	* unknown contexts. As such, use a per-cpu batched allocator
387	* which doesn't require the per-sb stat_lock unless we are at
388	* the batch boundary.
389	*
390	* We don't need to worry about inode{32,64} since SB_KERNMOUNT
391	* shmem mounts are not exposed to userspace, so we don't need
392	* to worry about things like glibc compatibility.
393	*/
394	ino_t *next_ino;
395
396	next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
397	ino = *next_ino;
398	if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
399	raw_spin_lock(&sbinfo->stat_lock);
400	ino = sbinfo->next_ino;
401	sbinfo->next_ino += SHMEM_INO_BATCH;
402	raw_spin_unlock(&sbinfo->stat_lock);
403	if (unlikely(is_zero_ino(ino)))
404	ino++;
405	}
406	*inop = ino;
407	*next_ino = ++ino;
408	put_cpu();
409	}
410
411	return 0;
412	}
413
414	static void shmem_free_inode(struct super_block *sb, size_t freed_ispace)
415	{
416	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
417	if (sbinfo->max_inodes) {
418	raw_spin_lock(&sbinfo->stat_lock);
419	sbinfo->free_ispace += BOGO_INODE_SIZE + freed_ispace;
420	raw_spin_unlock(&sbinfo->stat_lock);
421	}
422	}
423
424	/**
425	* shmem_recalc_inode - recalculate the block usage of an inode
426	* @inode: inode to recalc
427	* @alloced: the change in number of pages allocated to inode
428	* @swapped: the change in number of pages swapped from inode
429	*
430	* We have to calculate the free blocks since the mm can drop
431	* undirtied hole pages behind our back.
432	*
433	* But normally info->alloced == inode->i_mapping->nrpages + info->swapped
434	* So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
435	*/
436	static void shmem_recalc_inode(struct inode *inode, long alloced, long swapped)
437	{
438	struct shmem_inode_info *info = SHMEM_I(inode);
439	long freed;
440
441	spin_lock(lock: &info->lock);
442	info->alloced += alloced;
443	info->swapped += swapped;
444	freed = info->alloced - info->swapped -
445	READ_ONCE(inode->i_mapping->nrpages);
446	/*
447	* Special case: whereas normally shmem_recalc_inode() is called
448	* after i_mapping->nrpages has already been adjusted (up or down),
449	* shmem_writeout() has to raise swapped before nrpages is lowered -
450	* to stop a racing shmem_recalc_inode() from thinking that a page has
451	* been freed. Compensate here, to avoid the need for a followup call.
452	*/
453	if (swapped > 0)
454	freed += swapped;
455	if (freed > 0)
456	info->alloced -= freed;
457	spin_unlock(lock: &info->lock);
458
459	/* The quota case may block */
460	if (freed > 0)
461	shmem_inode_unacct_blocks(inode, pages: freed);
462	}
463
464	bool shmem_charge(struct inode *inode, long pages)
465	{
466	struct address_space *mapping = inode->i_mapping;
467
468	if (shmem_inode_acct_blocks(inode, pages))
469	return false;
470
471	/* nrpages adjustment first, then shmem_recalc_inode() when balanced */
472	xa_lock_irq(&mapping->i_pages);
473	mapping->nrpages += pages;
474	xa_unlock_irq(&mapping->i_pages);
475
476	shmem_recalc_inode(inode, alloced: pages, swapped: 0);
477	return true;
478	}
479
480	void shmem_uncharge(struct inode *inode, long pages)
481	{
482	/* pages argument is currently unused: keep it to help debugging */
483	/* nrpages adjustment done by __filemap_remove_folio() or caller */
484
485	shmem_recalc_inode(inode, alloced: 0, swapped: 0);
486	}
487
488	/*
489	* Replace item expected in xarray by a new item, while holding xa_lock.
490	*/
491	static int shmem_replace_entry(struct address_space *mapping,
492	pgoff_t index, void *expected, void *replacement)
493	{
494	XA_STATE(xas, &mapping->i_pages, index);
495	void *item;
496
497	VM_BUG_ON(!expected);
498	VM_BUG_ON(!replacement);
499	item = xas_load(&xas);
500	if (item != expected)
501	return -ENOENT;
502	xas_store(&xas, entry: replacement);
503	return 0;
504	}
505
506	/*
507	* Sometimes, before we decide whether to proceed or to fail, we must check
508	* that an entry was not already brought back from swap by a racing thread.
509	*
510	* Checking folio is not enough: by the time a swapcache folio is locked, it
511	* might be reused, and again be swapcache, using the same swap as before.
512	*/
513	static bool shmem_confirm_swap(struct address_space *mapping,
514	pgoff_t index, swp_entry_t swap)
515	{
516	return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(entry: swap);
517	}
518
519	/*
520	* Definitions for "huge tmpfs": tmpfs mounted with the huge= option
521	*
522	* SHMEM_HUGE_NEVER:
523	* disables huge pages for the mount;
524	* SHMEM_HUGE_ALWAYS:
525	* enables huge pages for the mount;
526	* SHMEM_HUGE_WITHIN_SIZE:
527	* only allocate huge pages if the page will be fully within i_size,
528	* also respect madvise() hints;
529	* SHMEM_HUGE_ADVISE:
530	* only allocate huge pages if requested with madvise();
531	*/
532
533	#define SHMEM_HUGE_NEVER 0
534	#define SHMEM_HUGE_ALWAYS 1
535	#define SHMEM_HUGE_WITHIN_SIZE 2
536	#define SHMEM_HUGE_ADVISE 3
537
538	/*
539	* Special values.
540	* Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
541	*
542	* SHMEM_HUGE_DENY:
543	* disables huge on shm_mnt and all mounts, for emergency use;
544	* SHMEM_HUGE_FORCE:
545	* enables huge on shm_mnt and all mounts, w/o needing option, for testing;
546	*
547	*/
548	#define SHMEM_HUGE_DENY (-1)
549	#define SHMEM_HUGE_FORCE (-2)
550
551	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
552	/* ifdef here to avoid bloating shmem.o when not necessary */
553
554	static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
555	static int tmpfs_huge __read_mostly = SHMEM_HUGE_NEVER;
556
557	/**
558	* shmem_mapping_size_orders - Get allowable folio orders for the given file size.
559	* @mapping: Target address_space.
560	* @index: The page index.
561	* @write_end: end of a write, could extend inode size.
562	*
563	* This returns huge orders for folios (when supported) based on the file size
564	* which the mapping currently allows at the given index. The index is relevant
565	* due to alignment considerations the mapping might have. The returned order
566	* may be less than the size passed.
567	*
568	* Return: The orders.
569	*/
570	static inline unsigned int
571	shmem_mapping_size_orders(struct address_space *mapping, pgoff_t index, loff_t write_end)
572	{
573	unsigned int order;
574	size_t size;
575
576	if (!mapping_large_folio_support(mapping) || !write_end)
577	return 0;
578
579	/* Calculate the write size based on the write_end */
580	size = write_end - (index << PAGE_SHIFT);
581	order = filemap_get_order(size);
582	if (!order)
583	return 0;
584
585	/* If we're not aligned, allocate a smaller folio */
586	if (index & ((1UL << order) - 1))
587	order = __ffs(index);
588
589	order = min_t(size_t, order, MAX_PAGECACHE_ORDER);
590	return order > 0 ? BIT(order + 1) - 1 : 0;
591	}
592
593	static unsigned int shmem_get_orders_within_size(struct inode *inode,
594	unsigned long within_size_orders, pgoff_t index,
595	loff_t write_end)
596	{
597	pgoff_t aligned_index;
598	unsigned long order;
599	loff_t i_size;
600
601	order = highest_order(orders: within_size_orders);
602	while (within_size_orders) {
603	aligned_index = round_up(index + 1, 1 << order);
604	i_size = max(write_end, i_size_read(inode));
605	i_size = round_up(i_size, PAGE_SIZE);
606	if (i_size >> PAGE_SHIFT >= aligned_index)
607	return within_size_orders;
608
609	order = next_order(orders: &within_size_orders, prev: order);
610	}
611
612	return 0;
613	}
614
615	static unsigned int shmem_huge_global_enabled(struct inode *inode, pgoff_t index,
616	loff_t write_end, bool shmem_huge_force,
617	struct vm_area_struct *vma,
618	unsigned long vm_flags)
619	{
620	unsigned int maybe_pmd_order = HPAGE_PMD_ORDER > MAX_PAGECACHE_ORDER ?
621	0 : BIT(HPAGE_PMD_ORDER);
622	unsigned long within_size_orders;
623
624	if (!S_ISREG(inode->i_mode))
625	return 0;
626	if (shmem_huge == SHMEM_HUGE_DENY)
627	return 0;
628	if (shmem_huge_force || shmem_huge == SHMEM_HUGE_FORCE)
629	return maybe_pmd_order;
630
631	/*
632	* The huge order allocation for anon shmem is controlled through
633	* the mTHP interface, so we still use PMD-sized huge order to
634	* check whether global control is enabled.
635	*
636	* For tmpfs mmap()'s huge order, we still use PMD-sized order to
637	* allocate huge pages due to lack of a write size hint.
638	*
639	* Otherwise, tmpfs will allow getting a highest order hint based on
640	* the size of write and fallocate paths, then will try each allowable
641	* huge orders.
642	*/
643	switch (SHMEM_SB(sb: inode->i_sb)->huge) {
644	case SHMEM_HUGE_ALWAYS:
645	if (vma)
646	return maybe_pmd_order;
647
648	return shmem_mapping_size_orders(mapping: inode->i_mapping, index, write_end);
649	case SHMEM_HUGE_WITHIN_SIZE:
650	if (vma)
651	within_size_orders = maybe_pmd_order;
652	else
653	within_size_orders = shmem_mapping_size_orders(mapping: inode->i_mapping,
654	index, write_end);
655
656	within_size_orders = shmem_get_orders_within_size(inode, within_size_orders,
657	index, write_end);
658	if (within_size_orders > 0)
659	return within_size_orders;
660
661	fallthrough;
662	case SHMEM_HUGE_ADVISE:
663	if (vm_flags & VM_HUGEPAGE)
664	return maybe_pmd_order;
665	fallthrough;
666	default:
667	return 0;
668	}
669	}
670
671	static int shmem_parse_huge(const char *str)
672	{
673	int huge;
674
675	if (!str)
676	return -EINVAL;
677
678	if (!strcmp(str, "never"))
679	huge = SHMEM_HUGE_NEVER;
680	else if (!strcmp(str, "always"))
681	huge = SHMEM_HUGE_ALWAYS;
682	else if (!strcmp(str, "within_size"))
683	huge = SHMEM_HUGE_WITHIN_SIZE;
684	else if (!strcmp(str, "advise"))
685	huge = SHMEM_HUGE_ADVISE;
686	else if (!strcmp(str, "deny"))
687	huge = SHMEM_HUGE_DENY;
688	else if (!strcmp(str, "force"))
689	huge = SHMEM_HUGE_FORCE;
690	else
691	return -EINVAL;
692
693	if (!has_transparent_hugepage() &&
694	huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
695	return -EINVAL;
696
697	/* Do not override huge allocation policy with non-PMD sized mTHP */
698	if (huge == SHMEM_HUGE_FORCE &&
699	huge_shmem_orders_inherit != BIT(HPAGE_PMD_ORDER))
700	return -EINVAL;
701
702	return huge;
703	}
704
705	#if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
706	static const char *shmem_format_huge(int huge)
707	{
708	switch (huge) {
709	case SHMEM_HUGE_NEVER:
710	return "never";
711	case SHMEM_HUGE_ALWAYS:
712	return "always";
713	case SHMEM_HUGE_WITHIN_SIZE:
714	return "within_size";
715	case SHMEM_HUGE_ADVISE:
716	return "advise";
717	case SHMEM_HUGE_DENY:
718	return "deny";
719	case SHMEM_HUGE_FORCE:
720	return "force";
721	default:
722	VM_BUG_ON(1);
723	return "bad_val";
724	}
725	}
726	#endif
727
728	static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
729	struct shrink_control *sc, unsigned long nr_to_free)
730	{
731	LIST_HEAD(list), *pos, *next;
732	struct inode *inode;
733	struct shmem_inode_info *info;
734	struct folio *folio;
735	unsigned long batch = sc ? sc->nr_to_scan : 128;
736	unsigned long split = 0, freed = 0;
737
738	if (list_empty(head: &sbinfo->shrinklist))
739	return SHRINK_STOP;
740
741	spin_lock(lock: &sbinfo->shrinklist_lock);
742	list_for_each_safe(pos, next, &sbinfo->shrinklist) {
743	info = list_entry(pos, struct shmem_inode_info, shrinklist);
744
745	/* pin the inode */
746	inode = igrab(&info->vfs_inode);
747
748	/* inode is about to be evicted */
749	if (!inode) {
750	list_del_init(entry: &info->shrinklist);
751	goto next;
752	}
753
754	list_move(list: &info->shrinklist, head: &list);
755	next:
756	sbinfo->shrinklist_len--;
757	if (!--batch)
758	break;
759	}
760	spin_unlock(lock: &sbinfo->shrinklist_lock);
761
762	list_for_each_safe(pos, next, &list) {
763	pgoff_t next, end;
764	loff_t i_size;
765	int ret;
766
767	info = list_entry(pos, struct shmem_inode_info, shrinklist);
768	inode = &info->vfs_inode;
769
770	if (nr_to_free && freed >= nr_to_free)
771	goto move_back;
772
773	i_size = i_size_read(inode);
774	folio = filemap_get_entry(mapping: inode->i_mapping, index: i_size / PAGE_SIZE);
775	if (!folio || xa_is_value(entry: folio))
776	goto drop;
777
778	/* No large folio at the end of the file: nothing to split */
779	if (!folio_test_large(folio)) {
780	folio_put(folio);
781	goto drop;
782	}
783
784	/* Check if there is anything to gain from splitting */
785	next = folio_next_index(folio);
786	end = shmem_fallocend(inode, DIV_ROUND_UP(i_size, PAGE_SIZE));
787	if (end <= folio->index || end >= next) {
788	folio_put(folio);
789	goto drop;
790	}
791
792	/*
793	* Move the inode on the list back to shrinklist if we failed
794	* to lock the page at this time.
795	*
796	* Waiting for the lock may lead to deadlock in the
797	* reclaim path.
798	*/
799	if (!folio_trylock(folio)) {
800	folio_put(folio);
801	goto move_back;
802	}
803
804	ret = split_folio(folio);
805	folio_unlock(folio);
806	folio_put(folio);
807
808	/* If split failed move the inode on the list back to shrinklist */
809	if (ret)
810	goto move_back;
811
812	freed += next - end;
813	split++;
814	drop:
815	list_del_init(entry: &info->shrinklist);
816	goto put;
817	move_back:
818	/*
819	* Make sure the inode is either on the global list or deleted
820	* from any local list before iput() since it could be deleted
821	* in another thread once we put the inode (then the local list
822	* is corrupted).
823	*/
824	spin_lock(lock: &sbinfo->shrinklist_lock);
825	list_move(list: &info->shrinklist, head: &sbinfo->shrinklist);
826	sbinfo->shrinklist_len++;
827	spin_unlock(lock: &sbinfo->shrinklist_lock);
828	put:
829	iput(inode);
830	}
831
832	return split;
833	}
834
835	static long shmem_unused_huge_scan(struct super_block *sb,
836	struct shrink_control *sc)
837	{
838	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
839
840	if (!READ_ONCE(sbinfo->shrinklist_len))
841	return SHRINK_STOP;
842
843	return shmem_unused_huge_shrink(sbinfo, sc, nr_to_free: 0);
844	}
845
846	static long shmem_unused_huge_count(struct super_block *sb,
847	struct shrink_control *sc)
848	{
849	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
850	return READ_ONCE(sbinfo->shrinklist_len);
851	}
852	#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
853
854	#define shmem_huge SHMEM_HUGE_DENY
855
856	static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
857	struct shrink_control *sc, unsigned long nr_to_free)
858	{
859	return 0;
860	}
861
862	static unsigned int shmem_huge_global_enabled(struct inode *inode, pgoff_t index,
863	loff_t write_end, bool shmem_huge_force,
864	struct vm_area_struct *vma,
865	unsigned long vm_flags)
866	{
867	return 0;
868	}
869	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
870
871	static void shmem_update_stats(struct folio *folio, int nr_pages)
872	{
873	if (folio_test_pmd_mappable(folio))
874	__lruvec_stat_mod_folio(folio, idx: NR_SHMEM_THPS, val: nr_pages);
875	__lruvec_stat_mod_folio(folio, idx: NR_FILE_PAGES, val: nr_pages);
876	__lruvec_stat_mod_folio(folio, idx: NR_SHMEM, val: nr_pages);
877	}
878
879	/*
880	* Somewhat like filemap_add_folio, but error if expected item has gone.
881	*/
882	static int shmem_add_to_page_cache(struct folio *folio,
883	struct address_space *mapping,
884	pgoff_t index, void *expected, gfp_t gfp)
885	{
886	XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
887	long nr = folio_nr_pages(folio);
888
889	VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
890	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
891	VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
892
893	folio_ref_add(folio, nr);
894	folio->mapping = mapping;
895	folio->index = index;
896
897	gfp &= GFP_RECLAIM_MASK;
898	folio_throttle_swaprate(folio, gfp);
899
900	do {
901	xas_lock_irq(&xas);
902	if (expected != xas_find_conflict(&xas)) {
903	xas_set_err(xas: &xas, err: -EEXIST);
904	goto unlock;
905	}
906	if (expected && xas_find_conflict(&xas)) {
907	xas_set_err(xas: &xas, err: -EEXIST);
908	goto unlock;
909	}
910	xas_store(&xas, entry: folio);
911	if (xas_error(xas: &xas))
912	goto unlock;
913	shmem_update_stats(folio, nr_pages: nr);
914	mapping->nrpages += nr;
915	unlock:
916	xas_unlock_irq(&xas);
917	} while (xas_nomem(&xas, gfp));
918
919	if (xas_error(xas: &xas)) {
920	folio->mapping = NULL;
921	folio_ref_sub(folio, nr);
922	return xas_error(xas: &xas);
923	}
924
925	return 0;
926	}
927
928	/*
929	* Somewhat like filemap_remove_folio, but substitutes swap for @folio.
930	*/
931	static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
932	{
933	struct address_space *mapping = folio->mapping;
934	long nr = folio_nr_pages(folio);
935	int error;
936
937	xa_lock_irq(&mapping->i_pages);
938	error = shmem_replace_entry(mapping, index: folio->index, expected: folio, replacement: radswap);
939	folio->mapping = NULL;
940	mapping->nrpages -= nr;
941	shmem_update_stats(folio, nr_pages: -nr);
942	xa_unlock_irq(&mapping->i_pages);
943	folio_put_refs(folio, refs: nr);
944	BUG_ON(error);
945	}
946
947	/*
948	* Remove swap entry from page cache, free the swap and its page cache. Returns
949	* the number of pages being freed. 0 means entry not found in XArray (0 pages
950	* being freed).
951	*/
952	static long shmem_free_swap(struct address_space *mapping,
953	pgoff_t index, void *radswap)
954	{
955	int order = xa_get_order(&mapping->i_pages, index);
956	void *old;
957
958	old = xa_cmpxchg_irq(xa: &mapping->i_pages, index, old: radswap, NULL, gfp: 0);
959	if (old != radswap)
960	return 0;
961	free_swap_and_cache_nr(entry: radix_to_swp_entry(arg: radswap), nr: 1 << order);
962
963	return 1 << order;
964	}
965
966	/*
967	* Determine (in bytes) how many of the shmem object's pages mapped by the
968	* given offsets are swapped out.
969	*
970	* This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
971	* as long as the inode doesn't go away and racy results are not a problem.
972	*/
973	unsigned long shmem_partial_swap_usage(struct address_space *mapping,
974	pgoff_t start, pgoff_t end)
975	{
976	XA_STATE(xas, &mapping->i_pages, start);
977	struct page *page;
978	unsigned long swapped = 0;
979	unsigned long max = end - 1;
980
981	rcu_read_lock();
982	xas_for_each(&xas, page, max) {
983	if (xas_retry(xas: &xas, entry: page))
984	continue;
985	if (xa_is_value(entry: page))
986	swapped += 1 << xas_get_order(xas: &xas);
987	if (xas.xa_index == max)
988	break;
989	if (need_resched()) {
990	xas_pause(&xas);
991	cond_resched_rcu();
992	}
993	}
994	rcu_read_unlock();
995
996	return swapped << PAGE_SHIFT;
997	}
998
999	/*
1000	* Determine (in bytes) how many of the shmem object's pages mapped by the
1001	* given vma is swapped out.
1002	*
1003	* This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
1004	* as long as the inode doesn't go away and racy results are not a problem.
1005	*/
1006	unsigned long shmem_swap_usage(struct vm_area_struct *vma)
1007	{
1008	struct inode *inode = file_inode(f: vma->vm_file);
1009	struct shmem_inode_info *info = SHMEM_I(inode);
1010	struct address_space *mapping = inode->i_mapping;
1011	unsigned long swapped;
1012
1013	/* Be careful as we don't hold info->lock */
1014	swapped = READ_ONCE(info->swapped);
1015
1016	/*
1017	* The easier cases are when the shmem object has nothing in swap, or
1018	* the vma maps it whole. Then we can simply use the stats that we
1019	* already track.
1020	*/
1021	if (!swapped)
1022	return 0;
1023
1024	if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
1025	return swapped << PAGE_SHIFT;
1026
1027	/* Here comes the more involved part */
1028	return shmem_partial_swap_usage(mapping, start: vma->vm_pgoff,
1029	end: vma->vm_pgoff + vma_pages(vma));
1030	}
1031
1032	/*
1033	* SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
1034	*/
1035	void shmem_unlock_mapping(struct address_space *mapping)
1036	{
1037	struct folio_batch fbatch;
1038	pgoff_t index = 0;
1039
1040	folio_batch_init(fbatch: &fbatch);
1041	/*
1042	* Minor point, but we might as well stop if someone else SHM_LOCKs it.
1043	*/
1044	while (!mapping_unevictable(mapping) &&
1045	filemap_get_folios(mapping, start: &index, end: ~0UL, fbatch: &fbatch)) {
1046	check_move_unevictable_folios(fbatch: &fbatch);
1047	folio_batch_release(fbatch: &fbatch);
1048	cond_resched();
1049	}
1050	}
1051
1052	static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
1053	{
1054	struct folio *folio;
1055
1056	/*
1057	* At first avoid shmem_get_folio(,,,SGP_READ): that fails
1058	* beyond i_size, and reports fallocated folios as holes.
1059	*/
1060	folio = filemap_get_entry(mapping: inode->i_mapping, index);
1061	if (!folio)
1062	return folio;
1063	if (!xa_is_value(entry: folio)) {
1064	folio_lock(folio);
1065	if (folio->mapping == inode->i_mapping)
1066	return folio;
1067	/* The folio has been swapped out */
1068	folio_unlock(folio);
1069	folio_put(folio);
1070	}
1071	/*
1072	* But read a folio back from swap if any of it is within i_size
1073	* (although in some cases this is just a waste of time).
1074	*/
1075	folio = NULL;
1076	shmem_get_folio(inode, index, write_end: 0, foliop: &folio, sgp: SGP_READ);
1077	return folio;
1078	}
1079
1080	/*
1081	* Remove range of pages and swap entries from page cache, and free them.
1082	* If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
1083	*/
1084	static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
1085	bool unfalloc)
1086	{
1087	struct address_space *mapping = inode->i_mapping;
1088	struct shmem_inode_info *info = SHMEM_I(inode);
1089	pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
1090	pgoff_t end = (lend + 1) >> PAGE_SHIFT;
1091	struct folio_batch fbatch;
1092	pgoff_t indices[PAGEVEC_SIZE];
1093	struct folio *folio;
1094	bool same_folio;
1095	long nr_swaps_freed = 0;
1096	pgoff_t index;
1097	int i;
1098
1099	if (lend == -1)
1100	end = -1; /* unsigned, so actually very big */
1101
1102	if (info->fallocend > start && info->fallocend <= end && !unfalloc)
1103	info->fallocend = start;
1104
1105	folio_batch_init(fbatch: &fbatch);
1106	index = start;
1107	while (index < end && find_lock_entries(mapping, start: &index, end: end - 1,
1108	fbatch: &fbatch, indices)) {
1109	for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) {
1110	folio = fbatch.folios[i];
1111
1112	if (xa_is_value(entry: folio)) {
1113	if (unfalloc)
1114	continue;
1115	nr_swaps_freed += shmem_free_swap(mapping,
1116	index: indices[i], radswap: folio);
1117	continue;
1118	}
1119
1120	if (!unfalloc || !folio_test_uptodate(folio))
1121	truncate_inode_folio(mapping, folio);
1122	folio_unlock(folio);
1123	}
1124	folio_batch_remove_exceptionals(fbatch: &fbatch);
1125	folio_batch_release(fbatch: &fbatch);
1126	cond_resched();
1127	}
1128
1129	/*
1130	* When undoing a failed fallocate, we want none of the partial folio
1131	* zeroing and splitting below, but shall want to truncate the whole
1132	* folio when !uptodate indicates that it was added by this fallocate,
1133	* even when [lstart, lend] covers only a part of the folio.
1134	*/
1135	if (unfalloc)
1136	goto whole_folios;
1137
1138	same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
1139	folio = shmem_get_partial_folio(inode, index: lstart >> PAGE_SHIFT);
1140	if (folio) {
1141	same_folio = lend < folio_pos(folio) + folio_size(folio);
1142	folio_mark_dirty(folio);
1143	if (!truncate_inode_partial_folio(folio, start: lstart, end: lend)) {
1144	start = folio_next_index(folio);
1145	if (same_folio)
1146	end = folio->index;
1147	}
1148	folio_unlock(folio);
1149	folio_put(folio);
1150	folio = NULL;
1151	}
1152
1153	if (!same_folio)
1154	folio = shmem_get_partial_folio(inode, index: lend >> PAGE_SHIFT);
1155	if (folio) {
1156	folio_mark_dirty(folio);
1157	if (!truncate_inode_partial_folio(folio, start: lstart, end: lend))
1158	end = folio->index;
1159	folio_unlock(folio);
1160	folio_put(folio);
1161	}
1162
1163	whole_folios:
1164
1165	index = start;
1166	while (index < end) {
1167	cond_resched();
1168
1169	if (!find_get_entries(mapping, start: &index, end: end - 1, fbatch: &fbatch,
1170	indices)) {
1171	/* If all gone or hole-punch or unfalloc, we're done */
1172	if (index == start || end != -1)
1173	break;
1174	/* But if truncating, restart to make sure all gone */
1175	index = start;
1176	continue;
1177	}
1178	for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) {
1179	folio = fbatch.folios[i];
1180
1181	if (xa_is_value(entry: folio)) {
1182	long swaps_freed;
1183
1184	if (unfalloc)
1185	continue;
1186	swaps_freed = shmem_free_swap(mapping, index: indices[i], radswap: folio);
1187	if (!swaps_freed) {
1188	/* Swap was replaced by page: retry */
1189	index = indices[i];
1190	break;
1191	}
1192	nr_swaps_freed += swaps_freed;
1193	continue;
1194	}
1195
1196	folio_lock(folio);
1197
1198	if (!unfalloc || !folio_test_uptodate(folio)) {
1199	if (folio_mapping(folio) != mapping) {
1200	/* Page was replaced by swap: retry */
1201	folio_unlock(folio);
1202	index = indices[i];
1203	break;
1204	}
1205	VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1206	folio);
1207
1208	if (!folio_test_large(folio)) {
1209	truncate_inode_folio(mapping, folio);
1210	} else if (truncate_inode_partial_folio(folio, start: lstart, end: lend)) {
1211	/*
1212	* If we split a page, reset the loop so
1213	* that we pick up the new sub pages.
1214	* Otherwise the THP was entirely
1215	* dropped or the target range was
1216	* zeroed, so just continue the loop as
1217	* is.
1218	*/
1219	if (!folio_test_large(folio)) {
1220	folio_unlock(folio);
1221	index = start;
1222	break;
1223	}
1224	}
1225	}
1226	folio_unlock(folio);
1227	}
1228	folio_batch_remove_exceptionals(fbatch: &fbatch);
1229	folio_batch_release(fbatch: &fbatch);
1230	}
1231
1232	shmem_recalc_inode(inode, alloced: 0, swapped: -nr_swaps_freed);
1233	}
1234
1235	void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1236	{
1237	shmem_undo_range(inode, lstart, lend, unfalloc: false);
1238	inode_set_mtime_to_ts(inode, ts: inode_set_ctime_current(inode));
1239	inode_inc_iversion(inode);
1240	}
1241	EXPORT_SYMBOL_GPL(shmem_truncate_range);
1242
1243	static int shmem_getattr(struct mnt_idmap *idmap,
1244	const struct path *path, struct kstat *stat,
1245	u32 request_mask, unsigned int query_flags)
1246	{
1247	struct inode *inode = path->dentry->d_inode;
1248	struct shmem_inode_info *info = SHMEM_I(inode);
1249
1250	if (info->alloced - info->swapped != inode->i_mapping->nrpages)
1251	shmem_recalc_inode(inode, alloced: 0, swapped: 0);
1252
1253	if (info->fsflags & FS_APPEND_FL)
1254	stat->attributes |= STATX_ATTR_APPEND;
1255	if (info->fsflags & FS_IMMUTABLE_FL)
1256	stat->attributes |= STATX_ATTR_IMMUTABLE;
1257	if (info->fsflags & FS_NODUMP_FL)
1258	stat->attributes |= STATX_ATTR_NODUMP;
1259	stat->attributes_mask |= (STATX_ATTR_APPEND |
1260	STATX_ATTR_IMMUTABLE |
1261	STATX_ATTR_NODUMP);
1262	generic_fillattr(idmap, request_mask, inode, stat);
1263
1264	if (shmem_huge_global_enabled(inode, index: 0, write_end: 0, shmem_huge_force: false, NULL, vm_flags: 0))
1265	stat->blksize = HPAGE_PMD_SIZE;
1266
1267	if (request_mask & STATX_BTIME) {
1268	stat->result_mask |= STATX_BTIME;
1269	stat->btime.tv_sec = info->i_crtime.tv_sec;
1270	stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1271	}
1272
1273	return 0;
1274	}
1275
1276	static int shmem_setattr(struct mnt_idmap *idmap,
1277	struct dentry *dentry, struct iattr *attr)
1278	{
1279	struct inode *inode = d_inode(dentry);
1280	struct shmem_inode_info *info = SHMEM_I(inode);
1281	int error;
1282	bool update_mtime = false;
1283	bool update_ctime = true;
1284
1285	error = setattr_prepare(idmap, dentry, attr);
1286	if (error)
1287	return error;
1288
1289	if ((info->seals & F_SEAL_EXEC) && (attr->ia_valid & ATTR_MODE)) {
1290	if ((inode->i_mode ^ attr->ia_mode) & 0111) {
1291	return -EPERM;
1292	}
1293	}
1294
1295	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1296	loff_t oldsize = inode->i_size;
1297	loff_t newsize = attr->ia_size;
1298
1299	/* protected by i_rwsem */
1300	if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1301	(newsize > oldsize && (info->seals & F_SEAL_GROW)))
1302	return -EPERM;
1303
1304	if (newsize != oldsize) {
1305	error = shmem_reacct_size(flags: SHMEM_I(inode)->flags,
1306	oldsize, newsize);
1307	if (error)
1308	return error;
1309	i_size_write(inode, i_size: newsize);
1310	update_mtime = true;
1311	} else {
1312	update_ctime = false;
1313	}
1314	if (newsize <= oldsize) {
1315	loff_t holebegin = round_up(newsize, PAGE_SIZE);
1316	if (oldsize > holebegin)
1317	unmap_mapping_range(mapping: inode->i_mapping,
1318	holebegin, holelen: 0, even_cows: 1);
1319	if (info->alloced)
1320	shmem_truncate_range(inode,
1321	newsize, (loff_t)-1);
1322	/* unmap again to remove racily COWed private pages */
1323	if (oldsize > holebegin)
1324	unmap_mapping_range(mapping: inode->i_mapping,
1325	holebegin, holelen: 0, even_cows: 1);
1326	}
1327	}
1328
1329	if (is_quota_modification(idmap, inode, ia: attr)) {
1330	error = dquot_initialize(inode);
1331	if (error)
1332	return error;
1333	}
1334
1335	/* Transfer quota accounting */
1336	if (i_uid_needs_update(idmap, attr, inode) ||
1337	i_gid_needs_update(idmap, attr, inode)) {
1338	error = dquot_transfer(idmap, inode, iattr: attr);
1339	if (error)
1340	return error;
1341	}
1342
1343	setattr_copy(idmap, inode, attr);
1344	if (attr->ia_valid & ATTR_MODE)
1345	error = posix_acl_chmod(idmap, dentry, inode->i_mode);
1346	if (!error && update_ctime) {
1347	inode_set_ctime_current(inode);
1348	if (update_mtime)
1349	inode_set_mtime_to_ts(inode, ts: inode_get_ctime(inode));
1350	inode_inc_iversion(inode);
1351	}
1352	return error;
1353	}
1354
1355	static void shmem_evict_inode(struct inode *inode)
1356	{
1357	struct shmem_inode_info *info = SHMEM_I(inode);
1358	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
1359	size_t freed = 0;
1360
1361	if (shmem_mapping(inode->i_mapping)) {
1362	shmem_unacct_size(flags: info->flags, size: inode->i_size);
1363	inode->i_size = 0;
1364	mapping_set_exiting(mapping: inode->i_mapping);
1365	shmem_truncate_range(inode, 0, (loff_t)-1);
1366	if (!list_empty(head: &info->shrinklist)) {
1367	spin_lock(lock: &sbinfo->shrinklist_lock);
1368	if (!list_empty(head: &info->shrinklist)) {
1369	list_del_init(entry: &info->shrinklist);
1370	sbinfo->shrinklist_len--;
1371	}
1372	spin_unlock(lock: &sbinfo->shrinklist_lock);
1373	}
1374	while (!list_empty(head: &info->swaplist)) {
1375	/* Wait while shmem_unuse() is scanning this inode... */
1376	wait_var_event(&info->stop_eviction,
1377	!atomic_read(&info->stop_eviction));
1378	mutex_lock(&shmem_swaplist_mutex);
1379	/* ...but beware of the race if we peeked too early */
1380	if (!atomic_read(v: &info->stop_eviction))
1381	list_del_init(entry: &info->swaplist);
1382	mutex_unlock(lock: &shmem_swaplist_mutex);
1383	}
1384	}
1385
1386	simple_xattrs_free(xattrs: &info->xattrs, freed_space: sbinfo->max_inodes ? &freed : NULL);
1387	shmem_free_inode(sb: inode->i_sb, freed_ispace: freed);
1388	WARN_ON(inode->i_blocks);
1389	clear_inode(inode);
1390	#ifdef CONFIG_TMPFS_QUOTA
1391	dquot_free_inode(inode);
1392	dquot_drop(inode);
1393	#endif
1394	}
1395
1396	static unsigned int shmem_find_swap_entries(struct address_space *mapping,
1397	pgoff_t start, struct folio_batch *fbatch,
1398	pgoff_t *indices, unsigned int type)
1399	{
1400	XA_STATE(xas, &mapping->i_pages, start);
1401	struct folio *folio;
1402	swp_entry_t entry;
1403
1404	rcu_read_lock();
1405	xas_for_each(&xas, folio, ULONG_MAX) {
1406	if (xas_retry(xas: &xas, entry: folio))
1407	continue;
1408
1409	if (!xa_is_value(entry: folio))
1410	continue;
1411
1412	entry = radix_to_swp_entry(arg: folio);
1413	/*
1414	* swapin error entries can be found in the mapping. But they're
1415	* deliberately ignored here as we've done everything we can do.
1416	*/
1417	if (swp_type(entry) != type)
1418	continue;
1419
1420	indices[folio_batch_count(fbatch)] = xas.xa_index;
1421	if (!folio_batch_add(fbatch, folio))
1422	break;
1423
1424	if (need_resched()) {
1425	xas_pause(&xas);
1426	cond_resched_rcu();
1427	}
1428	}
1429	rcu_read_unlock();
1430
1431	return folio_batch_count(fbatch);
1432	}
1433
1434	/*
1435	* Move the swapped pages for an inode to page cache. Returns the count
1436	* of pages swapped in, or the error in case of failure.
1437	*/
1438	static int shmem_unuse_swap_entries(struct inode *inode,
1439	struct folio_batch *fbatch, pgoff_t *indices)
1440	{
1441	int i = 0;
1442	int ret = 0;
1443	int error = 0;
1444	struct address_space *mapping = inode->i_mapping;
1445
1446	for (i = 0; i < folio_batch_count(fbatch); i++) {
1447	struct folio *folio = fbatch->folios[i];
1448
1449	error = shmem_swapin_folio(inode, index: indices[i], foliop: &folio, sgp: SGP_CACHE,
1450	gfp: mapping_gfp_mask(mapping), NULL, NULL);
1451	if (error == 0) {
1452	folio_unlock(folio);
1453	folio_put(folio);
1454	ret++;
1455	}
1456	if (error == -ENOMEM)
1457	break;
1458	error = 0;
1459	}
1460	return error ? error : ret;
1461	}
1462
1463	/*
1464	* If swap found in inode, free it and move page from swapcache to filecache.
1465	*/
1466	static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1467	{
1468	struct address_space *mapping = inode->i_mapping;
1469	pgoff_t start = 0;
1470	struct folio_batch fbatch;
1471	pgoff_t indices[PAGEVEC_SIZE];
1472	int ret = 0;
1473
1474	do {
1475	folio_batch_init(fbatch: &fbatch);
1476	if (!shmem_find_swap_entries(mapping, start, fbatch: &fbatch,
1477	indices, type)) {
1478	ret = 0;
1479	break;
1480	}
1481
1482	ret = shmem_unuse_swap_entries(inode, fbatch: &fbatch, indices);
1483	if (ret < 0)
1484	break;
1485
1486	start = indices[folio_batch_count(fbatch: &fbatch) - 1];
1487	} while (true);
1488
1489	return ret;
1490	}
1491
1492	/*
1493	* Read all the shared memory data that resides in the swap
1494	* device 'type' back into memory, so the swap device can be
1495	* unused.
1496	*/
1497	int shmem_unuse(unsigned int type)
1498	{
1499	struct shmem_inode_info *info, *next;
1500	int error = 0;
1501
1502	if (list_empty(head: &shmem_swaplist))
1503	return 0;
1504
1505	mutex_lock(&shmem_swaplist_mutex);
1506	start_over:
1507	list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1508	if (!info->swapped) {
1509	list_del_init(entry: &info->swaplist);
1510	continue;
1511	}
1512	/*
1513	* Drop the swaplist mutex while searching the inode for swap;
1514	* but before doing so, make sure shmem_evict_inode() will not
1515	* remove placeholder inode from swaplist, nor let it be freed
1516	* (igrab() would protect from unlink, but not from unmount).
1517	*/
1518	atomic_inc(v: &info->stop_eviction);
1519	mutex_unlock(lock: &shmem_swaplist_mutex);
1520
1521	error = shmem_unuse_inode(inode: &info->vfs_inode, type);
1522	cond_resched();
1523
1524	mutex_lock(&shmem_swaplist_mutex);
1525	if (atomic_dec_and_test(v: &info->stop_eviction))
1526	wake_up_var(var: &info->stop_eviction);
1527	if (error)
1528	break;
1529	if (list_empty(head: &info->swaplist))
1530	goto start_over;
1531	next = list_next_entry(info, swaplist);
1532	if (!info->swapped)
1533	list_del_init(entry: &info->swaplist);
1534	}
1535	mutex_unlock(lock: &shmem_swaplist_mutex);
1536
1537	return error;
1538	}
1539
1540	/**
1541	* shmem_writeout - Write the folio to swap
1542	* @folio: The folio to write
1543	* @wbc: How writeback is to be done
1544	*
1545	* Move the folio from the page cache to the swap cache.
1546	*/
1547	int shmem_writeout(struct folio *folio, struct writeback_control *wbc)
1548	{
1549	struct address_space *mapping = folio->mapping;
1550	struct inode *inode = mapping->host;
1551	struct shmem_inode_info *info = SHMEM_I(inode);
1552	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
1553	pgoff_t index;
1554	int nr_pages;
1555	bool split = false;
1556
1557	if (WARN_ON_ONCE(!wbc->for_reclaim))
1558	goto redirty;
1559
1560	if ((info->flags & VM_LOCKED) || sbinfo->noswap)
1561	goto redirty;
1562
1563	if (!total_swap_pages)
1564	goto redirty;
1565
1566	/*
1567	* If CONFIG_THP_SWAP is not enabled, the large folio should be
1568	* split when swapping.
1569	*
1570	* And shrinkage of pages beyond i_size does not split swap, so
1571	* swapout of a large folio crossing i_size needs to split too
1572	* (unless fallocate has been used to preallocate beyond EOF).
1573	*/
1574	if (folio_test_large(folio)) {
1575	index = shmem_fallocend(inode,
1576	DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE));
1577	if ((index > folio->index && index < folio_next_index(folio)) ||
1578	!IS_ENABLED(CONFIG_THP_SWAP))
1579	split = true;
1580	}
1581
1582	if (split) {
1583	try_split:
1584	/* Ensure the subpages are still dirty */
1585	folio_test_set_dirty(folio);
1586	if (split_folio_to_list(folio, list: wbc->list))
1587	goto redirty;
1588	folio_clear_dirty(folio);
1589	}
1590
1591	index = folio->index;
1592	nr_pages = folio_nr_pages(folio);
1593
1594	/*
1595	* This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1596	* value into swapfile.c, the only way we can correctly account for a
1597	* fallocated folio arriving here is now to initialize it and write it.
1598	*
1599	* That's okay for a folio already fallocated earlier, but if we have
1600	* not yet completed the fallocation, then (a) we want to keep track
1601	* of this folio in case we have to undo it, and (b) it may not be a
1602	* good idea to continue anyway, once we're pushing into swap. So
1603	* reactivate the folio, and let shmem_fallocate() quit when too many.
1604	*/
1605	if (!folio_test_uptodate(folio)) {
1606	if (inode->i_private) {
1607	struct shmem_falloc *shmem_falloc;
1608	spin_lock(lock: &inode->i_lock);
1609	shmem_falloc = inode->i_private;
1610	if (shmem_falloc &&
1611	!shmem_falloc->waitq &&
1612	index >= shmem_falloc->start &&
1613	index < shmem_falloc->next)
1614	shmem_falloc->nr_unswapped += nr_pages;
1615	else
1616	shmem_falloc = NULL;
1617	spin_unlock(lock: &inode->i_lock);
1618	if (shmem_falloc)
1619	goto redirty;
1620	}
1621	folio_zero_range(folio, start: 0, length: folio_size(folio));
1622	flush_dcache_folio(folio);
1623	folio_mark_uptodate(folio);
1624	}
1625
1626	/*
1627	* Add inode to shmem_unuse()'s list of swapped-out inodes,
1628	* if it's not already there. Do it now before the folio is
1629	* moved to swap cache, when its pagelock no longer protects
1630	* the inode from eviction. But don't unlock the mutex until
1631	* we've incremented swapped, because shmem_unuse_inode() will
1632	* prune a !swapped inode from the swaplist under this mutex.
1633	*/
1634	mutex_lock(&shmem_swaplist_mutex);
1635	if (list_empty(head: &info->swaplist))
1636	list_add(new: &info->swaplist, head: &shmem_swaplist);
1637
1638	if (!folio_alloc_swap(folio, __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
1639	shmem_recalc_inode(inode, alloced: 0, swapped: nr_pages);
1640	swap_shmem_alloc(folio->swap, nr_pages);
1641	shmem_delete_from_page_cache(folio, radswap: swp_to_radix_entry(entry: folio->swap));
1642
1643	mutex_unlock(lock: &shmem_swaplist_mutex);
1644	BUG_ON(folio_mapped(folio));
1645	return swap_writeout(folio, wbc);
1646	}
1647	if (!info->swapped)
1648	list_del_init(entry: &info->swaplist);
1649	mutex_unlock(lock: &shmem_swaplist_mutex);
1650	if (nr_pages > 1)
1651	goto try_split;
1652	redirty:
1653	folio_mark_dirty(folio);
1654	if (wbc->for_reclaim)
1655	return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */
1656	folio_unlock(folio);
1657	return 0;
1658	}
1659	EXPORT_SYMBOL_GPL(shmem_writeout);
1660
1661	#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1662	static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1663	{
1664	char buffer[64];
1665
1666	if (!mpol || mpol->mode == MPOL_DEFAULT)
1667	return; /* show nothing */
1668
1669	mpol_to_str(buffer, maxlen: sizeof(buffer), pol: mpol);
1670
1671	seq_printf(m: seq, fmt: ",mpol=%s", buffer);
1672	}
1673
1674	static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1675	{
1676	struct mempolicy *mpol = NULL;
1677	if (sbinfo->mpol) {
1678	raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
1679	mpol = sbinfo->mpol;
1680	mpol_get(pol: mpol);
1681	raw_spin_unlock(&sbinfo->stat_lock);
1682	}
1683	return mpol;
1684	}
1685	#else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1686	static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1687	{
1688	}
1689	static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1690	{
1691	return NULL;
1692	}
1693	#endif /* CONFIG_NUMA && CONFIG_TMPFS */
1694
1695	static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
1696	pgoff_t index, unsigned int order, pgoff_t *ilx);
1697
1698	static struct folio *shmem_swapin_cluster(swp_entry_t swap, gfp_t gfp,
1699	struct shmem_inode_info *info, pgoff_t index)
1700	{
1701	struct mempolicy *mpol;
1702	pgoff_t ilx;
1703	struct folio *folio;
1704
1705	mpol = shmem_get_pgoff_policy(info, index, order: 0, ilx: &ilx);
1706	folio = swap_cluster_readahead(entry: swap, flag: gfp, mpol, ilx);
1707	mpol_cond_put(pol: mpol);
1708
1709	return folio;
1710	}
1711
1712	/*
1713	* Make sure huge_gfp is always more limited than limit_gfp.
1714	* Some of the flags set permissions, while others set limitations.
1715	*/
1716	static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1717	{
1718	gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1719	gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1720	gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1721	gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1722
1723	/* Allow allocations only from the originally specified zones. */
1724	result |= zoneflags;
1725
1726	/*
1727	* Minimize the result gfp by taking the union with the deny flags,
1728	* and the intersection of the allow flags.
1729	*/
1730	result |= (limit_gfp & denyflags);
1731	result |= (huge_gfp & limit_gfp) & allowflags;
1732
1733	return result;
1734	}
1735
1736	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1737	bool shmem_hpage_pmd_enabled(void)
1738	{
1739	if (shmem_huge == SHMEM_HUGE_DENY)
1740	return false;
1741	if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_always))
1742	return true;
1743	if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_madvise))
1744	return true;
1745	if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_within_size))
1746	return true;
1747	if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_inherit) &&
1748	shmem_huge != SHMEM_HUGE_NEVER)
1749	return true;
1750
1751	return false;
1752	}
1753
1754	unsigned long shmem_allowable_huge_orders(struct inode *inode,
1755	struct vm_area_struct *vma, pgoff_t index,
1756	loff_t write_end, bool shmem_huge_force)
1757	{
1758	unsigned long mask = READ_ONCE(huge_shmem_orders_always);
1759	unsigned long within_size_orders = READ_ONCE(huge_shmem_orders_within_size);
1760	unsigned long vm_flags = vma ? vma->vm_flags : 0;
1761	unsigned int global_orders;
1762
1763	if (thp_disabled_by_hw() || (vma && vma_thp_disabled(vma, vm_flags)))
1764	return 0;
1765
1766	global_orders = shmem_huge_global_enabled(inode, index, write_end,
1767	shmem_huge_force, vma, vm_flags);
1768	/* Tmpfs huge pages allocation */
1769	if (!vma || !vma_is_anon_shmem(vma))
1770	return global_orders;
1771
1772	/*
1773	* Following the 'deny' semantics of the top level, force the huge
1774	* option off from all mounts.
1775	*/
1776	if (shmem_huge == SHMEM_HUGE_DENY)
1777	return 0;
1778
1779	/*
1780	* Only allow inherit orders if the top-level value is 'force', which
1781	* means non-PMD sized THP can not override 'huge' mount option now.
1782	*/
1783	if (shmem_huge == SHMEM_HUGE_FORCE)
1784	return READ_ONCE(huge_shmem_orders_inherit);
1785
1786	/* Allow mTHP that will be fully within i_size. */
1787	mask |= shmem_get_orders_within_size(inode, within_size_orders, index, write_end: 0);
1788
1789	if (vm_flags & VM_HUGEPAGE)
1790	mask |= READ_ONCE(huge_shmem_orders_madvise);
1791
1792	if (global_orders > 0)
1793	mask |= READ_ONCE(huge_shmem_orders_inherit);
1794
1795	return THP_ORDERS_ALL_FILE_DEFAULT & mask;
1796	}
1797
1798	static unsigned long shmem_suitable_orders(struct inode *inode, struct vm_fault *vmf,
1799	struct address_space *mapping, pgoff_t index,
1800	unsigned long orders)
1801	{
1802	struct vm_area_struct *vma = vmf ? vmf->vma : NULL;
1803	pgoff_t aligned_index;
1804	unsigned long pages;
1805	int order;
1806
1807	if (vma) {
1808	orders = thp_vma_suitable_orders(vma, addr: vmf->address, orders);
1809	if (!orders)
1810	return 0;
1811	}
1812
1813	/* Find the highest order that can add into the page cache */
1814	order = highest_order(orders);
1815	while (orders) {
1816	pages = 1UL << order;
1817	aligned_index = round_down(index, pages);
1818	/*
1819	* Check for conflict before waiting on a huge allocation.
1820	* Conflict might be that a huge page has just been allocated
1821	* and added to page cache by a racing thread, or that there
1822	* is already at least one small page in the huge extent.
1823	* Be careful to retry when appropriate, but not forever!
1824	* Elsewhere -EEXIST would be the right code, but not here.
1825	*/
1826	if (!xa_find(xa: &mapping->i_pages, index: &aligned_index,
1827	max: aligned_index + pages - 1, XA_PRESENT))
1828	break;
1829	order = next_order(orders: &orders, prev: order);
1830	}
1831
1832	return orders;
1833	}
1834	#else
1835	static unsigned long shmem_suitable_orders(struct inode *inode, struct vm_fault *vmf,
1836	struct address_space *mapping, pgoff_t index,
1837	unsigned long orders)
1838	{
1839	return 0;
1840	}
1841	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1842
1843	static struct folio *shmem_alloc_folio(gfp_t gfp, int order,
1844	struct shmem_inode_info *info, pgoff_t index)
1845	{
1846	struct mempolicy *mpol;
1847	pgoff_t ilx;
1848	struct folio *folio;
1849
1850	mpol = shmem_get_pgoff_policy(info, index, order, ilx: &ilx);
1851	folio = folio_alloc_mpol(gfp, order, mpol, ilx, numa_node_id());
1852	mpol_cond_put(pol: mpol);
1853
1854	return folio;
1855	}
1856
1857	static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf,
1858	gfp_t gfp, struct inode *inode, pgoff_t index,
1859	struct mm_struct *fault_mm, unsigned long orders)
1860	{
1861	struct address_space *mapping = inode->i_mapping;
1862	struct shmem_inode_info *info = SHMEM_I(inode);
1863	unsigned long suitable_orders = 0;
1864	struct folio *folio = NULL;
1865	long pages;
1866	int error, order;
1867
1868	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1869	orders = 0;
1870
1871	if (orders > 0) {
1872	suitable_orders = shmem_suitable_orders(inode, vmf,
1873	mapping, index, orders);
1874
1875	order = highest_order(orders: suitable_orders);
1876	while (suitable_orders) {
1877	pages = 1UL << order;
1878	index = round_down(index, pages);
1879	folio = shmem_alloc_folio(gfp, order, info, index);
1880	if (folio)
1881	goto allocated;
1882
1883	if (pages == HPAGE_PMD_NR)
1884	count_vm_event(item: THP_FILE_FALLBACK);
1885	count_mthp_stat(order, item: MTHP_STAT_SHMEM_FALLBACK);
1886	order = next_order(orders: &suitable_orders, prev: order);
1887	}
1888	} else {
1889	pages = 1;
1890	folio = shmem_alloc_folio(gfp, order: 0, info, index);
1891	}
1892	if (!folio)
1893	return ERR_PTR(error: -ENOMEM);
1894
1895	allocated:
1896	__folio_set_locked(folio);
1897	__folio_set_swapbacked(folio);
1898
1899	gfp &= GFP_RECLAIM_MASK;
1900	error = mem_cgroup_charge(folio, mm: fault_mm, gfp);
1901	if (error) {
1902	if (xa_find(xa: &mapping->i_pages, index: &index,
1903	max: index + pages - 1, XA_PRESENT)) {
1904	error = -EEXIST;
1905	} else if (pages > 1) {
1906	if (pages == HPAGE_PMD_NR) {
1907	count_vm_event(item: THP_FILE_FALLBACK);
1908	count_vm_event(item: THP_FILE_FALLBACK_CHARGE);
1909	}
1910	count_mthp_stat(order: folio_order(folio), item: MTHP_STAT_SHMEM_FALLBACK);
1911	count_mthp_stat(order: folio_order(folio), item: MTHP_STAT_SHMEM_FALLBACK_CHARGE);
1912	}
1913	goto unlock;
1914	}
1915
1916	error = shmem_add_to_page_cache(folio, mapping, index, NULL, gfp);
1917	if (error)
1918	goto unlock;
1919
1920	error = shmem_inode_acct_blocks(inode, pages);
1921	if (error) {
1922	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
1923	long freed;
1924	/*
1925	* Try to reclaim some space by splitting a few
1926	* large folios beyond i_size on the filesystem.
1927	*/
1928	shmem_unused_huge_shrink(sbinfo, NULL, nr_to_free: pages);
1929	/*
1930	* And do a shmem_recalc_inode() to account for freed pages:
1931	* except our folio is there in cache, so not quite balanced.
1932	*/
1933	spin_lock(lock: &info->lock);
1934	freed = pages + info->alloced - info->swapped -
1935	READ_ONCE(mapping->nrpages);
1936	if (freed > 0)
1937	info->alloced -= freed;
1938	spin_unlock(lock: &info->lock);
1939	if (freed > 0)
1940	shmem_inode_unacct_blocks(inode, pages: freed);
1941	error = shmem_inode_acct_blocks(inode, pages);
1942	if (error) {
1943	filemap_remove_folio(folio);
1944	goto unlock;
1945	}
1946	}
1947
1948	shmem_recalc_inode(inode, alloced: pages, swapped: 0);
1949	folio_add_lru(folio);
1950	return folio;
1951
1952	unlock:
1953	folio_unlock(folio);
1954	folio_put(folio);
1955	return ERR_PTR(error);
1956	}
1957
1958	static struct folio *shmem_swap_alloc_folio(struct inode *inode,
1959	struct vm_area_struct *vma, pgoff_t index,
1960	swp_entry_t entry, int order, gfp_t gfp)
1961	{
1962	struct shmem_inode_info *info = SHMEM_I(inode);
1963	struct folio *new;
1964	void *shadow;
1965	int nr_pages;
1966
1967	/*
1968	* We have arrived here because our zones are constrained, so don't
1969	* limit chance of success with further cpuset and node constraints.
1970	*/
1971	gfp &= ~GFP_CONSTRAINT_MASK;
1972	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && order > 0) {
1973	gfp_t huge_gfp = vma_thp_gfp_mask(vma);
1974
1975	gfp = limit_gfp_mask(huge_gfp, limit_gfp: gfp);
1976	}
1977
1978	new = shmem_alloc_folio(gfp, order, info, index);
1979	if (!new)
1980	return ERR_PTR(error: -ENOMEM);
1981
1982	nr_pages = folio_nr_pages(folio: new);
1983	if (mem_cgroup_swapin_charge_folio(folio: new, mm: vma ? vma->vm_mm : NULL,
1984	gfp, entry)) {
1985	folio_put(folio: new);
1986	return ERR_PTR(error: -ENOMEM);
1987	}
1988
1989	/*
1990	* Prevent parallel swapin from proceeding with the swap cache flag.
1991	*
1992	* Of course there is another possible concurrent scenario as well,
1993	* that is to say, the swap cache flag of a large folio has already
1994	* been set by swapcache_prepare(), while another thread may have
1995	* already split the large swap entry stored in the shmem mapping.
1996	* In this case, shmem_add_to_page_cache() will help identify the
1997	* concurrent swapin and return -EEXIST.
1998	*/
1999	if (swapcache_prepare(entry, nr: nr_pages)) {
2000	folio_put(folio: new);
2001	return ERR_PTR(error: -EEXIST);
2002	}
2003
2004	__folio_set_locked(folio: new);
2005	__folio_set_swapbacked(folio: new);
2006	new->swap = entry;
2007
2008	memcg1_swapin(entry, nr_pages);
2009	shadow = get_shadow_from_swap_cache(entry);
2010	if (shadow)
2011	workingset_refault(folio: new, shadow);
2012	folio_add_lru(new);
2013	swap_read_folio(folio: new, NULL);
2014	return new;
2015	}
2016
2017	/*
2018	* When a page is moved from swapcache to shmem filecache (either by the
2019	* usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
2020	* shmem_unuse_inode()), it may have been read in earlier from swap, in
2021	* ignorance of the mapping it belongs to. If that mapping has special
2022	* constraints (like the gma500 GEM driver, which requires RAM below 4GB),
2023	* we may need to copy to a suitable page before moving to filecache.
2024	*
2025	* In a future release, this may well be extended to respect cpuset and
2026	* NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
2027	* but for now it is a simple matter of zone.
2028	*/
2029	static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
2030	{
2031	return folio_zonenum(folio) > gfp_zone(flags: gfp);
2032	}
2033
2034	static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
2035	struct shmem_inode_info *info, pgoff_t index,
2036	struct vm_area_struct *vma)
2037	{
2038	struct folio *new, *old = *foliop;
2039	swp_entry_t entry = old->swap;
2040	struct address_space *swap_mapping = swap_address_space(entry);
2041	pgoff_t swap_index = swap_cache_index(entry);
2042	XA_STATE(xas, &swap_mapping->i_pages, swap_index);
2043	int nr_pages = folio_nr_pages(folio: old);
2044	int error = 0, i;
2045
2046	/*
2047	* We have arrived here because our zones are constrained, so don't
2048	* limit chance of success by further cpuset and node constraints.
2049	*/
2050	gfp &= ~GFP_CONSTRAINT_MASK;
2051	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2052	if (nr_pages > 1) {
2053	gfp_t huge_gfp = vma_thp_gfp_mask(vma);
2054
2055	gfp = limit_gfp_mask(huge_gfp, limit_gfp: gfp);
2056	}
2057	#endif
2058
2059	new = shmem_alloc_folio(gfp, order: folio_order(folio: old), info, index);
2060	if (!new)
2061	return -ENOMEM;
2062
2063	folio_ref_add(folio: new, nr: nr_pages);
2064	folio_copy(dst: new, src: old);
2065	flush_dcache_folio(folio: new);
2066
2067	__folio_set_locked(folio: new);
2068	__folio_set_swapbacked(folio: new);
2069	folio_mark_uptodate(folio: new);
2070	new->swap = entry;
2071	folio_set_swapcache(folio: new);
2072
2073	/* Swap cache still stores N entries instead of a high-order entry */
2074	xa_lock_irq(&swap_mapping->i_pages);
2075	for (i = 0; i < nr_pages; i++) {
2076	void *item = xas_load(&xas);
2077
2078	if (item != old) {
2079	error = -ENOENT;
2080	break;
2081	}
2082
2083	xas_store(&xas, entry: new);
2084	xas_next(xas: &xas);
2085	}
2086	if (!error) {
2087	mem_cgroup_replace_folio(old, new);
2088	shmem_update_stats(folio: new, nr_pages);
2089	shmem_update_stats(folio: old, nr_pages: -nr_pages);
2090	}
2091	xa_unlock_irq(&swap_mapping->i_pages);
2092
2093	if (unlikely(error)) {
2094	/*
2095	* Is this possible? I think not, now that our callers
2096	* check both the swapcache flag and folio->private
2097	* after getting the folio lock; but be defensive.
2098	* Reverse old to newpage for clear and free.
2099	*/
2100	old = new;
2101	} else {
2102	folio_add_lru(new);
2103	*foliop = new;
2104	}
2105
2106	folio_clear_swapcache(folio: old);
2107	old->private = NULL;
2108
2109	folio_unlock(folio: old);
2110	/*
2111	* The old folio are removed from swap cache, drop the 'nr_pages'
2112	* reference, as well as one temporary reference getting from swap
2113	* cache.
2114	*/
2115	folio_put_refs(folio: old, refs: nr_pages + 1);
2116	return error;
2117	}
2118
2119	static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
2120	struct folio *folio, swp_entry_t swap,
2121	bool skip_swapcache)
2122	{
2123	struct address_space *mapping = inode->i_mapping;
2124	swp_entry_t swapin_error;
2125	void *old;
2126	int nr_pages;
2127
2128	swapin_error = make_poisoned_swp_entry();
2129	old = xa_cmpxchg_irq(xa: &mapping->i_pages, index,
2130	old: swp_to_radix_entry(entry: swap),
2131	entry: swp_to_radix_entry(entry: swapin_error), gfp: 0);
2132	if (old != swp_to_radix_entry(entry: swap))
2133	return;
2134
2135	nr_pages = folio_nr_pages(folio);
2136	folio_wait_writeback(folio);
2137	if (!skip_swapcache)
2138	delete_from_swap_cache(folio);
2139	/*
2140	* Don't treat swapin error folio as alloced. Otherwise inode->i_blocks
2141	* won't be 0 when inode is released and thus trigger WARN_ON(i_blocks)
2142	* in shmem_evict_inode().
2143	*/
2144	shmem_recalc_inode(inode, alloced: -nr_pages, swapped: -nr_pages);
2145	swap_free_nr(entry: swap, nr_pages);
2146	}
2147
2148	static int shmem_split_large_entry(struct inode *inode, pgoff_t index,
2149	swp_entry_t swap, gfp_t gfp)
2150	{
2151	struct address_space *mapping = inode->i_mapping;
2152	XA_STATE_ORDER(xas, &mapping->i_pages, index, 0);
2153	int split_order = 0, entry_order;
2154	int i;
2155
2156	/* Convert user data gfp flags to xarray node gfp flags */
2157	gfp &= GFP_RECLAIM_MASK;
2158
2159	for (;;) {
2160	void *old = NULL;
2161	int cur_order;
2162	pgoff_t swap_index;
2163
2164	xas_lock_irq(&xas);
2165	old = xas_load(&xas);
2166	if (!xa_is_value(entry: old) || swp_to_radix_entry(entry: swap) != old) {
2167	xas_set_err(xas: &xas, err: -EEXIST);
2168	goto unlock;
2169	}
2170
2171	entry_order = xas_get_order(xas: &xas);
2172
2173	if (!entry_order)
2174	goto unlock;
2175
2176	/* Try to split large swap entry in pagecache */
2177	cur_order = entry_order;
2178	swap_index = round_down(index, 1 << entry_order);
2179
2180	split_order = xas_try_split_min_order(order: cur_order);
2181
2182	while (cur_order > 0) {
2183	pgoff_t aligned_index =
2184	round_down(index, 1 << cur_order);
2185	pgoff_t swap_offset = aligned_index - swap_index;
2186
2187	xas_set_order(xas: &xas, index, order: split_order);
2188	xas_try_split(xas: &xas, entry: old, order: cur_order);
2189	if (xas_error(xas: &xas))
2190	goto unlock;
2191
2192	/*
2193	* Re-set the swap entry after splitting, and the swap
2194	* offset of the original large entry must be continuous.
2195	*/
2196	for (i = 0; i < 1 << cur_order;
2197	i += (1 << split_order)) {
2198	swp_entry_t tmp;
2199
2200	tmp = swp_entry(type: swp_type(entry: swap),
2201	offset: swp_offset(entry: swap) + swap_offset +
2202	i);
2203	__xa_store(&mapping->i_pages, index: aligned_index + i,
2204	entry: swp_to_radix_entry(entry: tmp), 0);
2205	}
2206	cur_order = split_order;
2207	split_order = xas_try_split_min_order(order: split_order);
2208	}
2209
2210	unlock:
2211	xas_unlock_irq(&xas);
2212
2213	if (!xas_nomem(&xas, gfp))
2214	break;
2215	}
2216
2217	if (xas_error(xas: &xas))
2218	return xas_error(xas: &xas);
2219
2220	return entry_order;
2221	}
2222
2223	/*
2224	* Swap in the folio pointed to by *foliop.
2225	* Caller has to make sure that *foliop contains a valid swapped folio.
2226	* Returns 0 and the folio in foliop if success. On failure, returns the
2227	* error code and NULL in *foliop.
2228	*/
2229	static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
2230	struct folio **foliop, enum sgp_type sgp,
2231	gfp_t gfp, struct vm_area_struct *vma,
2232	vm_fault_t *fault_type)
2233	{
2234	struct address_space *mapping = inode->i_mapping;
2235	struct mm_struct *fault_mm = vma ? vma->vm_mm : NULL;
2236	struct shmem_inode_info *info = SHMEM_I(inode);
2237	struct swap_info_struct *si;
2238	struct folio *folio = NULL;
2239	bool skip_swapcache = false;
2240	swp_entry_t swap;
2241	int error, nr_pages, order, split_order;
2242
2243	VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
2244	swap = radix_to_swp_entry(arg: *foliop);
2245	*foliop = NULL;
2246
2247	if (is_poisoned_swp_entry(entry: swap))
2248	return -EIO;
2249
2250	si = get_swap_device(entry: swap);
2251	if (!si) {
2252	if (!shmem_confirm_swap(mapping, index, swap))
2253	return -EEXIST;
2254	else
2255	return -EINVAL;
2256	}
2257
2258	/* Look it up and read it in.. */
2259	folio = swap_cache_get_folio(entry: swap, NULL, addr: 0);
2260	order = xa_get_order(&mapping->i_pages, index);
2261	if (!folio) {
2262	bool fallback_order0 = false;
2263
2264	/* Or update major stats only when swapin succeeds?? */
2265	if (fault_type) {
2266	*fault_type |= VM_FAULT_MAJOR;
2267	count_vm_event(item: PGMAJFAULT);
2268	count_memcg_event_mm(mm: fault_mm, idx: PGMAJFAULT);
2269	}
2270
2271	/*
2272	* If uffd is active for the vma, we need per-page fault
2273	* fidelity to maintain the uffd semantics, then fallback
2274	* to swapin order-0 folio, as well as for zswap case.
2275	*/
2276	if (order > 0 && ((vma && unlikely(userfaultfd_armed(vma))) ||
2277	!zswap_never_enabled()))
2278	fallback_order0 = true;
2279
2280	/* Skip swapcache for synchronous device. */
2281	if (!fallback_order0 && data_race(si->flags & SWP_SYNCHRONOUS_IO)) {
2282	folio = shmem_swap_alloc_folio(inode, vma, index, entry: swap, order, gfp);
2283	if (!IS_ERR(ptr: folio)) {
2284	skip_swapcache = true;
2285	goto alloced;
2286	}
2287
2288	/*
2289	* Fallback to swapin order-0 folio unless the swap entry
2290	* already exists.
2291	*/
2292	error = PTR_ERR(ptr: folio);
2293	folio = NULL;
2294	if (error == -EEXIST)
2295	goto failed;
2296	}
2297
2298	/*
2299	* Now swap device can only swap in order 0 folio, then we
2300	* should split the large swap entry stored in the pagecache
2301	* if necessary.
2302	*/
2303	split_order = shmem_split_large_entry(inode, index, swap, gfp);
2304	if (split_order < 0) {
2305	error = split_order;
2306	goto failed;
2307	}
2308
2309	/*
2310	* If the large swap entry has already been split, it is
2311	* necessary to recalculate the new swap entry based on
2312	* the old order alignment.
2313	*/
2314	if (split_order > 0) {
2315	pgoff_t offset = index - round_down(index, 1 << split_order);
2316
2317	swap = swp_entry(type: swp_type(entry: swap), offset: swp_offset(entry: swap) + offset);
2318	}
2319
2320	/* Here we actually start the io */
2321	folio = shmem_swapin_cluster(swap, gfp, info, index);
2322	if (!folio) {
2323	error = -ENOMEM;
2324	goto failed;
2325	}
2326	} else if (order != folio_order(folio)) {
2327	/*
2328	* Swap readahead may swap in order 0 folios into swapcache
2329	* asynchronously, while the shmem mapping can still stores
2330	* large swap entries. In such cases, we should split the
2331	* large swap entry to prevent possible data corruption.
2332	*/
2333	split_order = shmem_split_large_entry(inode, index, swap, gfp);
2334	if (split_order < 0) {
2335	folio_put(folio);
2336	folio = NULL;
2337	error = split_order;
2338	goto failed;
2339	}
2340
2341	/*
2342	* If the large swap entry has already been split, it is
2343	* necessary to recalculate the new swap entry based on
2344	* the old order alignment.
2345	*/
2346	if (split_order > 0) {
2347	pgoff_t offset = index - round_down(index, 1 << split_order);
2348
2349	swap = swp_entry(type: swp_type(entry: swap), offset: swp_offset(entry: swap) + offset);
2350	}
2351	}
2352
2353	alloced:
2354	/* We have to do this with folio locked to prevent races */
2355	folio_lock(folio);
2356	if ((!skip_swapcache && !folio_test_swapcache(folio)) ||
2357	folio->swap.val != swap.val ||
2358	!shmem_confirm_swap(mapping, index, swap) ||
2359	xa_get_order(&mapping->i_pages, index) != folio_order(folio)) {
2360	error = -EEXIST;
2361	goto unlock;
2362	}
2363	if (!folio_test_uptodate(folio)) {
2364	error = -EIO;
2365	goto failed;
2366	}
2367	folio_wait_writeback(folio);
2368	nr_pages = folio_nr_pages(folio);
2369
2370	/*
2371	* Some architectures may have to restore extra metadata to the
2372	* folio after reading from swap.
2373	*/
2374	arch_swap_restore(entry: folio_swap(entry: swap, folio), folio);
2375
2376	if (shmem_should_replace_folio(folio, gfp)) {
2377	error = shmem_replace_folio(foliop: &folio, gfp, info, index, vma);
2378	if (error)
2379	goto failed;
2380	}
2381
2382	error = shmem_add_to_page_cache(folio, mapping,
2383	round_down(index, nr_pages),
2384	expected: swp_to_radix_entry(entry: swap), gfp);
2385	if (error)
2386	goto failed;
2387
2388	shmem_recalc_inode(inode, alloced: 0, swapped: -nr_pages);
2389
2390	if (sgp == SGP_WRITE)
2391	folio_mark_accessed(folio);
2392
2393	if (skip_swapcache) {
2394	folio->swap.val = 0;
2395	swapcache_clear(si, entry: swap, nr: nr_pages);
2396	} else {
2397	delete_from_swap_cache(folio);
2398	}
2399	folio_mark_dirty(folio);
2400	swap_free_nr(entry: swap, nr_pages);
2401	put_swap_device(si);
2402
2403	*foliop = folio;
2404	return 0;
2405	failed:
2406	if (!shmem_confirm_swap(mapping, index, swap))
2407	error = -EEXIST;
2408	if (error == -EIO)
2409	shmem_set_folio_swapin_error(inode, index, folio, swap,
2410	skip_swapcache);
2411	unlock:
2412	if (skip_swapcache)
2413	swapcache_clear(si, entry: swap, nr: folio_nr_pages(folio));
2414	if (folio) {
2415	folio_unlock(folio);
2416	folio_put(folio);
2417	}
2418	put_swap_device(si);
2419
2420	return error;
2421	}
2422
2423	/*
2424	* shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
2425	*
2426	* If we allocate a new one we do not mark it dirty. That's up to the
2427	* vm. If we swap it in we mark it dirty since we also free the swap
2428	* entry since a page cannot live in both the swap and page cache.
2429	*
2430	* vmf and fault_type are only supplied by shmem_fault: otherwise they are NULL.
2431	*/
2432	static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
2433	loff_t write_end, struct folio **foliop, enum sgp_type sgp,
2434	gfp_t gfp, struct vm_fault *vmf, vm_fault_t *fault_type)
2435	{
2436	struct vm_area_struct *vma = vmf ? vmf->vma : NULL;
2437	struct mm_struct *fault_mm;
2438	struct folio *folio;
2439	int error;
2440	bool alloced;
2441	unsigned long orders = 0;
2442
2443	if (WARN_ON_ONCE(!shmem_mapping(inode->i_mapping)))
2444	return -EINVAL;
2445
2446	if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
2447	return -EFBIG;
2448	repeat:
2449	if (sgp <= SGP_CACHE &&
2450	((loff_t)index << PAGE_SHIFT) >= i_size_read(inode))
2451	return -EINVAL;
2452
2453	alloced = false;
2454	fault_mm = vma ? vma->vm_mm : NULL;
2455
2456	folio = filemap_get_entry(mapping: inode->i_mapping, index);
2457	if (folio && vma && userfaultfd_minor(vma)) {
2458	if (!xa_is_value(entry: folio))
2459	folio_put(folio);
2460	*fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
2461	return 0;
2462	}
2463
2464	if (xa_is_value(entry: folio)) {
2465	error = shmem_swapin_folio(inode, index, foliop: &folio,
2466	sgp, gfp, vma, fault_type);
2467	if (error == -EEXIST)
2468	goto repeat;
2469
2470	*foliop = folio;
2471	return error;
2472	}
2473
2474	if (folio) {
2475	folio_lock(folio);
2476
2477	/* Has the folio been truncated or swapped out? */
2478	if (unlikely(folio->mapping != inode->i_mapping)) {
2479	folio_unlock(folio);
2480	folio_put(folio);
2481	goto repeat;
2482	}
2483	if (sgp == SGP_WRITE)
2484	folio_mark_accessed(folio);
2485	if (folio_test_uptodate(folio))
2486	goto out;
2487	/* fallocated folio */
2488	if (sgp != SGP_READ)
2489	goto clear;
2490	folio_unlock(folio);
2491	folio_put(folio);
2492	}
2493
2494	/*
2495	* SGP_READ: succeed on hole, with NULL folio, letting caller zero.
2496	* SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
2497	*/
2498	*foliop = NULL;
2499	if (sgp == SGP_READ)
2500	return 0;
2501	if (sgp == SGP_NOALLOC)
2502	return -ENOENT;
2503
2504	/*
2505	* Fast cache lookup and swap lookup did not find it: allocate.
2506	*/
2507
2508	if (vma && userfaultfd_missing(vma)) {
2509	*fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
2510	return 0;
2511	}
2512
2513	/* Find hugepage orders that are allowed for anonymous shmem and tmpfs. */
2514	orders = shmem_allowable_huge_orders(inode, vma, index, write_end, shmem_huge_force: false);
2515	if (orders > 0) {
2516	gfp_t huge_gfp;
2517
2518	huge_gfp = vma_thp_gfp_mask(vma);
2519	huge_gfp = limit_gfp_mask(huge_gfp, limit_gfp: gfp);
2520	folio = shmem_alloc_and_add_folio(vmf, gfp: huge_gfp,
2521	inode, index, fault_mm, orders);
2522	if (!IS_ERR(ptr: folio)) {
2523	if (folio_test_pmd_mappable(folio))
2524	count_vm_event(item: THP_FILE_ALLOC);
2525	count_mthp_stat(order: folio_order(folio), item: MTHP_STAT_SHMEM_ALLOC);
2526	goto alloced;
2527	}
2528	if (PTR_ERR(ptr: folio) == -EEXIST)
2529	goto repeat;
2530	}
2531
2532	folio = shmem_alloc_and_add_folio(vmf, gfp, inode, index, fault_mm, orders: 0);
2533	if (IS_ERR(ptr: folio)) {
2534	error = PTR_ERR(ptr: folio);
2535	if (error == -EEXIST)
2536	goto repeat;
2537	folio = NULL;
2538	goto unlock;
2539	}
2540
2541	alloced:
2542	alloced = true;
2543	if (folio_test_large(folio) &&
2544	DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
2545	folio_next_index(folio)) {
2546	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
2547	struct shmem_inode_info *info = SHMEM_I(inode);
2548	/*
2549	* Part of the large folio is beyond i_size: subject
2550	* to shrink under memory pressure.
2551	*/
2552	spin_lock(lock: &sbinfo->shrinklist_lock);
2553	/*
2554	* _careful to defend against unlocked access to
2555	* ->shrink_list in shmem_unused_huge_shrink()
2556	*/
2557	if (list_empty_careful(head: &info->shrinklist)) {
2558	list_add_tail(new: &info->shrinklist,
2559	head: &sbinfo->shrinklist);
2560	sbinfo->shrinklist_len++;
2561	}
2562	spin_unlock(lock: &sbinfo->shrinklist_lock);
2563	}
2564
2565	if (sgp == SGP_WRITE)
2566	folio_set_referenced(folio);
2567	/*
2568	* Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
2569	*/
2570	if (sgp == SGP_FALLOC)
2571	sgp = SGP_WRITE;
2572	clear:
2573	/*
2574	* Let SGP_WRITE caller clear ends if write does not fill folio;
2575	* but SGP_FALLOC on a folio fallocated earlier must initialize
2576	* it now, lest undo on failure cancel our earlier guarantee.
2577	*/
2578	if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
2579	long i, n = folio_nr_pages(folio);
2580
2581	for (i = 0; i < n; i++)
2582	clear_highpage(folio_page(folio, i));
2583	flush_dcache_folio(folio);
2584	folio_mark_uptodate(folio);
2585	}
2586
2587	/* Perhaps the file has been truncated since we checked */
2588	if (sgp <= SGP_CACHE &&
2589	((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
2590	error = -EINVAL;
2591	goto unlock;
2592	}
2593	out:
2594	*foliop = folio;
2595	return 0;
2596
2597	/*
2598	* Error recovery.
2599	*/
2600	unlock:
2601	if (alloced)
2602	filemap_remove_folio(folio);
2603	shmem_recalc_inode(inode, alloced: 0, swapped: 0);
2604	if (folio) {
2605	folio_unlock(folio);
2606	folio_put(folio);
2607	}
2608	return error;
2609	}
2610
2611	/**
2612	* shmem_get_folio - find, and lock a shmem folio.
2613	* @inode: inode to search
2614	* @index: the page index.
2615	* @write_end: end of a write, could extend inode size
2616	* @foliop: pointer to the folio if found
2617	* @sgp: SGP_* flags to control behavior
2618	*
2619	* Looks up the page cache entry at @inode & @index. If a folio is
2620	* present, it is returned locked with an increased refcount.
2621	*
2622	* If the caller modifies data in the folio, it must call folio_mark_dirty()
2623	* before unlocking the folio to ensure that the folio is not reclaimed.
2624	* There is no need to reserve space before calling folio_mark_dirty().
2625	*
2626	* When no folio is found, the behavior depends on @sgp:
2627	* - for SGP_READ, *@foliop is %NULL and 0 is returned
2628	* - for SGP_NOALLOC, *@foliop is %NULL and -ENOENT is returned
2629	* - for all other flags a new folio is allocated, inserted into the
2630	* page cache and returned locked in @foliop.
2631	*
2632	* Context: May sleep.
2633	* Return: 0 if successful, else a negative error code.
2634	*/
2635	int shmem_get_folio(struct inode *inode, pgoff_t index, loff_t write_end,
2636	struct folio **foliop, enum sgp_type sgp)
2637	{
2638	return shmem_get_folio_gfp(inode, index, write_end, foliop, sgp,
2639	gfp: mapping_gfp_mask(mapping: inode->i_mapping), NULL, NULL);
2640	}
2641	EXPORT_SYMBOL_GPL(shmem_get_folio);
2642
2643	/*
2644	* This is like autoremove_wake_function, but it removes the wait queue
2645	* entry unconditionally - even if something else had already woken the
2646	* target.
2647	*/
2648	static int synchronous_wake_function(wait_queue_entry_t *wait,
2649	unsigned int mode, int sync, void *key)
2650	{
2651	int ret = default_wake_function(wq_entry: wait, mode, flags: sync, key);
2652	list_del_init(entry: &wait->entry);
2653	return ret;
2654	}
2655
2656	/*
2657	* Trinity finds that probing a hole which tmpfs is punching can
2658	* prevent the hole-punch from ever completing: which in turn
2659	* locks writers out with its hold on i_rwsem. So refrain from
2660	* faulting pages into the hole while it's being punched. Although
2661	* shmem_undo_range() does remove the additions, it may be unable to
2662	* keep up, as each new page needs its own unmap_mapping_range() call,
2663	* and the i_mmap tree grows ever slower to scan if new vmas are added.
2664	*
2665	* It does not matter if we sometimes reach this check just before the
2666	* hole-punch begins, so that one fault then races with the punch:
2667	* we just need to make racing faults a rare case.
2668	*
2669	* The implementation below would be much simpler if we just used a
2670	* standard mutex or completion: but we cannot take i_rwsem in fault,
2671	* and bloating every shmem inode for this unlikely case would be sad.
2672	*/
2673	static vm_fault_t shmem_falloc_wait(struct vm_fault *vmf, struct inode *inode)
2674	{
2675	struct shmem_falloc *shmem_falloc;
2676	struct file *fpin = NULL;
2677	vm_fault_t ret = 0;
2678
2679	spin_lock(lock: &inode->i_lock);
2680	shmem_falloc = inode->i_private;
2681	if (shmem_falloc &&
2682	shmem_falloc->waitq &&
2683	vmf->pgoff >= shmem_falloc->start &&
2684	vmf->pgoff < shmem_falloc->next) {
2685	wait_queue_head_t *shmem_falloc_waitq;
2686	DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2687
2688	ret = VM_FAULT_NOPAGE;
2689	fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2690	shmem_falloc_waitq = shmem_falloc->waitq;
2691	prepare_to_wait(wq_head: shmem_falloc_waitq, wq_entry: &shmem_fault_wait,
2692	TASK_UNINTERRUPTIBLE);
2693	spin_unlock(lock: &inode->i_lock);
2694	schedule();
2695
2696	/*
2697	* shmem_falloc_waitq points into the shmem_fallocate()
2698	* stack of the hole-punching task: shmem_falloc_waitq
2699	* is usually invalid by the time we reach here, but
2700	* finish_wait() does not dereference it in that case;
2701	* though i_lock needed lest racing with wake_up_all().
2702	*/
2703	spin_lock(lock: &inode->i_lock);
2704	finish_wait(wq_head: shmem_falloc_waitq, wq_entry: &shmem_fault_wait);
2705	}
2706	spin_unlock(lock: &inode->i_lock);
2707	if (fpin) {
2708	fput(fpin);
2709	ret = VM_FAULT_RETRY;
2710	}
2711	return ret;
2712	}
2713
2714	static vm_fault_t shmem_fault(struct vm_fault *vmf)
2715	{
2716	struct inode *inode = file_inode(f: vmf->vma->vm_file);
2717	gfp_t gfp = mapping_gfp_mask(mapping: inode->i_mapping);
2718	struct folio *folio = NULL;
2719	vm_fault_t ret = 0;
2720	int err;
2721
2722	/*
2723	* Trinity finds that probing a hole which tmpfs is punching can
2724	* prevent the hole-punch from ever completing: noted in i_private.
2725	*/
2726	if (unlikely(inode->i_private)) {
2727	ret = shmem_falloc_wait(vmf, inode);
2728	if (ret)
2729	return ret;
2730	}
2731
2732	WARN_ON_ONCE(vmf->page != NULL);
2733	err = shmem_get_folio_gfp(inode, index: vmf->pgoff, write_end: 0, foliop: &folio, sgp: SGP_CACHE,
2734	gfp, vmf, fault_type: &ret);
2735	if (err)
2736	return vmf_error(err);
2737	if (folio) {
2738	vmf->page = folio_file_page(folio, index: vmf->pgoff);
2739	ret |= VM_FAULT_LOCKED;
2740	}
2741	return ret;
2742	}
2743
2744	unsigned long shmem_get_unmapped_area(struct file *file,
2745	unsigned long uaddr, unsigned long len,
2746	unsigned long pgoff, unsigned long flags)
2747	{
2748	unsigned long addr;
2749	unsigned long offset;
2750	unsigned long inflated_len;
2751	unsigned long inflated_addr;
2752	unsigned long inflated_offset;
2753	unsigned long hpage_size;
2754
2755	if (len > TASK_SIZE)
2756	return -ENOMEM;
2757
2758	addr = mm_get_unmapped_area(current->mm, filp: file, addr: uaddr, len, pgoff,
2759	flags);
2760
2761	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2762	return addr;
2763	if (IS_ERR_VALUE(addr))
2764	return addr;
2765	if (addr & ~PAGE_MASK)
2766	return addr;
2767	if (addr > TASK_SIZE - len)
2768	return addr;
2769
2770	if (shmem_huge == SHMEM_HUGE_DENY)
2771	return addr;
2772	if (flags & MAP_FIXED)
2773	return addr;
2774	/*
2775	* Our priority is to support MAP_SHARED mapped hugely;
2776	* and support MAP_PRIVATE mapped hugely too, until it is COWed.
2777	* But if caller specified an address hint and we allocated area there
2778	* successfully, respect that as before.
2779	*/
2780	if (uaddr == addr)
2781	return addr;
2782
2783	hpage_size = HPAGE_PMD_SIZE;
2784	if (shmem_huge != SHMEM_HUGE_FORCE) {
2785	struct super_block *sb;
2786	unsigned long __maybe_unused hpage_orders;
2787	int order = 0;
2788
2789	if (file) {
2790	VM_BUG_ON(file->f_op != &shmem_file_operations);
2791	sb = file_inode(f: file)->i_sb;
2792	} else {
2793	/*
2794	* Called directly from mm/mmap.c, or drivers/char/mem.c
2795	* for "/dev/zero", to create a shared anonymous object.
2796	*/
2797	if (IS_ERR(ptr: shm_mnt))
2798	return addr;
2799	sb = shm_mnt->mnt_sb;
2800
2801	/*
2802	* Find the highest mTHP order used for anonymous shmem to
2803	* provide a suitable alignment address.
2804	*/
2805	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2806	hpage_orders = READ_ONCE(huge_shmem_orders_always);
2807	hpage_orders |= READ_ONCE(huge_shmem_orders_within_size);
2808	hpage_orders |= READ_ONCE(huge_shmem_orders_madvise);
2809	if (SHMEM_SB(sb)->huge != SHMEM_HUGE_NEVER)
2810	hpage_orders |= READ_ONCE(huge_shmem_orders_inherit);
2811
2812	if (hpage_orders > 0) {
2813	order = highest_order(orders: hpage_orders);
2814	hpage_size = PAGE_SIZE << order;
2815	}
2816	#endif
2817	}
2818	if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER && !order)
2819	return addr;
2820	}
2821
2822	if (len < hpage_size)
2823	return addr;
2824
2825	offset = (pgoff << PAGE_SHIFT) & (hpage_size - 1);
2826	if (offset && offset + len < 2 * hpage_size)
2827	return addr;
2828	if ((addr & (hpage_size - 1)) == offset)
2829	return addr;
2830
2831	inflated_len = len + hpage_size - PAGE_SIZE;
2832	if (inflated_len > TASK_SIZE)
2833	return addr;
2834	if (inflated_len < len)
2835	return addr;
2836
2837	inflated_addr = mm_get_unmapped_area(current->mm, NULL, addr: uaddr,
2838	len: inflated_len, pgoff: 0, flags);
2839	if (IS_ERR_VALUE(inflated_addr))
2840	return addr;
2841	if (inflated_addr & ~PAGE_MASK)
2842	return addr;
2843
2844	inflated_offset = inflated_addr & (hpage_size - 1);
2845	inflated_addr += offset - inflated_offset;
2846	if (inflated_offset > offset)
2847	inflated_addr += hpage_size;
2848
2849	if (inflated_addr > TASK_SIZE - len)
2850	return addr;
2851	return inflated_addr;
2852	}
2853
2854	#ifdef CONFIG_NUMA
2855	static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2856	{
2857	struct inode *inode = file_inode(f: vma->vm_file);
2858	return mpol_set_shared_policy(sp: &SHMEM_I(inode)->policy, vma, mpol);
2859	}
2860
2861	static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2862	unsigned long addr, pgoff_t *ilx)
2863	{
2864	struct inode *inode = file_inode(f: vma->vm_file);
2865	pgoff_t index;
2866
2867	/*
2868	* Bias interleave by inode number to distribute better across nodes;
2869	* but this interface is independent of which page order is used, so
2870	* supplies only that bias, letting caller apply the offset (adjusted
2871	* by page order, as in shmem_get_pgoff_policy() and get_vma_policy()).
2872	*/
2873	*ilx = inode->i_ino;
2874	index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2875	return mpol_shared_policy_lookup(sp: &SHMEM_I(inode)->policy, idx: index);
2876	}
2877
2878	static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
2879	pgoff_t index, unsigned int order, pgoff_t *ilx)
2880	{
2881	struct mempolicy *mpol;
2882
2883	/* Bias interleave by inode number to distribute better across nodes */
2884	*ilx = info->vfs_inode.i_ino + (index >> order);
2885
2886	mpol = mpol_shared_policy_lookup(sp: &info->policy, idx: index);
2887	return mpol ? mpol : get_task_policy(current);
2888	}
2889	#else
2890	static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
2891	pgoff_t index, unsigned int order, pgoff_t *ilx)
2892	{
2893	*ilx = 0;
2894	return NULL;
2895	}
2896	#endif /* CONFIG_NUMA */
2897
2898	int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2899	{
2900	struct inode *inode = file_inode(f: file);
2901	struct shmem_inode_info *info = SHMEM_I(inode);
2902	int retval = -ENOMEM;
2903
2904	/*
2905	* What serializes the accesses to info->flags?
2906	* ipc_lock_object() when called from shmctl_do_lock(),
2907	* no serialization needed when called from shm_destroy().
2908	*/
2909	if (lock && !(info->flags & VM_LOCKED)) {
2910	if (!user_shm_lock(inode->i_size, ucounts))
2911	goto out_nomem;
2912	info->flags |= VM_LOCKED;
2913	mapping_set_unevictable(mapping: file->f_mapping);
2914	}
2915	if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2916	user_shm_unlock(inode->i_size, ucounts);
2917	info->flags &= ~VM_LOCKED;
2918	mapping_clear_unevictable(mapping: file->f_mapping);
2919	}
2920	retval = 0;
2921
2922	out_nomem:
2923	return retval;
2924	}
2925
2926	static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2927	{
2928	struct inode *inode = file_inode(f: file);
2929
2930	file_accessed(file);
2931	/* This is anonymous shared memory if it is unlinked at the time of mmap */
2932	if (inode->i_nlink)
2933	vma->vm_ops = &shmem_vm_ops;
2934	else
2935	vma->vm_ops = &shmem_anon_vm_ops;
2936	return 0;
2937	}
2938
2939	static int shmem_file_open(struct inode *inode, struct file *file)
2940	{
2941	file->f_mode |= FMODE_CAN_ODIRECT;
2942	return generic_file_open(inode, filp: file);
2943	}
2944
2945	#ifdef CONFIG_TMPFS_XATTR
2946	static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2947
2948	#if IS_ENABLED(CONFIG_UNICODE)
2949	/*
2950	* shmem_inode_casefold_flags - Deal with casefold file attribute flag
2951	*
2952	* The casefold file attribute needs some special checks. I can just be added to
2953	* an empty dir, and can't be removed from a non-empty dir.
2954	*/
2955	static int shmem_inode_casefold_flags(struct inode *inode, unsigned int fsflags,
2956	struct dentry *dentry, unsigned int *i_flags)
2957	{
2958	unsigned int old = inode->i_flags;
2959	struct super_block *sb = inode->i_sb;
2960
2961	if (fsflags & FS_CASEFOLD_FL) {
2962	if (!(old & S_CASEFOLD)) {
2963	if (!sb->s_encoding)
2964	return -EOPNOTSUPP;
2965
2966	if (!S_ISDIR(inode->i_mode))
2967	return -ENOTDIR;
2968
2969	if (dentry && !simple_empty(dentry))
2970	return -ENOTEMPTY;
2971	}
2972
2973	*i_flags = *i_flags | S_CASEFOLD;
2974	} else if (old & S_CASEFOLD) {
2975	if (dentry && !simple_empty(dentry))
2976	return -ENOTEMPTY;
2977	}
2978
2979	return 0;
2980	}
2981	#else
2982	static int shmem_inode_casefold_flags(struct inode *inode, unsigned int fsflags,
2983	struct dentry *dentry, unsigned int *i_flags)
2984	{
2985	if (fsflags & FS_CASEFOLD_FL)
2986	return -EOPNOTSUPP;
2987
2988	return 0;
2989	}
2990	#endif
2991
2992	/*
2993	* chattr's fsflags are unrelated to extended attributes,
2994	* but tmpfs has chosen to enable them under the same config option.
2995	*/
2996	static int shmem_set_inode_flags(struct inode *inode, unsigned int fsflags, struct dentry *dentry)
2997	{
2998	unsigned int i_flags = 0;
2999	int ret;
3000
3001	ret = shmem_inode_casefold_flags(inode, fsflags, dentry, i_flags: &i_flags);
3002	if (ret)
3003	return ret;
3004
3005	if (fsflags & FS_NOATIME_FL)
3006	i_flags |= S_NOATIME;
3007	if (fsflags & FS_APPEND_FL)
3008	i_flags |= S_APPEND;
3009	if (fsflags & FS_IMMUTABLE_FL)
3010	i_flags |= S_IMMUTABLE;
3011	/*
3012	* But FS_NODUMP_FL does not require any action in i_flags.
3013	*/
3014	inode_set_flags(inode, flags: i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE | S_CASEFOLD);
3015
3016	return 0;
3017	}
3018	#else
3019	static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags, struct dentry *dentry)
3020	{
3021	}
3022	#define shmem_initxattrs NULL
3023	#endif
3024
3025	static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode)
3026	{
3027	return &SHMEM_I(inode)->dir_offsets;
3028	}
3029
3030	static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
3031	struct super_block *sb,
3032	struct inode *dir, umode_t mode,
3033	dev_t dev, unsigned long flags)
3034	{
3035	struct inode *inode;
3036	struct shmem_inode_info *info;
3037	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3038	ino_t ino;
3039	int err;
3040
3041	err = shmem_reserve_inode(sb, inop: &ino);
3042	if (err)
3043	return ERR_PTR(error: err);
3044
3045	inode = new_inode(sb);
3046	if (!inode) {
3047	shmem_free_inode(sb, freed_ispace: 0);
3048	return ERR_PTR(error: -ENOSPC);
3049	}
3050
3051	inode->i_ino = ino;
3052	inode_init_owner(idmap, inode, dir, mode);
3053	inode->i_blocks = 0;
3054	simple_inode_init_ts(inode);
3055	inode->i_generation = get_random_u32();
3056	info = SHMEM_I(inode);
3057	memset(info, 0, (char *)inode - (char *)info);
3058	spin_lock_init(&info->lock);
3059	atomic_set(v: &info->stop_eviction, i: 0);
3060	info->seals = F_SEAL_SEAL;
3061	info->flags = flags & VM_NORESERVE;
3062	info->i_crtime = inode_get_mtime(inode);
3063	info->fsflags = (dir == NULL) ? 0 :
3064	SHMEM_I(inode: dir)->fsflags & SHMEM_FL_INHERITED;
3065	if (info->fsflags)
3066	shmem_set_inode_flags(inode, fsflags: info->fsflags, NULL);
3067	INIT_LIST_HEAD(list: &info->shrinklist);
3068	INIT_LIST_HEAD(list: &info->swaplist);
3069	simple_xattrs_init(xattrs: &info->xattrs);
3070	cache_no_acl(inode);
3071	if (sbinfo->noswap)
3072	mapping_set_unevictable(mapping: inode->i_mapping);
3073
3074	/* Don't consider 'deny' for emergencies and 'force' for testing */
3075	if (sbinfo->huge)
3076	mapping_set_large_folios(mapping: inode->i_mapping);
3077
3078	switch (mode & S_IFMT) {
3079	default:
3080	inode->i_op = &shmem_special_inode_operations;
3081	init_special_inode(inode, mode, dev);
3082	break;
3083	case S_IFREG:
3084	inode->i_mapping->a_ops = &shmem_aops;
3085	inode->i_op = &shmem_inode_operations;
3086	inode->i_fop = &shmem_file_operations;
3087	mpol_shared_policy_init(sp: &info->policy,
3088	mpol: shmem_get_sbmpol(sbinfo));
3089	break;
3090	case S_IFDIR:
3091	inc_nlink(inode);
3092	/* Some things misbehave if size == 0 on a directory */
3093	inode->i_size = 2 * BOGO_DIRENT_SIZE;
3094	inode->i_op = &shmem_dir_inode_operations;
3095	inode->i_fop = &simple_offset_dir_operations;
3096	simple_offset_init(octx: shmem_get_offset_ctx(inode));
3097	break;
3098	case S_IFLNK:
3099	/*
3100	* Must not load anything in the rbtree,
3101	* mpol_free_shared_policy will not be called.
3102	*/
3103	mpol_shared_policy_init(sp: &info->policy, NULL);
3104	break;
3105	}
3106
3107	lockdep_annotate_inode_mutex_key(inode);
3108	return inode;
3109	}
3110
3111	#ifdef CONFIG_TMPFS_QUOTA
3112	static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
3113	struct super_block *sb, struct inode *dir,
3114	umode_t mode, dev_t dev, unsigned long flags)
3115	{
3116	int err;
3117	struct inode *inode;
3118
3119	inode = __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
3120	if (IS_ERR(ptr: inode))
3121	return inode;
3122
3123	err = dquot_initialize(inode);
3124	if (err)
3125	goto errout;
3126
3127	err = dquot_alloc_inode(inode);
3128	if (err) {
3129	dquot_drop(inode);
3130	goto errout;
3131	}
3132	return inode;
3133
3134	errout:
3135	inode->i_flags |= S_NOQUOTA;
3136	iput(inode);
3137	return ERR_PTR(error: err);
3138	}
3139	#else
3140	static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
3141	struct super_block *sb, struct inode *dir,
3142	umode_t mode, dev_t dev, unsigned long flags)
3143	{
3144	return __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
3145	}
3146	#endif /* CONFIG_TMPFS_QUOTA */
3147
3148	#ifdef CONFIG_USERFAULTFD
3149	int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
3150	struct vm_area_struct *dst_vma,
3151	unsigned long dst_addr,
3152	unsigned long src_addr,
3153	uffd_flags_t flags,
3154	struct folio **foliop)
3155	{
3156	struct inode *inode = file_inode(f: dst_vma->vm_file);
3157	struct shmem_inode_info *info = SHMEM_I(inode);
3158	struct address_space *mapping = inode->i_mapping;
3159	gfp_t gfp = mapping_gfp_mask(mapping);
3160	pgoff_t pgoff = linear_page_index(vma: dst_vma, address: dst_addr);
3161	void *page_kaddr;
3162	struct folio *folio;
3163	int ret;
3164	pgoff_t max_off;
3165
3166	if (shmem_inode_acct_blocks(inode, pages: 1)) {
3167	/*
3168	* We may have got a page, returned -ENOENT triggering a retry,
3169	* and now we find ourselves with -ENOMEM. Release the page, to
3170	* avoid a BUG_ON in our caller.
3171	*/
3172	if (unlikely(*foliop)) {
3173	folio_put(folio: *foliop);
3174	*foliop = NULL;
3175	}
3176	return -ENOMEM;
3177	}
3178
3179	if (!*foliop) {
3180	ret = -ENOMEM;
3181	folio = shmem_alloc_folio(gfp, order: 0, info, index: pgoff);
3182	if (!folio)
3183	goto out_unacct_blocks;
3184
3185	if (uffd_flags_mode_is(flags, expected: MFILL_ATOMIC_COPY)) {
3186	page_kaddr = kmap_local_folio(folio, offset: 0);
3187	/*
3188	* The read mmap_lock is held here. Despite the
3189	* mmap_lock being read recursive a deadlock is still
3190	* possible if a writer has taken a lock. For example:
3191	*
3192	* process A thread 1 takes read lock on own mmap_lock
3193	* process A thread 2 calls mmap, blocks taking write lock
3194	* process B thread 1 takes page fault, read lock on own mmap lock
3195	* process B thread 2 calls mmap, blocks taking write lock
3196	* process A thread 1 blocks taking read lock on process B
3197	* process B thread 1 blocks taking read lock on process A
3198	*
3199	* Disable page faults to prevent potential deadlock
3200	* and retry the copy outside the mmap_lock.
3201	*/
3202	pagefault_disable();
3203	ret = copy_from_user(to: page_kaddr,
3204	from: (const void __user *)src_addr,
3205	PAGE_SIZE);
3206	pagefault_enable();
3207	kunmap_local(page_kaddr);
3208
3209	/* fallback to copy_from_user outside mmap_lock */
3210	if (unlikely(ret)) {
3211	*foliop = folio;
3212	ret = -ENOENT;
3213	/* don't free the page */
3214	goto out_unacct_blocks;
3215	}
3216
3217	flush_dcache_folio(folio);
3218	} else { /* ZEROPAGE */
3219	clear_user_highpage(page: &folio->page, vaddr: dst_addr);
3220	}
3221	} else {
3222	folio = *foliop;
3223	VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
3224	*foliop = NULL;
3225	}
3226
3227	VM_BUG_ON(folio_test_locked(folio));
3228	VM_BUG_ON(folio_test_swapbacked(folio));
3229	__folio_set_locked(folio);
3230	__folio_set_swapbacked(folio);
3231	__folio_mark_uptodate(folio);
3232
3233	ret = -EFAULT;
3234	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
3235	if (unlikely(pgoff >= max_off))
3236	goto out_release;
3237
3238	ret = mem_cgroup_charge(folio, mm: dst_vma->vm_mm, gfp);
3239	if (ret)
3240	goto out_release;
3241	ret = shmem_add_to_page_cache(folio, mapping, index: pgoff, NULL, gfp);
3242	if (ret)
3243	goto out_release;
3244
3245	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,
3246	page: &folio->page, newly_allocated: true, flags);
3247	if (ret)
3248	goto out_delete_from_cache;
3249
3250	shmem_recalc_inode(inode, alloced: 1, swapped: 0);
3251	folio_unlock(folio);
3252	return 0;
3253	out_delete_from_cache:
3254	filemap_remove_folio(folio);
3255	out_release:
3256	folio_unlock(folio);
3257	folio_put(folio);
3258	out_unacct_blocks:
3259	shmem_inode_unacct_blocks(inode, pages: 1);
3260	return ret;
3261	}
3262	#endif /* CONFIG_USERFAULTFD */
3263
3264	#ifdef CONFIG_TMPFS
3265	static const struct inode_operations shmem_symlink_inode_operations;
3266	static const struct inode_operations shmem_short_symlink_operations;
3267
3268	static int
3269	shmem_write_begin(struct file *file, struct address_space *mapping,
3270	loff_t pos, unsigned len,
3271	struct folio **foliop, void **fsdata)
3272	{
3273	struct inode *inode = mapping->host;
3274	struct shmem_inode_info *info = SHMEM_I(inode);
3275	pgoff_t index = pos >> PAGE_SHIFT;
3276	struct folio *folio;
3277	int ret = 0;
3278
3279	/* i_rwsem is held by caller */
3280	if (unlikely(info->seals & (F_SEAL_GROW |
3281	F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
3282	if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
3283	return -EPERM;
3284	if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
3285	return -EPERM;
3286	}
3287
3288	ret = shmem_get_folio(inode, index, pos + len, &folio, SGP_WRITE);
3289	if (ret)
3290	return ret;
3291
3292	if (folio_contain_hwpoisoned_page(folio)) {
3293	folio_unlock(folio);
3294	folio_put(folio);
3295	return -EIO;
3296	}
3297
3298	*foliop = folio;
3299	return 0;
3300	}
3301
3302	static int
3303	shmem_write_end(struct file *file, struct address_space *mapping,
3304	loff_t pos, unsigned len, unsigned copied,
3305	struct folio *folio, void *fsdata)
3306	{
3307	struct inode *inode = mapping->host;
3308
3309	if (pos + copied > inode->i_size)
3310	i_size_write(inode, i_size: pos + copied);
3311
3312	if (!folio_test_uptodate(folio)) {
3313	if (copied < folio_size(folio)) {
3314	size_t from = offset_in_folio(folio, pos);
3315	folio_zero_segments(folio, start1: 0, xend1: from,
3316	start2: from + copied, xend2: folio_size(folio));
3317	}
3318	folio_mark_uptodate(folio);
3319	}
3320	folio_mark_dirty(folio);
3321	folio_unlock(folio);
3322	folio_put(folio);
3323
3324	return copied;
3325	}
3326
3327	static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
3328	{
3329	struct file *file = iocb->ki_filp;
3330	struct inode *inode = file_inode(f: file);
3331	struct address_space *mapping = inode->i_mapping;
3332	pgoff_t index;
3333	unsigned long offset;
3334	int error = 0;
3335	ssize_t retval = 0;
3336
3337	for (;;) {
3338	struct folio *folio = NULL;
3339	struct page *page = NULL;
3340	unsigned long nr, ret;
3341	loff_t end_offset, i_size = i_size_read(inode);
3342	bool fallback_page_copy = false;
3343	size_t fsize;
3344
3345	if (unlikely(iocb->ki_pos >= i_size))
3346	break;
3347
3348	index = iocb->ki_pos >> PAGE_SHIFT;
3349	error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
3350	if (error) {
3351	if (error == -EINVAL)
3352	error = 0;
3353	break;
3354	}
3355	if (folio) {
3356	folio_unlock(folio);
3357
3358	page = folio_file_page(folio, index);
3359	if (PageHWPoison(page)) {
3360	folio_put(folio);
3361	error = -EIO;
3362	break;
3363	}
3364
3365	if (folio_test_large(folio) &&
3366	folio_test_has_hwpoisoned(folio))
3367	fallback_page_copy = true;
3368	}
3369
3370	/*
3371	* We must evaluate after, since reads (unlike writes)
3372	* are called without i_rwsem protection against truncate
3373	*/
3374	i_size = i_size_read(inode);
3375	if (unlikely(iocb->ki_pos >= i_size)) {
3376	if (folio)
3377	folio_put(folio);
3378	break;
3379	}
3380	end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
3381	if (folio && likely(!fallback_page_copy))
3382	fsize = folio_size(folio);
3383	else
3384	fsize = PAGE_SIZE;
3385	offset = iocb->ki_pos & (fsize - 1);
3386	nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
3387
3388	if (folio) {
3389	/*
3390	* If users can be writing to this page using arbitrary
3391	* virtual addresses, take care about potential aliasing
3392	* before reading the page on the kernel side.
3393	*/
3394	if (mapping_writably_mapped(mapping)) {
3395	if (likely(!fallback_page_copy))
3396	flush_dcache_folio(folio);
3397	else
3398	flush_dcache_page(page);
3399	}
3400
3401	/*
3402	* Mark the folio accessed if we read the beginning.
3403	*/
3404	if (!offset)
3405	folio_mark_accessed(folio);
3406	/*
3407	* Ok, we have the page, and it's up-to-date, so
3408	* now we can copy it to user space...
3409	*/
3410	if (likely(!fallback_page_copy))
3411	ret = copy_folio_to_iter(folio, offset, bytes: nr, i: to);
3412	else
3413	ret = copy_page_to_iter(page, offset, bytes: nr, i: to);
3414	folio_put(folio);
3415	} else if (user_backed_iter(i: to)) {
3416	/*
3417	* Copy to user tends to be so well optimized, but
3418	* clear_user() not so much, that it is noticeably
3419	* faster to copy the zero page instead of clearing.
3420	*/
3421	ret = copy_page_to_iter(ZERO_PAGE(0), offset, bytes: nr, i: to);
3422	} else {
3423	/*
3424	* But submitting the same page twice in a row to
3425	* splice() - or others? - can result in confusion:
3426	* so don't attempt that optimization on pipes etc.
3427	*/
3428	ret = iov_iter_zero(bytes: nr, to);
3429	}
3430
3431	retval += ret;
3432	iocb->ki_pos += ret;
3433
3434	if (!iov_iter_count(i: to))
3435	break;
3436	if (ret < nr) {
3437	error = -EFAULT;
3438	break;
3439	}
3440	cond_resched();
3441	}
3442
3443	file_accessed(file);
3444	return retval ? retval : error;
3445	}
3446
3447	static ssize_t shmem_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3448	{
3449	struct file *file = iocb->ki_filp;
3450	struct inode *inode = file->f_mapping->host;
3451	ssize_t ret;
3452
3453	inode_lock(inode);
3454	ret = generic_write_checks(iocb, from);
3455	if (ret <= 0)
3456	goto unlock;
3457	ret = file_remove_privs(file);
3458	if (ret)
3459	goto unlock;
3460	ret = file_update_time(file);
3461	if (ret)
3462	goto unlock;
3463	ret = generic_perform_write(iocb, from);
3464	unlock:
3465	inode_unlock(inode);
3466	return ret;
3467	}
3468
3469	static bool zero_pipe_buf_get(struct pipe_inode_info *pipe,
3470	struct pipe_buffer *buf)
3471	{
3472	return true;
3473	}
3474
3475	static void zero_pipe_buf_release(struct pipe_inode_info *pipe,
3476	struct pipe_buffer *buf)
3477	{
3478	}
3479
3480	static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe,
3481	struct pipe_buffer *buf)
3482	{
3483	return false;
3484	}
3485
3486	static const struct pipe_buf_operations zero_pipe_buf_ops = {
3487	.release = zero_pipe_buf_release,
3488	.try_steal = zero_pipe_buf_try_steal,
3489	.get = zero_pipe_buf_get,
3490	};
3491
3492	static size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe,
3493	loff_t fpos, size_t size)
3494	{
3495	size_t offset = fpos & ~PAGE_MASK;
3496
3497	size = min_t(size_t, size, PAGE_SIZE - offset);
3498
3499	if (!pipe_is_full(pipe)) {
3500	struct pipe_buffer *buf = pipe_head_buf(pipe);
3501
3502	*buf = (struct pipe_buffer) {
3503	.ops = &zero_pipe_buf_ops,
3504	.page = ZERO_PAGE(0),
3505	.offset = offset,
3506	.len = size,
3507	};
3508	pipe->head++;
3509	}
3510
3511	return size;
3512	}
3513
3514	static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
3515	struct pipe_inode_info *pipe,
3516	size_t len, unsigned int flags)
3517	{
3518	struct inode *inode = file_inode(f: in);
3519	struct address_space *mapping = inode->i_mapping;
3520	struct folio *folio = NULL;
3521	size_t total_spliced = 0, used, npages, n, part;
3522	loff_t isize;
3523	int error = 0;
3524
3525	/* Work out how much data we can actually add into the pipe */
3526	used = pipe_buf_usage(pipe);
3527	npages = max_t(ssize_t, pipe->max_usage - used, 0);
3528	len = min_t(size_t, len, npages * PAGE_SIZE);
3529
3530	do {
3531	bool fallback_page_splice = false;
3532	struct page *page = NULL;
3533	pgoff_t index;
3534	size_t size;
3535
3536	if (*ppos >= i_size_read(inode))
3537	break;
3538
3539	index = *ppos >> PAGE_SHIFT;
3540	error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
3541	if (error) {
3542	if (error == -EINVAL)
3543	error = 0;
3544	break;
3545	}
3546	if (folio) {
3547	folio_unlock(folio);
3548
3549	page = folio_file_page(folio, index);
3550	if (PageHWPoison(page)) {
3551	error = -EIO;
3552	break;
3553	}
3554
3555	if (folio_test_large(folio) &&
3556	folio_test_has_hwpoisoned(folio))
3557	fallback_page_splice = true;
3558	}
3559
3560	/*
3561	* i_size must be checked after we know the pages are Uptodate.
3562	*
3563	* Checking i_size after the check allows us to calculate
3564	* the correct value for "nr", which means the zero-filled
3565	* part of the page is not copied back to userspace (unless
3566	* another truncate extends the file - this is desired though).
3567	*/
3568	isize = i_size_read(inode);
3569	if (unlikely(*ppos >= isize))
3570	break;
3571	/*
3572	* Fallback to PAGE_SIZE splice if the large folio has hwpoisoned
3573	* pages.
3574	*/
3575	size = len;
3576	if (unlikely(fallback_page_splice)) {
3577	size_t offset = *ppos & ~PAGE_MASK;
3578
3579	size = umin(size, PAGE_SIZE - offset);
3580	}
3581	part = min_t(loff_t, isize - *ppos, size);
3582
3583	if (folio) {
3584	/*
3585	* If users can be writing to this page using arbitrary
3586	* virtual addresses, take care about potential aliasing
3587	* before reading the page on the kernel side.
3588	*/
3589	if (mapping_writably_mapped(mapping)) {
3590	if (likely(!fallback_page_splice))
3591	flush_dcache_folio(folio);
3592	else
3593	flush_dcache_page(page);
3594	}
3595	folio_mark_accessed(folio);
3596	/*
3597	* Ok, we have the page, and it's up-to-date, so we can
3598	* now splice it into the pipe.
3599	*/
3600	n = splice_folio_into_pipe(pipe, folio, fpos: *ppos, size: part);
3601	folio_put(folio);
3602	folio = NULL;
3603	} else {
3604	n = splice_zeropage_into_pipe(pipe, fpos: *ppos, size: part);
3605	}
3606
3607	if (!n)
3608	break;
3609	len -= n;
3610	total_spliced += n;
3611	*ppos += n;
3612	in->f_ra.prev_pos = *ppos;
3613	if (pipe_is_full(pipe))
3614	break;
3615
3616	cond_resched();
3617	} while (len);
3618
3619	if (folio)
3620	folio_put(folio);
3621
3622	file_accessed(file: in);
3623	return total_spliced ? total_spliced : error;
3624	}
3625
3626	static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
3627	{
3628	struct address_space *mapping = file->f_mapping;
3629	struct inode *inode = mapping->host;
3630
3631	if (whence != SEEK_DATA && whence != SEEK_HOLE)
3632	return generic_file_llseek_size(file, offset, whence,
3633	MAX_LFS_FILESIZE, eof: i_size_read(inode));
3634	if (offset < 0)
3635	return -ENXIO;
3636
3637	inode_lock(inode);
3638	/* We're holding i_rwsem so we can access i_size directly */
3639	offset = mapping_seek_hole_data(mapping, start: offset, end: inode->i_size, whence);
3640	if (offset >= 0)
3641	offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
3642	inode_unlock(inode);
3643	return offset;
3644	}
3645
3646	static long shmem_fallocate(struct file *file, int mode, loff_t offset,
3647	loff_t len)
3648	{
3649	struct inode *inode = file_inode(f: file);
3650	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
3651	struct shmem_inode_info *info = SHMEM_I(inode);
3652	struct shmem_falloc shmem_falloc;
3653	pgoff_t start, index, end, undo_fallocend;
3654	int error;
3655
3656	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
3657	return -EOPNOTSUPP;
3658
3659	inode_lock(inode);
3660
3661	if (mode & FALLOC_FL_PUNCH_HOLE) {
3662	struct address_space *mapping = file->f_mapping;
3663	loff_t unmap_start = round_up(offset, PAGE_SIZE);
3664	loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
3665	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
3666
3667	/* protected by i_rwsem */
3668	if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
3669	error = -EPERM;
3670	goto out;
3671	}
3672
3673	shmem_falloc.waitq = &shmem_falloc_waitq;
3674	shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
3675	shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
3676	spin_lock(lock: &inode->i_lock);
3677	inode->i_private = &shmem_falloc;
3678	spin_unlock(lock: &inode->i_lock);
3679
3680	if ((u64)unmap_end > (u64)unmap_start)
3681	unmap_mapping_range(mapping, holebegin: unmap_start,
3682	holelen: 1 + unmap_end - unmap_start, even_cows: 0);
3683	shmem_truncate_range(inode, offset, offset + len - 1);
3684	/* No need to unmap again: hole-punching leaves COWed pages */
3685
3686	spin_lock(lock: &inode->i_lock);
3687	inode->i_private = NULL;
3688	wake_up_all(&shmem_falloc_waitq);
3689	WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
3690	spin_unlock(lock: &inode->i_lock);
3691	error = 0;
3692	goto out;
3693	}
3694
3695	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
3696	error = inode_newsize_ok(inode, offset: offset + len);
3697	if (error)
3698	goto out;
3699
3700	if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
3701	error = -EPERM;
3702	goto out;
3703	}
3704
3705	start = offset >> PAGE_SHIFT;
3706	end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
3707	/* Try to avoid a swapstorm if len is impossible to satisfy */
3708	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
3709	error = -ENOSPC;
3710	goto out;
3711	}
3712
3713	shmem_falloc.waitq = NULL;
3714	shmem_falloc.start = start;
3715	shmem_falloc.next = start;
3716	shmem_falloc.nr_falloced = 0;
3717	shmem_falloc.nr_unswapped = 0;
3718	spin_lock(lock: &inode->i_lock);
3719	inode->i_private = &shmem_falloc;
3720	spin_unlock(lock: &inode->i_lock);
3721
3722	/*
3723	* info->fallocend is only relevant when huge pages might be
3724	* involved: to prevent split_huge_page() freeing fallocated
3725	* pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
3726	*/
3727	undo_fallocend = info->fallocend;
3728	if (info->fallocend < end)
3729	info->fallocend = end;
3730
3731	for (index = start; index < end; ) {
3732	struct folio *folio;
3733
3734	/*
3735	* Check for fatal signal so that we abort early in OOM
3736	* situations. We don't want to abort in case of non-fatal
3737	* signals as large fallocate can take noticeable time and
3738	* e.g. periodic timers may result in fallocate constantly
3739	* restarting.
3740	*/
3741	if (fatal_signal_pending(current))
3742	error = -EINTR;
3743	else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
3744	error = -ENOMEM;
3745	else
3746	error = shmem_get_folio(inode, index, offset + len,
3747	&folio, SGP_FALLOC);
3748	if (error) {
3749	info->fallocend = undo_fallocend;
3750	/* Remove the !uptodate folios we added */
3751	if (index > start) {
3752	shmem_undo_range(inode,
3753	lstart: (loff_t)start << PAGE_SHIFT,
3754	lend: ((loff_t)index << PAGE_SHIFT) - 1, unfalloc: true);
3755	}
3756	goto undone;
3757	}
3758
3759	/*
3760	* Here is a more important optimization than it appears:
3761	* a second SGP_FALLOC on the same large folio will clear it,
3762	* making it uptodate and un-undoable if we fail later.
3763	*/
3764	index = folio_next_index(folio);
3765	/* Beware 32-bit wraparound */
3766	if (!index)
3767	index--;
3768
3769	/*
3770	* Inform shmem_writeout() how far we have reached.
3771	* No need for lock or barrier: we have the page lock.
3772	*/
3773	if (!folio_test_uptodate(folio))
3774	shmem_falloc.nr_falloced += index - shmem_falloc.next;
3775	shmem_falloc.next = index;
3776
3777	/*
3778	* If !uptodate, leave it that way so that freeable folios
3779	* can be recognized if we need to rollback on error later.
3780	* But mark it dirty so that memory pressure will swap rather
3781	* than free the folios we are allocating (and SGP_CACHE folios
3782	* might still be clean: we now need to mark those dirty too).
3783	*/
3784	folio_mark_dirty(folio);
3785	folio_unlock(folio);
3786	folio_put(folio);
3787	cond_resched();
3788	}
3789
3790	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
3791	i_size_write(inode, i_size: offset + len);
3792	undone:
3793	spin_lock(lock: &inode->i_lock);
3794	inode->i_private = NULL;
3795	spin_unlock(lock: &inode->i_lock);
3796	out:
3797	if (!error)
3798	file_modified(file);
3799	inode_unlock(inode);
3800	return error;
3801	}
3802
3803	static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
3804	{
3805	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: dentry->d_sb);
3806
3807	buf->f_type = TMPFS_MAGIC;
3808	buf->f_bsize = PAGE_SIZE;
3809	buf->f_namelen = NAME_MAX;
3810	if (sbinfo->max_blocks) {
3811	buf->f_blocks = sbinfo->max_blocks;
3812	buf->f_bavail =
3813	buf->f_bfree = sbinfo->max_blocks -
3814	percpu_counter_sum(fbc: &sbinfo->used_blocks);
3815	}
3816	if (sbinfo->max_inodes) {
3817	buf->f_files = sbinfo->max_inodes;
3818	buf->f_ffree = sbinfo->free_ispace / BOGO_INODE_SIZE;
3819	}
3820	/* else leave those fields 0 like simple_statfs */
3821
3822	buf->f_fsid = uuid_to_fsid(uuid: dentry->d_sb->s_uuid.b);
3823
3824	return 0;
3825	}
3826
3827	/*
3828	* File creation. Allocate an inode, and we're done..
3829	*/
3830	static int
3831	shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
3832	struct dentry *dentry, umode_t mode, dev_t dev)
3833	{
3834	struct inode *inode;
3835	int error;
3836
3837	if (!generic_ci_validate_strict_name(dir, name: &dentry->d_name))
3838	return -EINVAL;
3839
3840	inode = shmem_get_inode(idmap, sb: dir->i_sb, dir, mode, dev, VM_NORESERVE);
3841	if (IS_ERR(ptr: inode))
3842	return PTR_ERR(ptr: inode);
3843
3844	error = simple_acl_create(dir, inode);
3845	if (error)
3846	goto out_iput;
3847	error = security_inode_init_security(inode, dir, qstr: &dentry->d_name,
3848	initxattrs: shmem_initxattrs, NULL);
3849	if (error && error != -EOPNOTSUPP)
3850	goto out_iput;
3851
3852	error = simple_offset_add(octx: shmem_get_offset_ctx(inode: dir), dentry);
3853	if (error)
3854	goto out_iput;
3855
3856	dir->i_size += BOGO_DIRENT_SIZE;
3857	inode_set_mtime_to_ts(inode: dir, ts: inode_set_ctime_current(inode: dir));
3858	inode_inc_iversion(inode: dir);
3859
3860	if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
3861	d_add(dentry, inode);
3862	else
3863	d_instantiate(dentry, inode);
3864
3865	dget(dentry); /* Extra count - pin the dentry in core */
3866	return error;
3867
3868	out_iput:
3869	iput(inode);
3870	return error;
3871	}
3872
3873	static int
3874	shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
3875	struct file *file, umode_t mode)
3876	{
3877	struct inode *inode;
3878	int error;
3879
3880	inode = shmem_get_inode(idmap, sb: dir->i_sb, dir, mode, dev: 0, VM_NORESERVE);
3881	if (IS_ERR(ptr: inode)) {
3882	error = PTR_ERR(ptr: inode);
3883	goto err_out;
3884	}
3885	error = security_inode_init_security(inode, dir, NULL,
3886	initxattrs: shmem_initxattrs, NULL);
3887	if (error && error != -EOPNOTSUPP)
3888	goto out_iput;
3889	error = simple_acl_create(dir, inode);
3890	if (error)
3891	goto out_iput;
3892	d_tmpfile(file, inode);
3893
3894	err_out:
3895	return finish_open_simple(file, error);
3896	out_iput:
3897	iput(inode);
3898	return error;
3899	}
3900
3901	static struct dentry *shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir,
3902	struct dentry *dentry, umode_t mode)
3903	{
3904	int error;
3905
3906	error = shmem_mknod(idmap, dir, dentry, mode: mode | S_IFDIR, dev: 0);
3907	if (error)
3908	return ERR_PTR(error);
3909	inc_nlink(inode: dir);
3910	return NULL;
3911	}
3912
3913	static int shmem_create(struct mnt_idmap *idmap, struct inode *dir,
3914	struct dentry *dentry, umode_t mode, bool excl)
3915	{
3916	return shmem_mknod(idmap, dir, dentry, mode: mode | S_IFREG, dev: 0);
3917	}
3918
3919	/*
3920	* Link a file..
3921	*/
3922	static int shmem_link(struct dentry *old_dentry, struct inode *dir,
3923	struct dentry *dentry)
3924	{
3925	struct inode *inode = d_inode(dentry: old_dentry);
3926	int ret = 0;
3927
3928	/*
3929	* No ordinary (disk based) filesystem counts links as inodes;
3930	* but each new link needs a new dentry, pinning lowmem, and
3931	* tmpfs dentries cannot be pruned until they are unlinked.
3932	* But if an O_TMPFILE file is linked into the tmpfs, the
3933	* first link must skip that, to get the accounting right.
3934	*/
3935	if (inode->i_nlink) {
3936	ret = shmem_reserve_inode(sb: inode->i_sb, NULL);
3937	if (ret)
3938	goto out;
3939	}
3940
3941	ret = simple_offset_add(octx: shmem_get_offset_ctx(inode: dir), dentry);
3942	if (ret) {
3943	if (inode->i_nlink)
3944	shmem_free_inode(sb: inode->i_sb, freed_ispace: 0);
3945	goto out;
3946	}
3947
3948	dir->i_size += BOGO_DIRENT_SIZE;
3949	inode_set_mtime_to_ts(inode: dir,
3950	ts: inode_set_ctime_to_ts(inode: dir, ts: inode_set_ctime_current(inode)));
3951	inode_inc_iversion(inode: dir);
3952	inc_nlink(inode);
3953	ihold(inode); /* New dentry reference */
3954	dget(dentry); /* Extra pinning count for the created dentry */
3955	if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
3956	d_add(dentry, inode);
3957	else
3958	d_instantiate(dentry, inode);
3959	out:
3960	return ret;
3961	}
3962
3963	static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3964	{
3965	struct inode *inode = d_inode(dentry);
3966
3967	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3968	shmem_free_inode(sb: inode->i_sb, freed_ispace: 0);
3969
3970	simple_offset_remove(octx: shmem_get_offset_ctx(inode: dir), dentry);
3971
3972	dir->i_size -= BOGO_DIRENT_SIZE;
3973	inode_set_mtime_to_ts(inode: dir,
3974	ts: inode_set_ctime_to_ts(inode: dir, ts: inode_set_ctime_current(inode)));
3975	inode_inc_iversion(inode: dir);
3976	drop_nlink(inode);
3977	dput(dentry); /* Undo the count from "create" - does all the work */
3978
3979	/*
3980	* For now, VFS can't deal with case-insensitive negative dentries, so
3981	* we invalidate them
3982	*/
3983	if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
3984	d_invalidate(dentry);
3985
3986	return 0;
3987	}
3988
3989	static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3990	{
3991	if (!simple_empty(dentry))
3992	return -ENOTEMPTY;
3993
3994	drop_nlink(inode: d_inode(dentry));
3995	drop_nlink(inode: dir);
3996	return shmem_unlink(dir, dentry);
3997	}
3998
3999	static int shmem_whiteout(struct mnt_idmap *idmap,
4000	struct inode *old_dir, struct dentry *old_dentry)
4001	{
4002	struct dentry *whiteout;
4003	int error;
4004
4005	whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
4006	if (!whiteout)
4007	return -ENOMEM;
4008
4009	error = shmem_mknod(idmap, dir: old_dir, dentry: whiteout,
4010	S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
4011	dput(whiteout);
4012	if (error)
4013	return error;
4014
4015	/*
4016	* Cheat and hash the whiteout while the old dentry is still in
4017	* place, instead of playing games with FS_RENAME_DOES_D_MOVE.
4018	*
4019	* d_lookup() will consistently find one of them at this point,
4020	* not sure which one, but that isn't even important.
4021	*/
4022	d_rehash(whiteout);
4023	return 0;
4024	}
4025
4026	/*
4027	* The VFS layer already does all the dentry stuff for rename,
4028	* we just have to decrement the usage count for the target if
4029	* it exists so that the VFS layer correctly free's it when it
4030	* gets overwritten.
4031	*/
4032	static int shmem_rename2(struct mnt_idmap *idmap,
4033	struct inode *old_dir, struct dentry *old_dentry,
4034	struct inode *new_dir, struct dentry *new_dentry,
4035	unsigned int flags)
4036	{
4037	struct inode *inode = d_inode(dentry: old_dentry);
4038	int they_are_dirs = S_ISDIR(inode->i_mode);
4039	int error;
4040
4041	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
4042	return -EINVAL;
4043
4044	if (flags & RENAME_EXCHANGE)
4045	return simple_offset_rename_exchange(old_dir, old_dentry,
4046	new_dir, new_dentry);
4047
4048	if (!simple_empty(new_dentry))
4049	return -ENOTEMPTY;
4050
4051	if (flags & RENAME_WHITEOUT) {
4052	error = shmem_whiteout(idmap, old_dir, old_dentry);
4053	if (error)
4054	return error;
4055	}
4056
4057	error = simple_offset_rename(old_dir, old_dentry, new_dir, new_dentry);
4058	if (error)
4059	return error;
4060
4061	if (d_really_is_positive(dentry: new_dentry)) {
4062	(void) shmem_unlink(dir: new_dir, dentry: new_dentry);
4063	if (they_are_dirs) {
4064	drop_nlink(inode: d_inode(dentry: new_dentry));
4065	drop_nlink(inode: old_dir);
4066	}
4067	} else if (they_are_dirs) {
4068	drop_nlink(inode: old_dir);
4069	inc_nlink(inode: new_dir);
4070	}
4071
4072	old_dir->i_size -= BOGO_DIRENT_SIZE;
4073	new_dir->i_size += BOGO_DIRENT_SIZE;
4074	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
4075	inode_inc_iversion(inode: old_dir);
4076	inode_inc_iversion(inode: new_dir);
4077	return 0;
4078	}
4079
4080	static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
4081	struct dentry *dentry, const char *symname)
4082	{
4083	int error;
4084	int len;
4085	struct inode *inode;
4086	struct folio *folio;
4087	char *link;
4088
4089	len = strlen(symname) + 1;
4090	if (len > PAGE_SIZE)
4091	return -ENAMETOOLONG;
4092
4093	inode = shmem_get_inode(idmap, sb: dir->i_sb, dir, S_IFLNK | 0777, dev: 0,
4094	VM_NORESERVE);
4095	if (IS_ERR(ptr: inode))
4096	return PTR_ERR(ptr: inode);
4097
4098	error = security_inode_init_security(inode, dir, qstr: &dentry->d_name,
4099	initxattrs: shmem_initxattrs, NULL);
4100	if (error && error != -EOPNOTSUPP)
4101	goto out_iput;
4102
4103	error = simple_offset_add(octx: shmem_get_offset_ctx(inode: dir), dentry);
4104	if (error)
4105	goto out_iput;
4106
4107	inode->i_size = len-1;
4108	if (len <= SHORT_SYMLINK_LEN) {
4109	link = kmemdup(symname, len, GFP_KERNEL);
4110	if (!link) {
4111	error = -ENOMEM;
4112	goto out_remove_offset;
4113	}
4114	inode->i_op = &shmem_short_symlink_operations;
4115	inode_set_cached_link(inode, link, linklen: len - 1);
4116	} else {
4117	inode_nohighmem(inode);
4118	inode->i_mapping->a_ops = &shmem_aops;
4119	error = shmem_get_folio(inode, 0, 0, &folio, SGP_WRITE);
4120	if (error)
4121	goto out_remove_offset;
4122	inode->i_op = &shmem_symlink_inode_operations;
4123	memcpy(folio_address(folio), symname, len);
4124	folio_mark_uptodate(folio);
4125	folio_mark_dirty(folio);
4126	folio_unlock(folio);
4127	folio_put(folio);
4128	}
4129	dir->i_size += BOGO_DIRENT_SIZE;
4130	inode_set_mtime_to_ts(inode: dir, ts: inode_set_ctime_current(inode: dir));
4131	inode_inc_iversion(inode: dir);
4132	if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
4133	d_add(dentry, inode);
4134	else
4135	d_instantiate(dentry, inode);
4136	dget(dentry);
4137	return 0;
4138
4139	out_remove_offset:
4140	simple_offset_remove(octx: shmem_get_offset_ctx(inode: dir), dentry);
4141	out_iput:
4142	iput(inode);
4143	return error;
4144	}
4145
4146	static void shmem_put_link(void *arg)
4147	{
4148	folio_mark_accessed(arg);
4149	folio_put(folio: arg);
4150	}
4151
4152	static const char *shmem_get_link(struct dentry *dentry, struct inode *inode,
4153	struct delayed_call *done)
4154	{
4155	struct folio *folio = NULL;
4156	int error;
4157
4158	if (!dentry) {
4159	folio = filemap_get_folio(mapping: inode->i_mapping, index: 0);
4160	if (IS_ERR(ptr: folio))
4161	return ERR_PTR(error: -ECHILD);
4162	if (PageHWPoison(folio_page(folio, 0)) ||
4163	!folio_test_uptodate(folio)) {
4164	folio_put(folio);
4165	return ERR_PTR(error: -ECHILD);
4166	}
4167	} else {
4168	error = shmem_get_folio(inode, 0, 0, &folio, SGP_READ);
4169	if (error)
4170	return ERR_PTR(error);
4171	if (!folio)
4172	return ERR_PTR(error: -ECHILD);
4173	if (PageHWPoison(folio_page(folio, 0))) {
4174	folio_unlock(folio);
4175	folio_put(folio);
4176	return ERR_PTR(error: -ECHILD);
4177	}
4178	folio_unlock(folio);
4179	}
4180	set_delayed_call(call: done, fn: shmem_put_link, arg: folio);
4181	return folio_address(folio);
4182	}
4183
4184	#ifdef CONFIG_TMPFS_XATTR
4185
4186	static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
4187	{
4188	struct shmem_inode_info *info = SHMEM_I(inode: d_inode(dentry));
4189
4190	fileattr_fill_flags(fa, flags: info->fsflags & SHMEM_FL_USER_VISIBLE);
4191
4192	return 0;
4193	}
4194
4195	static int shmem_fileattr_set(struct mnt_idmap *idmap,
4196	struct dentry *dentry, struct fileattr *fa)
4197	{
4198	struct inode *inode = d_inode(dentry);
4199	struct shmem_inode_info *info = SHMEM_I(inode);
4200	int ret, flags;
4201
4202	if (fileattr_has_fsx(fa))
4203	return -EOPNOTSUPP;
4204	if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
4205	return -EOPNOTSUPP;
4206
4207	flags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
4208	(fa->flags & SHMEM_FL_USER_MODIFIABLE);
4209
4210	ret = shmem_set_inode_flags(inode, fsflags: flags, dentry);
4211
4212	if (ret)
4213	return ret;
4214
4215	info->fsflags = flags;
4216
4217	inode_set_ctime_current(inode);
4218	inode_inc_iversion(inode);
4219	return 0;
4220	}
4221
4222	/*
4223	* Superblocks without xattr inode operations may get some security.* xattr
4224	* support from the LSM "for free". As soon as we have any other xattrs
4225	* like ACLs, we also need to implement the security.* handlers at
4226	* filesystem level, though.
4227	*/
4228
4229	/*
4230	* Callback for security_inode_init_security() for acquiring xattrs.
4231	*/
4232	static int shmem_initxattrs(struct inode *inode,
4233	const struct xattr *xattr_array, void *fs_info)
4234	{
4235	struct shmem_inode_info *info = SHMEM_I(inode);
4236	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
4237	const struct xattr *xattr;
4238	struct simple_xattr *new_xattr;
4239	size_t ispace = 0;
4240	size_t len;
4241
4242	if (sbinfo->max_inodes) {
4243	for (xattr = xattr_array; xattr->name != NULL; xattr++) {
4244	ispace += simple_xattr_space(name: xattr->name,
4245	size: xattr->value_len + XATTR_SECURITY_PREFIX_LEN);
4246	}
4247	if (ispace) {
4248	raw_spin_lock(&sbinfo->stat_lock);
4249	if (sbinfo->free_ispace < ispace)
4250	ispace = 0;
4251	else
4252	sbinfo->free_ispace -= ispace;
4253	raw_spin_unlock(&sbinfo->stat_lock);
4254	if (!ispace)
4255	return -ENOSPC;
4256	}
4257	}
4258
4259	for (xattr = xattr_array; xattr->name != NULL; xattr++) {
4260	new_xattr = simple_xattr_alloc(value: xattr->value, size: xattr->value_len);
4261	if (!new_xattr)
4262	break;
4263
4264	len = strlen(xattr->name) + 1;
4265	new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
4266	GFP_KERNEL_ACCOUNT);
4267	if (!new_xattr->name) {
4268	kvfree(addr: new_xattr);
4269	break;
4270	}
4271
4272	memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
4273	XATTR_SECURITY_PREFIX_LEN);
4274	memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
4275	xattr->name, len);
4276
4277	simple_xattr_add(xattrs: &info->xattrs, new_xattr);
4278	}
4279
4280	if (xattr->name != NULL) {
4281	if (ispace) {
4282	raw_spin_lock(&sbinfo->stat_lock);
4283	sbinfo->free_ispace += ispace;
4284	raw_spin_unlock(&sbinfo->stat_lock);
4285	}
4286	simple_xattrs_free(xattrs: &info->xattrs, NULL);
4287	return -ENOMEM;
4288	}
4289
4290	return 0;
4291	}
4292
4293	static int shmem_xattr_handler_get(const struct xattr_handler *handler,
4294	struct dentry *unused, struct inode *inode,
4295	const char *name, void *buffer, size_t size)
4296	{
4297	struct shmem_inode_info *info = SHMEM_I(inode);
4298
4299	name = xattr_full_name(handler, name);
4300	return simple_xattr_get(xattrs: &info->xattrs, name, buffer, size);
4301	}
4302
4303	static int shmem_xattr_handler_set(const struct xattr_handler *handler,
4304	struct mnt_idmap *idmap,
4305	struct dentry *unused, struct inode *inode,
4306	const char *name, const void *value,
4307	size_t size, int flags)
4308	{
4309	struct shmem_inode_info *info = SHMEM_I(inode);
4310	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: inode->i_sb);
4311	struct simple_xattr *old_xattr;
4312	size_t ispace = 0;
4313
4314	name = xattr_full_name(handler, name);
4315	if (value && sbinfo->max_inodes) {
4316	ispace = simple_xattr_space(name, size);
4317	raw_spin_lock(&sbinfo->stat_lock);
4318	if (sbinfo->free_ispace < ispace)
4319	ispace = 0;
4320	else
4321	sbinfo->free_ispace -= ispace;
4322	raw_spin_unlock(&sbinfo->stat_lock);
4323	if (!ispace)
4324	return -ENOSPC;
4325	}
4326
4327	old_xattr = simple_xattr_set(xattrs: &info->xattrs, name, value, size, flags);
4328	if (!IS_ERR(ptr: old_xattr)) {
4329	ispace = 0;
4330	if (old_xattr && sbinfo->max_inodes)
4331	ispace = simple_xattr_space(name: old_xattr->name,
4332	size: old_xattr->size);
4333	simple_xattr_free(xattr: old_xattr);
4334	old_xattr = NULL;
4335	inode_set_ctime_current(inode);
4336	inode_inc_iversion(inode);
4337	}
4338	if (ispace) {
4339	raw_spin_lock(&sbinfo->stat_lock);
4340	sbinfo->free_ispace += ispace;
4341	raw_spin_unlock(&sbinfo->stat_lock);
4342	}
4343	return PTR_ERR(ptr: old_xattr);
4344	}
4345
4346	static const struct xattr_handler shmem_security_xattr_handler = {
4347	.prefix = XATTR_SECURITY_PREFIX,
4348	.get = shmem_xattr_handler_get,
4349	.set = shmem_xattr_handler_set,
4350	};
4351
4352	static const struct xattr_handler shmem_trusted_xattr_handler = {
4353	.prefix = XATTR_TRUSTED_PREFIX,
4354	.get = shmem_xattr_handler_get,
4355	.set = shmem_xattr_handler_set,
4356	};
4357
4358	static const struct xattr_handler shmem_user_xattr_handler = {
4359	.prefix = XATTR_USER_PREFIX,
4360	.get = shmem_xattr_handler_get,
4361	.set = shmem_xattr_handler_set,
4362	};
4363
4364	static const struct xattr_handler * const shmem_xattr_handlers[] = {
4365	&shmem_security_xattr_handler,
4366	&shmem_trusted_xattr_handler,
4367	&shmem_user_xattr_handler,
4368	NULL
4369	};
4370
4371	static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
4372	{
4373	struct shmem_inode_info *info = SHMEM_I(inode: d_inode(dentry));
4374	return simple_xattr_list(inode: d_inode(dentry), xattrs: &info->xattrs, buffer, size);
4375	}
4376	#endif /* CONFIG_TMPFS_XATTR */
4377
4378	static const struct inode_operations shmem_short_symlink_operations = {
4379	.getattr = shmem_getattr,
4380	.setattr = shmem_setattr,
4381	.get_link = simple_get_link,
4382	#ifdef CONFIG_TMPFS_XATTR
4383	.listxattr = shmem_listxattr,
4384	#endif
4385	};
4386
4387	static const struct inode_operations shmem_symlink_inode_operations = {
4388	.getattr = shmem_getattr,
4389	.setattr = shmem_setattr,
4390	.get_link = shmem_get_link,
4391	#ifdef CONFIG_TMPFS_XATTR
4392	.listxattr = shmem_listxattr,
4393	#endif
4394	};
4395
4396	static struct dentry *shmem_get_parent(struct dentry *child)
4397	{
4398	return ERR_PTR(error: -ESTALE);
4399	}
4400
4401	static int shmem_match(struct inode *ino, void *vfh)
4402	{
4403	__u32 *fh = vfh;
4404	__u64 inum = fh[2];
4405	inum = (inum << 32) | fh[1];
4406	return ino->i_ino == inum && fh[0] == ino->i_generation;
4407	}
4408
4409	/* Find any alias of inode, but prefer a hashed alias */
4410	static struct dentry *shmem_find_alias(struct inode *inode)
4411	{
4412	struct dentry *alias = d_find_alias(inode);
4413
4414	return alias ?: d_find_any_alias(inode);
4415	}
4416
4417	static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
4418	struct fid *fid, int fh_len, int fh_type)
4419	{
4420	struct inode *inode;
4421	struct dentry *dentry = NULL;
4422	u64 inum;
4423
4424	if (fh_len < 3)
4425	return NULL;
4426
4427	inum = fid->raw[2];
4428	inum = (inum << 32) | fid->raw[1];
4429
4430	inode = ilookup5(sb, hashval: (unsigned long)(inum + fid->raw[0]),
4431	test: shmem_match, data: fid->raw);
4432	if (inode) {
4433	dentry = shmem_find_alias(inode);
4434	iput(inode);
4435	}
4436
4437	return dentry;
4438	}
4439
4440	static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
4441	struct inode *parent)
4442	{
4443	if (*len < 3) {
4444	*len = 3;
4445	return FILEID_INVALID;
4446	}
4447
4448	if (inode_unhashed(inode)) {
4449	/* Unfortunately insert_inode_hash is not idempotent,
4450	* so as we hash inodes here rather than at creation
4451	* time, we need a lock to ensure we only try
4452	* to do it once
4453	*/
4454	static DEFINE_SPINLOCK(lock);
4455	spin_lock(lock: &lock);
4456	if (inode_unhashed(inode))
4457	__insert_inode_hash(inode,
4458	hashval: inode->i_ino + inode->i_generation);
4459	spin_unlock(lock: &lock);
4460	}
4461
4462	fh[0] = inode->i_generation;
4463	fh[1] = inode->i_ino;
4464	fh[2] = ((__u64)inode->i_ino) >> 32;
4465
4466	*len = 3;
4467	return 1;
4468	}
4469
4470	static const struct export_operations shmem_export_ops = {
4471	.get_parent = shmem_get_parent,
4472	.encode_fh = shmem_encode_fh,
4473	.fh_to_dentry = shmem_fh_to_dentry,
4474	};
4475
4476	enum shmem_param {
4477	Opt_gid,
4478	Opt_huge,
4479	Opt_mode,
4480	Opt_mpol,
4481	Opt_nr_blocks,
4482	Opt_nr_inodes,
4483	Opt_size,
4484	Opt_uid,
4485	Opt_inode32,
4486	Opt_inode64,
4487	Opt_noswap,
4488	Opt_quota,
4489	Opt_usrquota,
4490	Opt_grpquota,
4491	Opt_usrquota_block_hardlimit,
4492	Opt_usrquota_inode_hardlimit,
4493	Opt_grpquota_block_hardlimit,
4494	Opt_grpquota_inode_hardlimit,
4495	Opt_casefold_version,
4496	Opt_casefold,
4497	Opt_strict_encoding,
4498	};
4499
4500	static const struct constant_table shmem_param_enums_huge[] = {
4501	{"never", SHMEM_HUGE_NEVER },
4502	{"always", SHMEM_HUGE_ALWAYS },
4503	{"within_size", SHMEM_HUGE_WITHIN_SIZE },
4504	{"advise", SHMEM_HUGE_ADVISE },
4505	{}
4506	};
4507
4508	const struct fs_parameter_spec shmem_fs_parameters[] = {
4509	fsparam_gid ("gid", Opt_gid),
4510	fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge),
4511	fsparam_u32oct("mode", Opt_mode),
4512	fsparam_string("mpol", Opt_mpol),
4513	fsparam_string("nr_blocks", Opt_nr_blocks),
4514	fsparam_string("nr_inodes", Opt_nr_inodes),
4515	fsparam_string("size", Opt_size),
4516	fsparam_uid ("uid", Opt_uid),
4517	fsparam_flag ("inode32", Opt_inode32),
4518	fsparam_flag ("inode64", Opt_inode64),
4519	fsparam_flag ("noswap", Opt_noswap),
4520	#ifdef CONFIG_TMPFS_QUOTA
4521	fsparam_flag ("quota", Opt_quota),
4522	fsparam_flag ("usrquota", Opt_usrquota),
4523	fsparam_flag ("grpquota", Opt_grpquota),
4524	fsparam_string("usrquota_block_hardlimit", Opt_usrquota_block_hardlimit),
4525	fsparam_string("usrquota_inode_hardlimit", Opt_usrquota_inode_hardlimit),
4526	fsparam_string("grpquota_block_hardlimit", Opt_grpquota_block_hardlimit),
4527	fsparam_string("grpquota_inode_hardlimit", Opt_grpquota_inode_hardlimit),
4528	#endif
4529	fsparam_string("casefold", Opt_casefold_version),
4530	fsparam_flag ("casefold", Opt_casefold),
4531	fsparam_flag ("strict_encoding", Opt_strict_encoding),
4532	{}
4533	};
4534
4535	#if IS_ENABLED(CONFIG_UNICODE)
4536	static int shmem_parse_opt_casefold(struct fs_context *fc, struct fs_parameter *param,
4537	bool latest_version)
4538	{
4539	struct shmem_options *ctx = fc->fs_private;
4540	int version = UTF8_LATEST;
4541	struct unicode_map *encoding;
4542	char *version_str = param->string + 5;
4543
4544	if (!latest_version) {
4545	if (strncmp(param->string, "utf8-", 5))
4546	return invalfc(fc, "Only UTF-8 encodings are supported "
4547	"in the format: utf8-<version number>");
4548
4549	version = utf8_parse_version(version: version_str);
4550	if (version < 0)
4551	return invalfc(fc, "Invalid UTF-8 version: %s", version_str);
4552	}
4553
4554	encoding = utf8_load(version);
4555
4556	if (IS_ERR(ptr: encoding)) {
4557	return invalfc(fc, "Failed loading UTF-8 version: utf8-%u.%u.%u\n",
4558	unicode_major(version), unicode_minor(version),
4559	unicode_rev(version));
4560	}
4561
4562	pr_info("tmpfs: Using encoding : utf8-%u.%u.%u\n",
4563	unicode_major(version), unicode_minor(version), unicode_rev(version));
4564
4565	ctx->encoding = encoding;
4566
4567	return 0;
4568	}
4569	#else
4570	static int shmem_parse_opt_casefold(struct fs_context *fc, struct fs_parameter *param,
4571	bool latest_version)
4572	{
4573	return invalfc(fc, "tmpfs: Kernel not built with CONFIG_UNICODE\n");
4574	}
4575	#endif
4576
4577	static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
4578	{
4579	struct shmem_options *ctx = fc->fs_private;
4580	struct fs_parse_result result;
4581	unsigned long long size;
4582	char *rest;
4583	int opt;
4584	kuid_t kuid;
4585	kgid_t kgid;
4586
4587	opt = fs_parse(fc, desc: shmem_fs_parameters, param, result: &result);
4588	if (opt < 0)
4589	return opt;
4590
4591	switch (opt) {
4592	case Opt_size:
4593	size = memparse(ptr: param->string, retptr: &rest);
4594	if (*rest == '%') {
4595	size <<= PAGE_SHIFT;
4596	size *= totalram_pages();
4597	do_div(size, 100);
4598	rest++;
4599	}
4600	if (*rest)
4601	goto bad_value;
4602	ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
4603	ctx->seen |= SHMEM_SEEN_BLOCKS;
4604	break;
4605	case Opt_nr_blocks:
4606	ctx->blocks = memparse(ptr: param->string, retptr: &rest);
4607	if (*rest || ctx->blocks > LONG_MAX)
4608	goto bad_value;
4609	ctx->seen |= SHMEM_SEEN_BLOCKS;
4610	break;
4611	case Opt_nr_inodes:
4612	ctx->inodes = memparse(ptr: param->string, retptr: &rest);
4613	if (*rest || ctx->inodes > ULONG_MAX / BOGO_INODE_SIZE)
4614	goto bad_value;
4615	ctx->seen |= SHMEM_SEEN_INODES;
4616	break;
4617	case Opt_mode:
4618	ctx->mode = result.uint_32 & 07777;
4619	break;
4620	case Opt_uid:
4621	kuid = result.uid;
4622
4623	/*
4624	* The requested uid must be representable in the
4625	* filesystem's idmapping.
4626	*/
4627	if (!kuid_has_mapping(ns: fc->user_ns, uid: kuid))
4628	goto bad_value;
4629
4630	ctx->uid = kuid;
4631	break;
4632	case Opt_gid:
4633	kgid = result.gid;
4634
4635	/*
4636	* The requested gid must be representable in the
4637	* filesystem's idmapping.
4638	*/
4639	if (!kgid_has_mapping(ns: fc->user_ns, gid: kgid))
4640	goto bad_value;
4641
4642	ctx->gid = kgid;
4643	break;
4644	case Opt_huge:
4645	ctx->huge = result.uint_32;
4646	if (ctx->huge != SHMEM_HUGE_NEVER &&
4647	!(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4648	has_transparent_hugepage()))
4649	goto unsupported_parameter;
4650	ctx->seen |= SHMEM_SEEN_HUGE;
4651	break;
4652	case Opt_mpol:
4653	if (IS_ENABLED(CONFIG_NUMA)) {
4654	mpol_put(pol: ctx->mpol);
4655	ctx->mpol = NULL;
4656	if (mpol_parse_str(str: param->string, mpol: &ctx->mpol))
4657	goto bad_value;
4658	break;
4659	}
4660	goto unsupported_parameter;
4661	case Opt_inode32:
4662	ctx->full_inums = false;
4663	ctx->seen |= SHMEM_SEEN_INUMS;
4664	break;
4665	case Opt_inode64:
4666	if (sizeof(ino_t) < 8) {
4667	return invalfc(fc,
4668	"Cannot use inode64 with <64bit inums in kernel\n");
4669	}
4670	ctx->full_inums = true;
4671	ctx->seen |= SHMEM_SEEN_INUMS;
4672	break;
4673	case Opt_noswap:
4674	if ((fc->user_ns != &init_user_ns) || !capable(CAP_SYS_ADMIN)) {
4675	return invalfc(fc,
4676	"Turning off swap in unprivileged tmpfs mounts unsupported");
4677	}
4678	ctx->noswap = true;
4679	ctx->seen |= SHMEM_SEEN_NOSWAP;
4680	break;
4681	case Opt_quota:
4682	if (fc->user_ns != &init_user_ns)
4683	return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4684	ctx->seen |= SHMEM_SEEN_QUOTA;
4685	ctx->quota_types |= (QTYPE_MASK_USR | QTYPE_MASK_GRP);
4686	break;
4687	case Opt_usrquota:
4688	if (fc->user_ns != &init_user_ns)
4689	return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4690	ctx->seen |= SHMEM_SEEN_QUOTA;
4691	ctx->quota_types |= QTYPE_MASK_USR;
4692	break;
4693	case Opt_grpquota:
4694	if (fc->user_ns != &init_user_ns)
4695	return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4696	ctx->seen |= SHMEM_SEEN_QUOTA;
4697	ctx->quota_types |= QTYPE_MASK_GRP;
4698	break;
4699	case Opt_usrquota_block_hardlimit:
4700	size = memparse(ptr: param->string, retptr: &rest);
4701	if (*rest || !size)
4702	goto bad_value;
4703	if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4704	return invalfc(fc,
4705	"User quota block hardlimit too large.");
4706	ctx->qlimits.usrquota_bhardlimit = size;
4707	break;
4708	case Opt_grpquota_block_hardlimit:
4709	size = memparse(ptr: param->string, retptr: &rest);
4710	if (*rest || !size)
4711	goto bad_value;
4712	if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4713	return invalfc(fc,
4714	"Group quota block hardlimit too large.");
4715	ctx->qlimits.grpquota_bhardlimit = size;
4716	break;
4717	case Opt_usrquota_inode_hardlimit:
4718	size = memparse(ptr: param->string, retptr: &rest);
4719	if (*rest || !size)
4720	goto bad_value;
4721	if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4722	return invalfc(fc,
4723	"User quota inode hardlimit too large.");
4724	ctx->qlimits.usrquota_ihardlimit = size;
4725	break;
4726	case Opt_grpquota_inode_hardlimit:
4727	size = memparse(ptr: param->string, retptr: &rest);
4728	if (*rest || !size)
4729	goto bad_value;
4730	if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4731	return invalfc(fc,
4732	"Group quota inode hardlimit too large.");
4733	ctx->qlimits.grpquota_ihardlimit = size;
4734	break;
4735	case Opt_casefold_version:
4736	return shmem_parse_opt_casefold(fc, param, latest_version: false);
4737	case Opt_casefold:
4738	return shmem_parse_opt_casefold(fc, param, latest_version: true);
4739	case Opt_strict_encoding:
4740	#if IS_ENABLED(CONFIG_UNICODE)
4741	ctx->strict_encoding = true;
4742	break;
4743	#else
4744	return invalfc(fc, "tmpfs: Kernel not built with CONFIG_UNICODE\n");
4745	#endif
4746	}
4747	return 0;
4748
4749	unsupported_parameter:
4750	return invalfc(fc, "Unsupported parameter '%s'", param->key);
4751	bad_value:
4752	return invalfc(fc, "Bad value for '%s'", param->key);
4753	}
4754
4755	static char *shmem_next_opt(char **s)
4756	{
4757	char *sbegin = *s;
4758	char *p;
4759
4760	if (sbegin == NULL)
4761	return NULL;
4762
4763	/*
4764	* NUL-terminate this option: unfortunately,
4765	* mount options form a comma-separated list,
4766	* but mpol's nodelist may also contain commas.
4767	*/
4768	for (;;) {
4769	p = strchr(*s, ',');
4770	if (p == NULL)
4771	break;
4772	*s = p + 1;
4773	if (!isdigit(c: *(p+1))) {
4774	*p = '\0';
4775	return sbegin;
4776	}
4777	}
4778
4779	*s = NULL;
4780	return sbegin;
4781	}
4782
4783	static int shmem_parse_monolithic(struct fs_context *fc, void *data)
4784	{
4785	return vfs_parse_monolithic_sep(fc, data, sep: shmem_next_opt);
4786	}
4787
4788	/*
4789	* Reconfigure a shmem filesystem.
4790	*/
4791	static int shmem_reconfigure(struct fs_context *fc)
4792	{
4793	struct shmem_options *ctx = fc->fs_private;
4794	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: fc->root->d_sb);
4795	unsigned long used_isp;
4796	struct mempolicy *mpol = NULL;
4797	const char *err;
4798
4799	raw_spin_lock(&sbinfo->stat_lock);
4800	used_isp = sbinfo->max_inodes * BOGO_INODE_SIZE - sbinfo->free_ispace;
4801
4802	if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
4803	if (!sbinfo->max_blocks) {
4804	err = "Cannot retroactively limit size";
4805	goto out;
4806	}
4807	if (percpu_counter_compare(fbc: &sbinfo->used_blocks,
4808	rhs: ctx->blocks) > 0) {
4809	err = "Too small a size for current use";
4810	goto out;
4811	}
4812	}
4813	if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
4814	if (!sbinfo->max_inodes) {
4815	err = "Cannot retroactively limit inodes";
4816	goto out;
4817	}
4818	if (ctx->inodes * BOGO_INODE_SIZE < used_isp) {
4819	err = "Too few inodes for current use";
4820	goto out;
4821	}
4822	}
4823
4824	if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
4825	sbinfo->next_ino > UINT_MAX) {
4826	err = "Current inum too high to switch to 32-bit inums";
4827	goto out;
4828	}
4829	if ((ctx->seen & SHMEM_SEEN_NOSWAP) && ctx->noswap && !sbinfo->noswap) {
4830	err = "Cannot disable swap on remount";
4831	goto out;
4832	}
4833	if (!(ctx->seen & SHMEM_SEEN_NOSWAP) && !ctx->noswap && sbinfo->noswap) {
4834	err = "Cannot enable swap on remount if it was disabled on first mount";
4835	goto out;
4836	}
4837
4838	if (ctx->seen & SHMEM_SEEN_QUOTA &&
4839	!sb_any_quota_loaded(sb: fc->root->d_sb)) {
4840	err = "Cannot enable quota on remount";
4841	goto out;
4842	}
4843
4844	#ifdef CONFIG_TMPFS_QUOTA
4845	#define CHANGED_LIMIT(name) \
4846	(ctx->qlimits.name## hardlimit && \
4847	(ctx->qlimits.name## hardlimit != sbinfo->qlimits.name## hardlimit))
4848
4849	if (CHANGED_LIMIT(usrquota_b) || CHANGED_LIMIT(usrquota_i) ||
4850	CHANGED_LIMIT(grpquota_b) || CHANGED_LIMIT(grpquota_i)) {
4851	err = "Cannot change global quota limit on remount";
4852	goto out;
4853	}
4854	#endif /* CONFIG_TMPFS_QUOTA */
4855
4856	if (ctx->seen & SHMEM_SEEN_HUGE)
4857	sbinfo->huge = ctx->huge;
4858	if (ctx->seen & SHMEM_SEEN_INUMS)
4859	sbinfo->full_inums = ctx->full_inums;
4860	if (ctx->seen & SHMEM_SEEN_BLOCKS)
4861	sbinfo->max_blocks = ctx->blocks;
4862	if (ctx->seen & SHMEM_SEEN_INODES) {
4863	sbinfo->max_inodes = ctx->inodes;
4864	sbinfo->free_ispace = ctx->inodes * BOGO_INODE_SIZE - used_isp;
4865	}
4866
4867	/*
4868	* Preserve previous mempolicy unless mpol remount option was specified.
4869	*/
4870	if (ctx->mpol) {
4871	mpol = sbinfo->mpol;
4872	sbinfo->mpol = ctx->mpol; /* transfers initial ref */
4873	ctx->mpol = NULL;
4874	}
4875
4876	if (ctx->noswap)
4877	sbinfo->noswap = true;
4878
4879	raw_spin_unlock(&sbinfo->stat_lock);
4880	mpol_put(pol: mpol);
4881	return 0;
4882	out:
4883	raw_spin_unlock(&sbinfo->stat_lock);
4884	return invalfc(fc, "%s", err);
4885	}
4886
4887	static int shmem_show_options(struct seq_file *seq, struct dentry *root)
4888	{
4889	struct shmem_sb_info *sbinfo = SHMEM_SB(sb: root->d_sb);
4890	struct mempolicy *mpol;
4891
4892	if (sbinfo->max_blocks != shmem_default_max_blocks())
4893	seq_printf(m: seq, fmt: ",size=%luk", K(sbinfo->max_blocks));
4894	if (sbinfo->max_inodes != shmem_default_max_inodes())
4895	seq_printf(m: seq, fmt: ",nr_inodes=%lu", sbinfo->max_inodes);
4896	if (sbinfo->mode != (0777 | S_ISVTX))
4897	seq_printf(m: seq, fmt: ",mode=%03ho", sbinfo->mode);
4898	if (!uid_eq(left: sbinfo->uid, GLOBAL_ROOT_UID))
4899	seq_printf(seq, ",uid=%u",
4900	from_kuid_munged(&init_user_ns, sbinfo->uid));
4901	if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
4902	seq_printf(seq, ",gid=%u",
4903	from_kgid_munged(&init_user_ns, sbinfo->gid));
4904
4905	/*
4906	* Showing inode{64,32} might be useful even if it's the system default,
4907	* since then people don't have to resort to checking both here and
4908	* /proc/config.gz to confirm 64-bit inums were successfully applied
4909	* (which may not even exist if IKCONFIG_PROC isn't enabled).
4910	*
4911	* We hide it when inode64 isn't the default and we are using 32-bit
4912	* inodes, since that probably just means the feature isn't even under
4913	* consideration.
4914	*
4915	* As such:
4916	*
4917	* +-----------------+-----------------+
4918	* | TMPFS_INODE64=y | TMPFS_INODE64=n |
4919	* +------------------+-----------------+-----------------+
4920	* | full_inums=true | show | show |
4921	* | full_inums=false | show | hide |
4922	* +------------------+-----------------+-----------------+
4923	*
4924	*/
4925	if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
4926	seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
4927	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4928	/* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
4929	if (sbinfo->huge)
4930	seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
4931	#endif
4932	mpol = shmem_get_sbmpol(sbinfo);
4933	shmem_show_mpol(seq, mpol);
4934	mpol_put(mpol);
4935	if (sbinfo->noswap)
4936	seq_printf(seq, ",noswap");
4937	#ifdef CONFIG_TMPFS_QUOTA
4938	if (sb_has_quota_active(root->d_sb, USRQUOTA))
4939	seq_printf(seq, ",usrquota");
4940	if (sb_has_quota_active(root->d_sb, GRPQUOTA))
4941	seq_printf(seq, ",grpquota");
4942	if (sbinfo->qlimits.usrquota_bhardlimit)
4943	seq_printf(seq, ",usrquota_block_hardlimit=%lld",
4944	sbinfo->qlimits.usrquota_bhardlimit);
4945	if (sbinfo->qlimits.grpquota_bhardlimit)
4946	seq_printf(seq, ",grpquota_block_hardlimit=%lld",
4947	sbinfo->qlimits.grpquota_bhardlimit);
4948	if (sbinfo->qlimits.usrquota_ihardlimit)
4949	seq_printf(seq, ",usrquota_inode_hardlimit=%lld",
4950	sbinfo->qlimits.usrquota_ihardlimit);
4951	if (sbinfo->qlimits.grpquota_ihardlimit)
4952	seq_printf(seq, ",grpquota_inode_hardlimit=%lld",
4953	sbinfo->qlimits.grpquota_ihardlimit);
4954	#endif
4955	return 0;
4956	}
4957
4958	#endif /* CONFIG_TMPFS */
4959
4960	static void shmem_put_super(struct super_block *sb)
4961	{
4962	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
4963
4964	#if IS_ENABLED(CONFIG_UNICODE)
4965	if (sb->s_encoding)
4966	utf8_unload(um: sb->s_encoding);
4967	#endif
4968
4969	#ifdef CONFIG_TMPFS_QUOTA
4970	shmem_disable_quotas(sb);
4971	#endif
4972	free_percpu(pdata: sbinfo->ino_batch);
4973	percpu_counter_destroy(fbc: &sbinfo->used_blocks);
4974	mpol_put(pol: sbinfo->mpol);
4975	kfree(objp: sbinfo);
4976	sb->s_fs_info = NULL;
4977	}
4978
4979	#if IS_ENABLED(CONFIG_UNICODE) && defined(CONFIG_TMPFS)
4980	static const struct dentry_operations shmem_ci_dentry_ops = {
4981	.d_hash = generic_ci_d_hash,
4982	.d_compare = generic_ci_d_compare,
4983	.d_delete = always_delete_dentry,
4984	};
4985	#endif
4986
4987	static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
4988	{
4989	struct shmem_options *ctx = fc->fs_private;
4990	struct inode *inode;
4991	struct shmem_sb_info *sbinfo;
4992	int error = -ENOMEM;
4993
4994	/* Round up to L1_CACHE_BYTES to resist false sharing */
4995	sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
4996	L1_CACHE_BYTES), GFP_KERNEL);
4997	if (!sbinfo)
4998	return error;
4999
5000	sb->s_fs_info = sbinfo;
5001
5002	#ifdef CONFIG_TMPFS
5003	/*
5004	* Per default we only allow half of the physical ram per
5005	* tmpfs instance, limiting inodes to one per page of lowmem;
5006	* but the internal instance is left unlimited.
5007	*/
5008	if (!(sb->s_flags & SB_KERNMOUNT)) {
5009	if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
5010	ctx->blocks = shmem_default_max_blocks();
5011	if (!(ctx->seen & SHMEM_SEEN_INODES))
5012	ctx->inodes = shmem_default_max_inodes();
5013	if (!(ctx->seen & SHMEM_SEEN_INUMS))
5014	ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
5015	sbinfo->noswap = ctx->noswap;
5016	} else {
5017	sb->s_flags |= SB_NOUSER;
5018	}
5019	sb->s_export_op = &shmem_export_ops;
5020	sb->s_flags |= SB_NOSEC | SB_I_VERSION;
5021
5022	#if IS_ENABLED(CONFIG_UNICODE)
5023	if (!ctx->encoding && ctx->strict_encoding) {
5024	pr_err("tmpfs: strict_encoding option without encoding is forbidden\n");
5025	error = -EINVAL;
5026	goto failed;
5027	}
5028
5029	if (ctx->encoding) {
5030	sb->s_encoding = ctx->encoding;
5031	sb->s_d_op = &shmem_ci_dentry_ops;
5032	if (ctx->strict_encoding)
5033	sb->s_encoding_flags = SB_ENC_STRICT_MODE_FL;
5034	}
5035	#endif
5036
5037	#else
5038	sb->s_flags |= SB_NOUSER;
5039	#endif /* CONFIG_TMPFS */
5040	sbinfo->max_blocks = ctx->blocks;
5041	sbinfo->max_inodes = ctx->inodes;
5042	sbinfo->free_ispace = sbinfo->max_inodes * BOGO_INODE_SIZE;
5043	if (sb->s_flags & SB_KERNMOUNT) {
5044	sbinfo->ino_batch = alloc_percpu(ino_t);
5045	if (!sbinfo->ino_batch)
5046	goto failed;
5047	}
5048	sbinfo->uid = ctx->uid;
5049	sbinfo->gid = ctx->gid;
5050	sbinfo->full_inums = ctx->full_inums;
5051	sbinfo->mode = ctx->mode;
5052	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5053	if (ctx->seen & SHMEM_SEEN_HUGE)
5054	sbinfo->huge = ctx->huge;
5055	else
5056	sbinfo->huge = tmpfs_huge;
5057	#endif
5058	sbinfo->mpol = ctx->mpol;
5059	ctx->mpol = NULL;
5060
5061	raw_spin_lock_init(&sbinfo->stat_lock);
5062	if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
5063	goto failed;
5064	spin_lock_init(&sbinfo->shrinklist_lock);
5065	INIT_LIST_HEAD(list: &sbinfo->shrinklist);
5066
5067	sb->s_maxbytes = MAX_LFS_FILESIZE;
5068	sb->s_blocksize = PAGE_SIZE;
5069	sb->s_blocksize_bits = PAGE_SHIFT;
5070	sb->s_magic = TMPFS_MAGIC;
5071	sb->s_op = &shmem_ops;
5072	sb->s_time_gran = 1;
5073	#ifdef CONFIG_TMPFS_XATTR
5074	sb->s_xattr = shmem_xattr_handlers;
5075	#endif
5076	#ifdef CONFIG_TMPFS_POSIX_ACL
5077	sb->s_flags |= SB_POSIXACL;
5078	#endif
5079	uuid_t uuid;
5080	uuid_gen(u: &uuid);
5081	super_set_uuid(sb, uuid: uuid.b, len: sizeof(uuid));
5082
5083	#ifdef CONFIG_TMPFS_QUOTA
5084	if (ctx->seen & SHMEM_SEEN_QUOTA) {
5085	sb->dq_op = &shmem_quota_operations;
5086	sb->s_qcop = &dquot_quotactl_sysfile_ops;
5087	sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP;
5088
5089	/* Copy the default limits from ctx into sbinfo */
5090	memcpy(&sbinfo->qlimits, &ctx->qlimits,
5091	sizeof(struct shmem_quota_limits));
5092
5093	if (shmem_enable_quotas(sb, quota_types: ctx->quota_types))
5094	goto failed;
5095	}
5096	#endif /* CONFIG_TMPFS_QUOTA */
5097
5098	inode = shmem_get_inode(idmap: &nop_mnt_idmap, sb, NULL,
5099	S_IFDIR | sbinfo->mode, dev: 0, VM_NORESERVE);
5100	if (IS_ERR(ptr: inode)) {
5101	error = PTR_ERR(ptr: inode);
5102	goto failed;
5103	}
5104	inode->i_uid = sbinfo->uid;
5105	inode->i_gid = sbinfo->gid;
5106	sb->s_root = d_make_root(inode);
5107	if (!sb->s_root)
5108	goto failed;
5109	return 0;
5110
5111	failed:
5112	shmem_put_super(sb);
5113	return error;
5114	}
5115
5116	static int shmem_get_tree(struct fs_context *fc)
5117	{
5118	return get_tree_nodev(fc, fill_super: shmem_fill_super);
5119	}
5120
5121	static void shmem_free_fc(struct fs_context *fc)
5122	{
5123	struct shmem_options *ctx = fc->fs_private;
5124
5125	if (ctx) {
5126	mpol_put(pol: ctx->mpol);
5127	kfree(objp: ctx);
5128	}
5129	}
5130
5131	static const struct fs_context_operations shmem_fs_context_ops = {
5132	.free = shmem_free_fc,
5133	.get_tree = shmem_get_tree,
5134	#ifdef CONFIG_TMPFS
5135	.parse_monolithic = shmem_parse_monolithic,
5136	.parse_param = shmem_parse_one,
5137	.reconfigure = shmem_reconfigure,
5138	#endif
5139	};
5140
5141	static struct kmem_cache *shmem_inode_cachep __ro_after_init;
5142
5143	static struct inode *shmem_alloc_inode(struct super_block *sb)
5144	{
5145	struct shmem_inode_info *info;
5146	info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
5147	if (!info)
5148	return NULL;
5149	return &info->vfs_inode;
5150	}
5151
5152	static void shmem_free_in_core_inode(struct inode *inode)
5153	{
5154	if (S_ISLNK(inode->i_mode))
5155	kfree(objp: inode->i_link);
5156	kmem_cache_free(s: shmem_inode_cachep, objp: SHMEM_I(inode));
5157	}
5158
5159	static void shmem_destroy_inode(struct inode *inode)
5160	{
5161	if (S_ISREG(inode->i_mode))
5162	mpol_free_shared_policy(sp: &SHMEM_I(inode)->policy);
5163	if (S_ISDIR(inode->i_mode))
5164	simple_offset_destroy(octx: shmem_get_offset_ctx(inode));
5165	}
5166
5167	static void shmem_init_inode(void *foo)
5168	{
5169	struct shmem_inode_info *info = foo;
5170	inode_init_once(&info->vfs_inode);
5171	}
5172
5173	static void __init shmem_init_inodecache(void)
5174	{
5175	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
5176	sizeof(struct shmem_inode_info),
5177	0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
5178	}
5179
5180	static void __init shmem_destroy_inodecache(void)
5181	{
5182	kmem_cache_destroy(s: shmem_inode_cachep);
5183	}
5184
5185	/* Keep the page in page cache instead of truncating it */
5186	static int shmem_error_remove_folio(struct address_space *mapping,
5187	struct folio *folio)
5188	{
5189	return 0;
5190	}
5191
5192	static const struct address_space_operations shmem_aops = {
5193	.dirty_folio = noop_dirty_folio,
5194	#ifdef CONFIG_TMPFS
5195	.write_begin = shmem_write_begin,
5196	.write_end = shmem_write_end,
5197	#endif
5198	#ifdef CONFIG_MIGRATION
5199	.migrate_folio = migrate_folio,
5200	#endif
5201	.error_remove_folio = shmem_error_remove_folio,
5202	};
5203
5204	static const struct file_operations shmem_file_operations = {
5205	.mmap = shmem_mmap,
5206	.open = shmem_file_open,
5207	.get_unmapped_area = shmem_get_unmapped_area,
5208	#ifdef CONFIG_TMPFS
5209	.llseek = shmem_file_llseek,
5210	.read_iter = shmem_file_read_iter,
5211	.write_iter = shmem_file_write_iter,
5212	.fsync = noop_fsync,
5213	.splice_read = shmem_file_splice_read,
5214	.splice_write = iter_file_splice_write,
5215	.fallocate = shmem_fallocate,
5216	#endif
5217	};
5218
5219	static const struct inode_operations shmem_inode_operations = {
5220	.getattr = shmem_getattr,
5221	.setattr = shmem_setattr,
5222	#ifdef CONFIG_TMPFS_XATTR
5223	.listxattr = shmem_listxattr,
5224	.set_acl = simple_set_acl,
5225	.fileattr_get = shmem_fileattr_get,
5226	.fileattr_set = shmem_fileattr_set,
5227	#endif
5228	};
5229
5230	static const struct inode_operations shmem_dir_inode_operations = {
5231	#ifdef CONFIG_TMPFS
5232	.getattr = shmem_getattr,
5233	.create = shmem_create,
5234	.lookup = simple_lookup,
5235	.link = shmem_link,
5236	.unlink = shmem_unlink,
5237	.symlink = shmem_symlink,
5238	.mkdir = shmem_mkdir,
5239	.rmdir = shmem_rmdir,
5240	.mknod = shmem_mknod,
5241	.rename = shmem_rename2,
5242	.tmpfile = shmem_tmpfile,
5243	.get_offset_ctx = shmem_get_offset_ctx,
5244	#endif
5245	#ifdef CONFIG_TMPFS_XATTR
5246	.listxattr = shmem_listxattr,
5247	.fileattr_get = shmem_fileattr_get,
5248	.fileattr_set = shmem_fileattr_set,
5249	#endif
5250	#ifdef CONFIG_TMPFS_POSIX_ACL
5251	.setattr = shmem_setattr,
5252	.set_acl = simple_set_acl,
5253	#endif
5254	};
5255
5256	static const struct inode_operations shmem_special_inode_operations = {
5257	.getattr = shmem_getattr,
5258	#ifdef CONFIG_TMPFS_XATTR
5259	.listxattr = shmem_listxattr,
5260	#endif
5261	#ifdef CONFIG_TMPFS_POSIX_ACL
5262	.setattr = shmem_setattr,
5263	.set_acl = simple_set_acl,
5264	#endif
5265	};
5266
5267	static const struct super_operations shmem_ops = {
5268	.alloc_inode = shmem_alloc_inode,
5269	.free_inode = shmem_free_in_core_inode,
5270	.destroy_inode = shmem_destroy_inode,
5271	#ifdef CONFIG_TMPFS
5272	.statfs = shmem_statfs,
5273	.show_options = shmem_show_options,
5274	#endif
5275	#ifdef CONFIG_TMPFS_QUOTA
5276	.get_dquots = shmem_get_dquots,
5277	#endif
5278	.evict_inode = shmem_evict_inode,
5279	.drop_inode = generic_delete_inode,
5280	.put_super = shmem_put_super,
5281	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5282	.nr_cached_objects = shmem_unused_huge_count,
5283	.free_cached_objects = shmem_unused_huge_scan,
5284	#endif
5285	};
5286
5287	static const struct vm_operations_struct shmem_vm_ops = {
5288	.fault = shmem_fault,
5289	.map_pages = filemap_map_pages,
5290	#ifdef CONFIG_NUMA
5291	.set_policy = shmem_set_policy,
5292	.get_policy = shmem_get_policy,
5293	#endif
5294	};
5295
5296	static const struct vm_operations_struct shmem_anon_vm_ops = {
5297	.fault = shmem_fault,
5298	.map_pages = filemap_map_pages,
5299	#ifdef CONFIG_NUMA
5300	.set_policy = shmem_set_policy,
5301	.get_policy = shmem_get_policy,
5302	#endif
5303	};
5304
5305	int shmem_init_fs_context(struct fs_context *fc)
5306	{
5307	struct shmem_options *ctx;
5308
5309	ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
5310	if (!ctx)
5311	return -ENOMEM;
5312
5313	ctx->mode = 0777 | S_ISVTX;
5314	ctx->uid = current_fsuid();
5315	ctx->gid = current_fsgid();
5316
5317	#if IS_ENABLED(CONFIG_UNICODE)
5318	ctx->encoding = NULL;
5319	#endif
5320
5321	fc->fs_private = ctx;
5322	fc->ops = &shmem_fs_context_ops;
5323	return 0;
5324	}
5325
5326	static struct file_system_type shmem_fs_type = {
5327	.owner = THIS_MODULE,
5328	.name = "tmpfs",
5329	.init_fs_context = shmem_init_fs_context,
5330	#ifdef CONFIG_TMPFS
5331	.parameters = shmem_fs_parameters,
5332	#endif
5333	.kill_sb = kill_litter_super,
5334	.fs_flags = FS_USERNS_MOUNT | FS_ALLOW_IDMAP | FS_MGTIME,
5335	};
5336
5337	#if defined(CONFIG_SYSFS) && defined(CONFIG_TMPFS)
5338
5339	#define __INIT_KOBJ_ATTR(_name, _mode, _show, _store) \
5340	{ \
5341	.attr = { .name = __stringify(_name), .mode = _mode }, \
5342	.show = _show, \
5343	.store = _store, \
5344	}
5345
5346	#define TMPFS_ATTR_W(_name, _store) \
5347	static struct kobj_attribute tmpfs_attr_##_name = \
5348	__INIT_KOBJ_ATTR(_name, 0200, NULL, _store)
5349
5350	#define TMPFS_ATTR_RW(_name, _show, _store) \
5351	static struct kobj_attribute tmpfs_attr_##_name = \
5352	__INIT_KOBJ_ATTR(_name, 0644, _show, _store)
5353
5354	#define TMPFS_ATTR_RO(_name, _show) \
5355	static struct kobj_attribute tmpfs_attr_##_name = \
5356	__INIT_KOBJ_ATTR(_name, 0444, _show, NULL)
5357
5358	#if IS_ENABLED(CONFIG_UNICODE)
5359	static ssize_t casefold_show(struct kobject *kobj, struct kobj_attribute *a,
5360	char *buf)
5361	{
5362	return sysfs_emit(buf, fmt: "supported\n");
5363	}
5364	TMPFS_ATTR_RO(casefold, casefold_show);
5365	#endif
5366
5367	static struct attribute *tmpfs_attributes[] = {
5368	#if IS_ENABLED(CONFIG_UNICODE)
5369	&tmpfs_attr_casefold.attr,
5370	#endif
5371	NULL
5372	};
5373
5374	static const struct attribute_group tmpfs_attribute_group = {
5375	.attrs = tmpfs_attributes,
5376	.name = "features"
5377	};
5378
5379	static struct kobject *tmpfs_kobj;
5380
5381	static int __init tmpfs_sysfs_init(void)
5382	{
5383	int ret;
5384
5385	tmpfs_kobj = kobject_create_and_add(name: "tmpfs", parent: fs_kobj);
5386	if (!tmpfs_kobj)
5387	return -ENOMEM;
5388
5389	ret = sysfs_create_group(kobj: tmpfs_kobj, grp: &tmpfs_attribute_group);
5390	if (ret)
5391	kobject_put(kobj: tmpfs_kobj);
5392
5393	return ret;
5394	}
5395	#endif /* CONFIG_SYSFS && CONFIG_TMPFS */
5396
5397	void __init shmem_init(void)
5398	{
5399	int error;
5400
5401	shmem_init_inodecache();
5402
5403	#ifdef CONFIG_TMPFS_QUOTA
5404	register_quota_format(fmt: &shmem_quota_format);
5405	#endif
5406
5407	error = register_filesystem(&shmem_fs_type);
5408	if (error) {
5409	pr_err("Could not register tmpfs\n");
5410	goto out2;
5411	}
5412
5413	shm_mnt = kern_mount(&shmem_fs_type);
5414	if (IS_ERR(ptr: shm_mnt)) {
5415	error = PTR_ERR(ptr: shm_mnt);
5416	pr_err("Could not kern_mount tmpfs\n");
5417	goto out1;
5418	}
5419
5420	#if defined(CONFIG_SYSFS) && defined(CONFIG_TMPFS)
5421	error = tmpfs_sysfs_init();
5422	if (error) {
5423	pr_err("Could not init tmpfs sysfs\n");
5424	goto out1;
5425	}
5426	#endif
5427
5428	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5429	if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
5430	SHMEM_SB(sb: shm_mnt->mnt_sb)->huge = shmem_huge;
5431	else
5432	shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
5433
5434	/*
5435	* Default to setting PMD-sized THP to inherit the global setting and
5436	* disable all other multi-size THPs.
5437	*/
5438	if (!shmem_orders_configured)
5439	huge_shmem_orders_inherit = BIT(HPAGE_PMD_ORDER);
5440	#endif
5441	return;
5442
5443	out1:
5444	unregister_filesystem(&shmem_fs_type);
5445	out2:
5446	#ifdef CONFIG_TMPFS_QUOTA
5447	unregister_quota_format(fmt: &shmem_quota_format);
5448	#endif
5449	shmem_destroy_inodecache();
5450	shm_mnt = ERR_PTR(error);
5451	}
5452
5453	#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
5454	static ssize_t shmem_enabled_show(struct kobject *kobj,
5455	struct kobj_attribute *attr, char *buf)
5456	{
5457	static const int values[] = {
5458	SHMEM_HUGE_ALWAYS,
5459	SHMEM_HUGE_WITHIN_SIZE,
5460	SHMEM_HUGE_ADVISE,
5461	SHMEM_HUGE_NEVER,
5462	SHMEM_HUGE_DENY,
5463	SHMEM_HUGE_FORCE,
5464	};
5465	int len = 0;
5466	int i;
5467
5468	for (i = 0; i < ARRAY_SIZE(values); i++) {
5469	len += sysfs_emit_at(buf, at: len,
5470	fmt: shmem_huge == values[i] ? "%s[%s]" : "%s%s",
5471	i ? " " : "", shmem_format_huge(huge: values[i]));
5472	}
5473	len += sysfs_emit_at(buf, at: len, fmt: "\n");
5474
5475	return len;
5476	}
5477
5478	static ssize_t shmem_enabled_store(struct kobject *kobj,
5479	struct kobj_attribute *attr, const char *buf, size_t count)
5480	{
5481	char tmp[16];
5482	int huge, err;
5483
5484	if (count + 1 > sizeof(tmp))
5485	return -EINVAL;
5486	memcpy(tmp, buf, count);
5487	tmp[count] = '\0';
5488	if (count && tmp[count - 1] == '\n')
5489	tmp[count - 1] = '\0';
5490
5491	huge = shmem_parse_huge(str: tmp);
5492	if (huge == -EINVAL)
5493	return huge;
5494
5495	shmem_huge = huge;
5496	if (shmem_huge > SHMEM_HUGE_DENY)
5497	SHMEM_SB(sb: shm_mnt->mnt_sb)->huge = shmem_huge;
5498
5499	err = start_stop_khugepaged();
5500	return err ? err : count;
5501	}
5502
5503	struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
5504	static DEFINE_SPINLOCK(huge_shmem_orders_lock);
5505
5506	static ssize_t thpsize_shmem_enabled_show(struct kobject *kobj,
5507	struct kobj_attribute *attr, char *buf)
5508	{
5509	int order = to_thpsize(kobj)->order;
5510	const char *output;
5511
5512	if (test_bit(order, &huge_shmem_orders_always))
5513	output = "[always] inherit within_size advise never";
5514	else if (test_bit(order, &huge_shmem_orders_inherit))
5515	output = "always [inherit] within_size advise never";
5516	else if (test_bit(order, &huge_shmem_orders_within_size))
5517	output = "always inherit [within_size] advise never";
5518	else if (test_bit(order, &huge_shmem_orders_madvise))
5519	output = "always inherit within_size [advise] never";
5520	else
5521	output = "always inherit within_size advise [never]";
5522
5523	return sysfs_emit(buf, fmt: "%s\n", output);
5524	}
5525
5526	static ssize_t thpsize_shmem_enabled_store(struct kobject *kobj,
5527	struct kobj_attribute *attr,
5528	const char *buf, size_t count)
5529	{
5530	int order = to_thpsize(kobj)->order;
5531	ssize_t ret = count;
5532
5533	if (sysfs_streq(s1: buf, s2: "always")) {
5534	spin_lock(lock: &huge_shmem_orders_lock);
5535	clear_bit(nr: order, addr: &huge_shmem_orders_inherit);
5536	clear_bit(nr: order, addr: &huge_shmem_orders_madvise);
5537	clear_bit(nr: order, addr: &huge_shmem_orders_within_size);
5538	set_bit(nr: order, addr: &huge_shmem_orders_always);
5539	spin_unlock(lock: &huge_shmem_orders_lock);
5540	} else if (sysfs_streq(s1: buf, s2: "inherit")) {
5541	/* Do not override huge allocation policy with non-PMD sized mTHP */
5542	if (shmem_huge == SHMEM_HUGE_FORCE &&
5543	order != HPAGE_PMD_ORDER)
5544	return -EINVAL;
5545
5546	spin_lock(lock: &huge_shmem_orders_lock);
5547	clear_bit(nr: order, addr: &huge_shmem_orders_always);
5548	clear_bit(nr: order, addr: &huge_shmem_orders_madvise);
5549	clear_bit(nr: order, addr: &huge_shmem_orders_within_size);
5550	set_bit(nr: order, addr: &huge_shmem_orders_inherit);
5551	spin_unlock(lock: &huge_shmem_orders_lock);
5552	} else if (sysfs_streq(s1: buf, s2: "within_size")) {
5553	spin_lock(lock: &huge_shmem_orders_lock);
5554	clear_bit(nr: order, addr: &huge_shmem_orders_always);
5555	clear_bit(nr: order, addr: &huge_shmem_orders_inherit);
5556	clear_bit(nr: order, addr: &huge_shmem_orders_madvise);
5557	set_bit(nr: order, addr: &huge_shmem_orders_within_size);
5558	spin_unlock(lock: &huge_shmem_orders_lock);
5559	} else if (sysfs_streq(s1: buf, s2: "advise")) {
5560	spin_lock(lock: &huge_shmem_orders_lock);
5561	clear_bit(nr: order, addr: &huge_shmem_orders_always);
5562	clear_bit(nr: order, addr: &huge_shmem_orders_inherit);
5563	clear_bit(nr: order, addr: &huge_shmem_orders_within_size);
5564	set_bit(nr: order, addr: &huge_shmem_orders_madvise);
5565	spin_unlock(lock: &huge_shmem_orders_lock);
5566	} else if (sysfs_streq(s1: buf, s2: "never")) {
5567	spin_lock(lock: &huge_shmem_orders_lock);
5568	clear_bit(nr: order, addr: &huge_shmem_orders_always);
5569	clear_bit(nr: order, addr: &huge_shmem_orders_inherit);
5570	clear_bit(nr: order, addr: &huge_shmem_orders_within_size);
5571	clear_bit(nr: order, addr: &huge_shmem_orders_madvise);
5572	spin_unlock(lock: &huge_shmem_orders_lock);
5573	} else {
5574	ret = -EINVAL;
5575	}
5576
5577	if (ret > 0) {
5578	int err = start_stop_khugepaged();
5579
5580	if (err)
5581	ret = err;
5582	}
5583	return ret;
5584	}
5585
5586	struct kobj_attribute thpsize_shmem_enabled_attr =
5587	__ATTR(shmem_enabled, 0644, thpsize_shmem_enabled_show, thpsize_shmem_enabled_store);
5588	#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
5589
5590	#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
5591
5592	static int __init setup_transparent_hugepage_shmem(char *str)
5593	{
5594	int huge;
5595
5596	huge = shmem_parse_huge(str);
5597	if (huge == -EINVAL) {
5598	pr_warn("transparent_hugepage_shmem= cannot parse, ignored\n");
5599	return huge;
5600	}
5601
5602	shmem_huge = huge;
5603	return 1;
5604	}
5605	__setup("transparent_hugepage_shmem=", setup_transparent_hugepage_shmem);
5606
5607	static int __init setup_transparent_hugepage_tmpfs(char *str)
5608	{
5609	int huge;
5610
5611	huge = shmem_parse_huge(str);
5612	if (huge < 0) {
5613	pr_warn("transparent_hugepage_tmpfs= cannot parse, ignored\n");
5614	return huge;
5615	}
5616
5617	tmpfs_huge = huge;
5618	return 1;
5619	}
5620	__setup("transparent_hugepage_tmpfs=", setup_transparent_hugepage_tmpfs);
5621
5622	static char str_dup[PAGE_SIZE] __initdata;
5623	static int __init setup_thp_shmem(char *str)
5624	{
5625	char *token, *range, *policy, *subtoken;
5626	unsigned long always, inherit, madvise, within_size;
5627	char *start_size, *end_size;
5628	int start, end, nr;
5629	char *p;
5630
5631	if (!str || strlen(str) + 1 > PAGE_SIZE)
5632	goto err;
5633	strscpy(str_dup, str);
5634
5635	always = huge_shmem_orders_always;
5636	inherit = huge_shmem_orders_inherit;
5637	madvise = huge_shmem_orders_madvise;
5638	within_size = huge_shmem_orders_within_size;
5639	p = str_dup;
5640	while ((token = strsep(&p, ";")) != NULL) {
5641	range = strsep(&token, ":");
5642	policy = token;
5643
5644	if (!policy)
5645	goto err;
5646
5647	while ((subtoken = strsep(&range, ",")) != NULL) {
5648	if (strchr(subtoken, '-')) {
5649	start_size = strsep(&subtoken, "-");
5650	end_size = subtoken;
5651
5652	start = get_order_from_str(size_str: start_size,
5653	THP_ORDERS_ALL_FILE_DEFAULT);
5654	end = get_order_from_str(size_str: end_size,
5655	THP_ORDERS_ALL_FILE_DEFAULT);
5656	} else {
5657	start_size = end_size = subtoken;
5658	start = end = get_order_from_str(size_str: subtoken,
5659	THP_ORDERS_ALL_FILE_DEFAULT);
5660	}
5661
5662	if (start < 0) {
5663	pr_err("invalid size %s in thp_shmem boot parameter\n",
5664	start_size);
5665	goto err;
5666	}
5667
5668	if (end < 0) {
5669	pr_err("invalid size %s in thp_shmem boot parameter\n",
5670	end_size);
5671	goto err;
5672	}
5673
5674	if (start > end)
5675	goto err;
5676
5677	nr = end - start + 1;
5678	if (!strcmp(policy, "always")) {
5679	bitmap_set(map: &always, start, nbits: nr);
5680	bitmap_clear(map: &inherit, start, nbits: nr);
5681	bitmap_clear(map: &madvise, start, nbits: nr);
5682	bitmap_clear(map: &within_size, start, nbits: nr);
5683	} else if (!strcmp(policy, "advise")) {
5684	bitmap_set(map: &madvise, start, nbits: nr);
5685	bitmap_clear(map: &inherit, start, nbits: nr);
5686	bitmap_clear(map: &always, start, nbits: nr);
5687	bitmap_clear(map: &within_size, start, nbits: nr);
5688	} else if (!strcmp(policy, "inherit")) {
5689	bitmap_set(map: &inherit, start, nbits: nr);
5690	bitmap_clear(map: &madvise, start, nbits: nr);
5691	bitmap_clear(map: &always, start, nbits: nr);
5692	bitmap_clear(map: &within_size, start, nbits: nr);
5693	} else if (!strcmp(policy, "within_size")) {
5694	bitmap_set(map: &within_size, start, nbits: nr);
5695	bitmap_clear(map: &inherit, start, nbits: nr);
5696	bitmap_clear(map: &madvise, start, nbits: nr);
5697	bitmap_clear(map: &always, start, nbits: nr);
5698	} else if (!strcmp(policy, "never")) {
5699	bitmap_clear(map: &inherit, start, nbits: nr);
5700	bitmap_clear(map: &madvise, start, nbits: nr);
5701	bitmap_clear(map: &always, start, nbits: nr);
5702	bitmap_clear(map: &within_size, start, nbits: nr);
5703	} else {
5704	pr_err("invalid policy %s in thp_shmem boot parameter\n", policy);
5705	goto err;
5706	}
5707	}
5708	}
5709
5710	huge_shmem_orders_always = always;
5711	huge_shmem_orders_madvise = madvise;
5712	huge_shmem_orders_inherit = inherit;
5713	huge_shmem_orders_within_size = within_size;
5714	shmem_orders_configured = true;
5715	return 1;
5716
5717	err:
5718	pr_warn("thp_shmem=%s: error parsing string, ignoring setting\n", str);
5719	return 0;
5720	}
5721	__setup("thp_shmem=", setup_thp_shmem);
5722
5723	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
5724
5725	#else /* !CONFIG_SHMEM */
5726
5727	/*
5728	* tiny-shmem: simple shmemfs and tmpfs using ramfs code
5729	*
5730	* This is intended for small system where the benefits of the full
5731	* shmem code (swap-backed and resource-limited) are outweighed by
5732	* their complexity. On systems without swap this code should be
5733	* effectively equivalent, but much lighter weight.
5734	*/
5735
5736	static struct file_system_type shmem_fs_type = {
5737	.name = "tmpfs",
5738	.init_fs_context = ramfs_init_fs_context,
5739	.parameters = ramfs_fs_parameters,
5740	.kill_sb = ramfs_kill_sb,
5741	.fs_flags = FS_USERNS_MOUNT,
5742	};
5743
5744	void __init shmem_init(void)
5745	{
5746	BUG_ON(register_filesystem(&shmem_fs_type) != 0);
5747
5748	shm_mnt = kern_mount(&shmem_fs_type);
5749	BUG_ON(IS_ERR(shm_mnt));
5750	}
5751
5752	int shmem_unuse(unsigned int type)
5753	{
5754	return 0;
5755	}
5756
5757	int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
5758	{
5759	return 0;
5760	}
5761
5762	void shmem_unlock_mapping(struct address_space *mapping)
5763	{
5764	}
5765
5766	#ifdef CONFIG_MMU
5767	unsigned long shmem_get_unmapped_area(struct file *file,
5768	unsigned long addr, unsigned long len,
5769	unsigned long pgoff, unsigned long flags)
5770	{
5771	return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags);
5772	}
5773	#endif
5774
5775	void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
5776	{
5777	truncate_inode_pages_range(inode->i_mapping, lstart, lend);
5778	}
5779	EXPORT_SYMBOL_GPL(shmem_truncate_range);
5780
5781	#define shmem_vm_ops generic_file_vm_ops
5782	#define shmem_anon_vm_ops generic_file_vm_ops
5783	#define shmem_file_operations ramfs_file_operations
5784	#define shmem_acct_size(flags, size) 0
5785	#define shmem_unacct_size(flags, size) do {} while (0)
5786
5787	static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
5788	struct super_block *sb, struct inode *dir,
5789	umode_t mode, dev_t dev, unsigned long flags)
5790	{
5791	struct inode *inode = ramfs_get_inode(sb, dir, mode, dev);
5792	return inode ? inode : ERR_PTR(-ENOSPC);
5793	}
5794
5795	#endif /* CONFIG_SHMEM */
5796
5797	/* common code */
5798
5799	static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name,
5800	loff_t size, unsigned long flags, unsigned int i_flags)
5801	{
5802	struct inode *inode;
5803	struct file *res;
5804
5805	if (IS_ERR(ptr: mnt))
5806	return ERR_CAST(ptr: mnt);
5807
5808	if (size < 0 || size > MAX_LFS_FILESIZE)
5809	return ERR_PTR(error: -EINVAL);
5810
5811	if (is_idmapped_mnt(mnt))
5812	return ERR_PTR(error: -EINVAL);
5813
5814	if (shmem_acct_size(flags, size))
5815	return ERR_PTR(error: -ENOMEM);
5816
5817	inode = shmem_get_inode(idmap: &nop_mnt_idmap, sb: mnt->mnt_sb, NULL,
5818	S_IFREG | S_IRWXUGO, dev: 0, flags);
5819	if (IS_ERR(ptr: inode)) {
5820	shmem_unacct_size(flags, size);
5821	return ERR_CAST(ptr: inode);
5822	}
5823	inode->i_flags |= i_flags;
5824	inode->i_size = size;
5825	clear_nlink(inode); /* It is unlinked */
5826	res = ERR_PTR(error: ramfs_nommu_expand_for_mapping(inode, newsize: size));
5827	if (!IS_ERR(ptr: res))
5828	res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
5829	&shmem_file_operations);
5830	if (IS_ERR(ptr: res))
5831	iput(inode);
5832	return res;
5833	}
5834
5835	/**
5836	* shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
5837	* kernel internal. There will be NO LSM permission checks against the
5838	* underlying inode. So users of this interface must do LSM checks at a
5839	* higher layer. The users are the big_key and shm implementations. LSM
5840	* checks are provided at the key or shm level rather than the inode.
5841	* @name: name for dentry (to be seen in /proc/<pid>/maps)
5842	* @size: size to be set for the file
5843	* @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
5844	*/
5845	struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
5846	{
5847	return __shmem_file_setup(mnt: shm_mnt, name, size, flags, S_PRIVATE);
5848	}
5849	EXPORT_SYMBOL_GPL(shmem_kernel_file_setup);
5850
5851	/**
5852	* shmem_file_setup - get an unlinked file living in tmpfs
5853	* @name: name for dentry (to be seen in /proc/<pid>/maps)
5854	* @size: size to be set for the file
5855	* @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
5856	*/
5857	struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
5858	{
5859	return __shmem_file_setup(mnt: shm_mnt, name, size, flags, i_flags: 0);
5860	}
5861	EXPORT_SYMBOL_GPL(shmem_file_setup);
5862
5863	/**
5864	* shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
5865	* @mnt: the tmpfs mount where the file will be created
5866	* @name: name for dentry (to be seen in /proc/<pid>/maps)
5867	* @size: size to be set for the file
5868	* @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
5869	*/
5870	struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
5871	loff_t size, unsigned long flags)
5872	{
5873	return __shmem_file_setup(mnt, name, size, flags, i_flags: 0);
5874	}
5875	EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
5876
5877	/**
5878	* shmem_zero_setup - setup a shared anonymous mapping
5879	* @vma: the vma to be mmapped is prepared by do_mmap
5880	*/
5881	int shmem_zero_setup(struct vm_area_struct *vma)
5882	{
5883	struct file *file;
5884	loff_t size = vma->vm_end - vma->vm_start;
5885
5886	/*
5887	* Cloning a new file under mmap_lock leads to a lock ordering conflict
5888	* between XFS directory reading and selinux: since this file is only
5889	* accessible to the user through its mapping, use S_PRIVATE flag to
5890	* bypass file security, in the same way as shmem_kernel_file_setup().
5891	*/
5892	file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
5893	if (IS_ERR(ptr: file))
5894	return PTR_ERR(ptr: file);
5895
5896	if (vma->vm_file)
5897	fput(vma->vm_file);
5898	vma->vm_file = file;
5899	vma->vm_ops = &shmem_anon_vm_ops;
5900
5901	return 0;
5902	}
5903
5904	/**
5905	* shmem_read_folio_gfp - read into page cache, using specified page allocation flags.
5906	* @mapping: the folio's address_space
5907	* @index: the folio index
5908	* @gfp: the page allocator flags to use if allocating
5909	*
5910	* This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
5911	* with any new page allocations done using the specified allocation flags.
5912	* But read_cache_page_gfp() uses the ->read_folio() method: which does not
5913	* suit tmpfs, since it may have pages in swapcache, and needs to find those
5914	* for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
5915	*
5916	* i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
5917	* with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
5918	*/
5919	struct folio *shmem_read_folio_gfp(struct address_space *mapping,
5920	pgoff_t index, gfp_t gfp)
5921	{
5922	#ifdef CONFIG_SHMEM
5923	struct inode *inode = mapping->host;
5924	struct folio *folio;
5925	int error;
5926
5927	error = shmem_get_folio_gfp(inode, index, write_end: 0, foliop: &folio, sgp: SGP_CACHE,
5928	gfp, NULL, NULL);
5929	if (error)
5930	return ERR_PTR(error);
5931
5932	folio_unlock(folio);
5933	return folio;
5934	#else
5935	/*
5936	* The tiny !SHMEM case uses ramfs without swap
5937	*/
5938	return mapping_read_folio_gfp(mapping, index, gfp);
5939	#endif
5940	}
5941	EXPORT_SYMBOL_GPL(shmem_read_folio_gfp);
5942
5943	struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
5944	pgoff_t index, gfp_t gfp)
5945	{
5946	struct folio *folio = shmem_read_folio_gfp(mapping, index, gfp);
5947	struct page *page;
5948
5949	if (IS_ERR(ptr: folio))
5950	return &folio->page;
5951
5952	page = folio_file_page(folio, index);
5953	if (PageHWPoison(page)) {
5954	folio_put(folio);
5955	return ERR_PTR(error: -EIO);
5956	}
5957
5958	return page;
5959	}
5960	EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
5961