1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* mm/page-writeback.c
4	*
5	* Copyright (C) 2002, Linus Torvalds.
6	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
7	*
8	* Contains functions related to writing back dirty pages at the
9	* address_space level.
10	*
11	* 10Apr2002 Andrew Morton
12	* Initial version
13	*/
14
15	#include <linux/kernel.h>
16	#include <linux/math64.h>
17	#include <linux/export.h>
18	#include <linux/spinlock.h>
19	#include <linux/fs.h>
20	#include <linux/mm.h>
21	#include <linux/swap.h>
22	#include <linux/slab.h>
23	#include <linux/pagemap.h>
24	#include <linux/writeback.h>
25	#include <linux/init.h>
26	#include <linux/backing-dev.h>
27	#include <linux/task_io_accounting_ops.h>
28	#include <linux/blkdev.h>
29	#include <linux/mpage.h>
30	#include <linux/rmap.h>
31	#include <linux/percpu.h>
32	#include <linux/smp.h>
33	#include <linux/sysctl.h>
34	#include <linux/cpu.h>
35	#include <linux/syscalls.h>
36	#include <linux/pagevec.h>
37	#include <linux/timer.h>
38	#include <linux/sched/rt.h>
39	#include <linux/sched/signal.h>
40	#include <linux/mm_inline.h>
41	#include <trace/events/writeback.h>
42
43	#include "internal.h"
44	#include "swap.h"
45
46	/*
47	* Sleep at most 200ms at a time in balance_dirty_pages().
48	*/
49	#define MAX_PAUSE max(HZ/5, 1)
50
51	/*
52	* Try to keep balance_dirty_pages() call intervals higher than this many pages
53	* by raising pause time to max_pause when falls below it.
54	*/
55	#define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10))
56
57	/*
58	* Estimate write bandwidth or update dirty limit at 200ms intervals.
59	*/
60	#define BANDWIDTH_INTERVAL max(HZ/5, 1)
61
62	#define RATELIMIT_CALC_SHIFT 10
63
64	/*
65	* After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
66	* will look to see if it needs to force writeback or throttling.
67	*/
68	static long ratelimit_pages = 32;
69
70	/* The following parameters are exported via /proc/sys/vm */
71
72	/*
73	* Start background writeback (via writeback threads) at this percentage
74	*/
75	static int dirty_background_ratio = 10;
76
77	/*
78	* dirty_background_bytes starts at 0 (disabled) so that it is a function of
79	* dirty_background_ratio * the amount of dirtyable memory
80	*/
81	static unsigned long dirty_background_bytes;
82
83	/*
84	* free highmem will not be subtracted from the total free memory
85	* for calculating free ratios if vm_highmem_is_dirtyable is true
86	*/
87	static int vm_highmem_is_dirtyable;
88
89	/*
90	* The generator of dirty data starts writeback at this percentage
91	*/
92	static int vm_dirty_ratio = 20;
93
94	/*
95	* vm_dirty_bytes starts at 0 (disabled) so that it is a function of
96	* vm_dirty_ratio * the amount of dirtyable memory
97	*/
98	static unsigned long vm_dirty_bytes;
99
100	/*
101	* The interval between `kupdate'-style writebacks
102	*/
103	unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
104
105	EXPORT_SYMBOL_GPL(dirty_writeback_interval);
106
107	/*
108	* The longest time for which data is allowed to remain dirty
109	*/
110	unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */
111
112	/*
113	* Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
114	* a full sync is triggered after this time elapses without any disk activity.
115	*/
116	int laptop_mode;
117
118	EXPORT_SYMBOL(laptop_mode);
119
120	/* End of sysctl-exported parameters */
121
122	struct wb_domain global_wb_domain;
123
124	/*
125	* Length of period for aging writeout fractions of bdis. This is an
126	* arbitrarily chosen number. The longer the period, the slower fractions will
127	* reflect changes in current writeout rate.
128	*/
129	#define VM_COMPLETIONS_PERIOD_LEN (3*HZ)
130
131	#ifdef CONFIG_CGROUP_WRITEBACK
132
133	#define GDTC_INIT(__wb) .wb = (__wb), \
134	.dom = &global_wb_domain, \
135	.wb_completions = &(__wb)->completions
136
137	#define GDTC_INIT_NO_WB .dom = &global_wb_domain
138
139	#define MDTC_INIT(__wb, __gdtc) .wb = (__wb), \
140	.dom = mem_cgroup_wb_domain(__wb), \
141	.wb_completions = &(__wb)->memcg_completions, \
142	.gdtc = __gdtc
143
144	static bool mdtc_valid(struct dirty_throttle_control *dtc)
145	{
146	return dtc->dom;
147	}
148
149	static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
150	{
151	return dtc->dom;
152	}
153
154	static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
155	{
156	return mdtc->gdtc;
157	}
158
159	static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
160	{
161	return &wb->memcg_completions;
162	}
163
164	static void wb_min_max_ratio(struct bdi_writeback *wb,
165	unsigned long *minp, unsigned long *maxp)
166	{
167	unsigned long this_bw = READ_ONCE(wb->avg_write_bandwidth);
168	unsigned long tot_bw = atomic_long_read(v: &wb->bdi->tot_write_bandwidth);
169	unsigned long long min = wb->bdi->min_ratio;
170	unsigned long long max = wb->bdi->max_ratio;
171
172	/*
173	* @wb may already be clean by the time control reaches here and
174	* the total may not include its bw.
175	*/
176	if (this_bw < tot_bw) {
177	if (min) {
178	min *= this_bw;
179	min = div64_ul(min, tot_bw);
180	}
181	if (max < 100 * BDI_RATIO_SCALE) {
182	max *= this_bw;
183	max = div64_ul(max, tot_bw);
184	}
185	}
186
187	*minp = min;
188	*maxp = max;
189	}
190
191	#else /* CONFIG_CGROUP_WRITEBACK */
192
193	#define GDTC_INIT(__wb) .wb = (__wb), \
194	.wb_completions = &(__wb)->completions
195	#define GDTC_INIT_NO_WB
196	#define MDTC_INIT(__wb, __gdtc)
197
198	static bool mdtc_valid(struct dirty_throttle_control *dtc)
199	{
200	return false;
201	}
202
203	static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
204	{
205	return &global_wb_domain;
206	}
207
208	static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
209	{
210	return NULL;
211	}
212
213	static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
214	{
215	return NULL;
216	}
217
218	static void wb_min_max_ratio(struct bdi_writeback *wb,
219	unsigned long *minp, unsigned long *maxp)
220	{
221	*minp = wb->bdi->min_ratio;
222	*maxp = wb->bdi->max_ratio;
223	}
224
225	#endif /* CONFIG_CGROUP_WRITEBACK */
226
227	/*
228	* In a memory zone, there is a certain amount of pages we consider
229	* available for the page cache, which is essentially the number of
230	* free and reclaimable pages, minus some zone reserves to protect
231	* lowmem and the ability to uphold the zone's watermarks without
232	* requiring writeback.
233	*
234	* This number of dirtyable pages is the base value of which the
235	* user-configurable dirty ratio is the effective number of pages that
236	* are allowed to be actually dirtied. Per individual zone, or
237	* globally by using the sum of dirtyable pages over all zones.
238	*
239	* Because the user is allowed to specify the dirty limit globally as
240	* absolute number of bytes, calculating the per-zone dirty limit can
241	* require translating the configured limit into a percentage of
242	* global dirtyable memory first.
243	*/
244
245	/**
246	* node_dirtyable_memory - number of dirtyable pages in a node
247	* @pgdat: the node
248	*
249	* Return: the node's number of pages potentially available for dirty
250	* page cache. This is the base value for the per-node dirty limits.
251	*/
252	static unsigned long node_dirtyable_memory(struct pglist_data *pgdat)
253	{
254	unsigned long nr_pages = 0;
255	int z;
256
257	for (z = 0; z < MAX_NR_ZONES; z++) {
258	struct zone *zone = pgdat->node_zones + z;
259
260	if (!populated_zone(zone))
261	continue;
262
263	nr_pages += zone_page_state(zone, item: NR_FREE_PAGES);
264	}
265
266	/*
267	* Pages reserved for the kernel should not be considered
268	* dirtyable, to prevent a situation where reclaim has to
269	* clean pages in order to balance the zones.
270	*/
271	nr_pages -= min(nr_pages, pgdat->totalreserve_pages);
272
273	nr_pages += node_page_state(pgdat, item: NR_INACTIVE_FILE);
274	nr_pages += node_page_state(pgdat, item: NR_ACTIVE_FILE);
275
276	return nr_pages;
277	}
278
279	static unsigned long highmem_dirtyable_memory(unsigned long total)
280	{
281	#ifdef CONFIG_HIGHMEM
282	int node;
283	unsigned long x = 0;
284	int i;
285
286	for_each_node_state(node, N_HIGH_MEMORY) {
287	for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) {
288	struct zone *z;
289	unsigned long nr_pages;
290
291	if (!is_highmem_idx(i))
292	continue;
293
294	z = &NODE_DATA(node)->node_zones[i];
295	if (!populated_zone(z))
296	continue;
297
298	nr_pages = zone_page_state(z, NR_FREE_PAGES);
299	/* watch for underflows */
300	nr_pages -= min(nr_pages, high_wmark_pages(z));
301	nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE);
302	nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE);
303	x += nr_pages;
304	}
305	}
306
307	/*
308	* Make sure that the number of highmem pages is never larger
309	* than the number of the total dirtyable memory. This can only
310	* occur in very strange VM situations but we want to make sure
311	* that this does not occur.
312	*/
313	return min(x, total);
314	#else
315	return 0;
316	#endif
317	}
318
319	/**
320	* global_dirtyable_memory - number of globally dirtyable pages
321	*
322	* Return: the global number of pages potentially available for dirty
323	* page cache. This is the base value for the global dirty limits.
324	*/
325	static unsigned long global_dirtyable_memory(void)
326	{
327	unsigned long x;
328
329	x = global_zone_page_state(item: NR_FREE_PAGES);
330	/*
331	* Pages reserved for the kernel should not be considered
332	* dirtyable, to prevent a situation where reclaim has to
333	* clean pages in order to balance the zones.
334	*/
335	x -= min(x, totalreserve_pages);
336
337	x += global_node_page_state(item: NR_INACTIVE_FILE);
338	x += global_node_page_state(item: NR_ACTIVE_FILE);
339
340	if (!vm_highmem_is_dirtyable)
341	x -= highmem_dirtyable_memory(total: x);
342
343	return x + 1; /* Ensure that we never return 0 */
344	}
345
346	/**
347	* domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain
348	* @dtc: dirty_throttle_control of interest
349	*
350	* Calculate @dtc->thresh and ->bg_thresh considering
351	* vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}. The caller
352	* must ensure that @dtc->avail is set before calling this function. The
353	* dirty limits will be lifted by 1/4 for real-time tasks.
354	*/
355	static void domain_dirty_limits(struct dirty_throttle_control *dtc)
356	{
357	const unsigned long available_memory = dtc->avail;
358	struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc: dtc);
359	unsigned long bytes = vm_dirty_bytes;
360	unsigned long bg_bytes = dirty_background_bytes;
361	/* convert ratios to per-PAGE_SIZE for higher precision */
362	unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;
363	unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;
364	unsigned long thresh;
365	unsigned long bg_thresh;
366	struct task_struct *tsk;
367
368	/* gdtc is !NULL iff @dtc is for memcg domain */
369	if (gdtc) {
370	unsigned long global_avail = gdtc->avail;
371
372	/*
373	* The byte settings can't be applied directly to memcg
374	* domains. Convert them to ratios by scaling against
375	* globally available memory. As the ratios are in
376	* per-PAGE_SIZE, they can be obtained by dividing bytes by
377	* number of pages.
378	*/
379	if (bytes)
380	ratio = min(DIV_ROUND_UP(bytes, global_avail),
381	PAGE_SIZE);
382	if (bg_bytes)
383	bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),
384	PAGE_SIZE);
385	bytes = bg_bytes = 0;
386	}
387
388	if (bytes)
389	thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);
390	else
391	thresh = (ratio * available_memory) / PAGE_SIZE;
392
393	if (bg_bytes)
394	bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);
395	else
396	bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;
397
398	tsk = current;
399	if (rt_or_dl_task(p: tsk)) {
400	bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
401	thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
402	}
403	/*
404	* Dirty throttling logic assumes the limits in page units fit into
405	* 32-bits. This gives 16TB dirty limits max which is hopefully enough.
406	*/
407	if (thresh > UINT_MAX)
408	thresh = UINT_MAX;
409	/* This makes sure bg_thresh is within 32-bits as well */
410	if (bg_thresh >= thresh)
411	bg_thresh = thresh / 2;
412	dtc->thresh = thresh;
413	dtc->bg_thresh = bg_thresh;
414
415	/* we should eventually report the domain in the TP */
416	if (!gdtc)
417	trace_global_dirty_state(background_thresh: bg_thresh, dirty_thresh: thresh);
418	}
419
420	/**
421	* global_dirty_limits - background-writeback and dirty-throttling thresholds
422	* @pbackground: out parameter for bg_thresh
423	* @pdirty: out parameter for thresh
424	*
425	* Calculate bg_thresh and thresh for global_wb_domain. See
426	* domain_dirty_limits() for details.
427	*/
428	void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
429	{
430	struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };
431
432	gdtc.avail = global_dirtyable_memory();
433	domain_dirty_limits(dtc: &gdtc);
434
435	*pbackground = gdtc.bg_thresh;
436	*pdirty = gdtc.thresh;
437	}
438
439	/**
440	* node_dirty_limit - maximum number of dirty pages allowed in a node
441	* @pgdat: the node
442	*
443	* Return: the maximum number of dirty pages allowed in a node, based
444	* on the node's dirtyable memory.
445	*/
446	static unsigned long node_dirty_limit(struct pglist_data *pgdat)
447	{
448	unsigned long node_memory = node_dirtyable_memory(pgdat);
449	struct task_struct *tsk = current;
450	unsigned long dirty;
451
452	if (vm_dirty_bytes)
453	dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
454	node_memory / global_dirtyable_memory();
455	else
456	dirty = vm_dirty_ratio * node_memory / 100;
457
458	if (rt_or_dl_task(p: tsk))
459	dirty += dirty / 4;
460
461	/*
462	* Dirty throttling logic assumes the limits in page units fit into
463	* 32-bits. This gives 16TB dirty limits max which is hopefully enough.
464	*/
465	return min_t(unsigned long, dirty, UINT_MAX);
466	}
467
468	/**
469	* node_dirty_ok - tells whether a node is within its dirty limits
470	* @pgdat: the node to check
471	*
472	* Return: %true when the dirty pages in @pgdat are within the node's
473	* dirty limit, %false if the limit is exceeded.
474	*/
475	bool node_dirty_ok(struct pglist_data *pgdat)
476	{
477	unsigned long limit = node_dirty_limit(pgdat);
478	unsigned long nr_pages = 0;
479
480	nr_pages += node_page_state(pgdat, item: NR_FILE_DIRTY);
481	nr_pages += node_page_state(pgdat, item: NR_WRITEBACK);
482
483	return nr_pages <= limit;
484	}
485
486	#ifdef CONFIG_SYSCTL
487	static int dirty_background_ratio_handler(const struct ctl_table *table, int write,
488	void *buffer, size_t *lenp, loff_t *ppos)
489	{
490	int ret;
491
492	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
493	if (ret == 0 && write)
494	dirty_background_bytes = 0;
495	return ret;
496	}
497
498	static int dirty_background_bytes_handler(const struct ctl_table *table, int write,
499	void *buffer, size_t *lenp, loff_t *ppos)
500	{
501	int ret;
502	unsigned long old_bytes = dirty_background_bytes;
503
504	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
505	if (ret == 0 && write) {
506	if (DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE) >
507	UINT_MAX) {
508	dirty_background_bytes = old_bytes;
509	return -ERANGE;
510	}
511	dirty_background_ratio = 0;
512	}
513	return ret;
514	}
515
516	static int dirty_ratio_handler(const struct ctl_table *table, int write, void *buffer,
517	size_t *lenp, loff_t *ppos)
518	{
519	int old_ratio = vm_dirty_ratio;
520	int ret;
521
522	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
523	if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
524	vm_dirty_bytes = 0;
525	writeback_set_ratelimit();
526	}
527	return ret;
528	}
529
530	static int dirty_bytes_handler(const struct ctl_table *table, int write,
531	void *buffer, size_t *lenp, loff_t *ppos)
532	{
533	unsigned long old_bytes = vm_dirty_bytes;
534	int ret;
535
536	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
537	if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
538	if (DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) > UINT_MAX) {
539	vm_dirty_bytes = old_bytes;
540	return -ERANGE;
541	}
542	writeback_set_ratelimit();
543	vm_dirty_ratio = 0;
544	}
545	return ret;
546	}
547	#endif
548
549	static unsigned long wp_next_time(unsigned long cur_time)
550	{
551	cur_time += VM_COMPLETIONS_PERIOD_LEN;
552	/* 0 has a special meaning... */
553	if (!cur_time)
554	return 1;
555	return cur_time;
556	}
557
558	static void wb_domain_writeout_add(struct wb_domain *dom,
559	struct fprop_local_percpu *completions,
560	unsigned int max_prop_frac, long nr)
561	{
562	__fprop_add_percpu_max(p: &dom->completions, pl: completions,
563	max_frac: max_prop_frac, nr);
564	/* First event after period switching was turned off? */
565	if (unlikely(!dom->period_time)) {
566	/*
567	* We can race with other wb_domain_writeout_add calls here but
568	* it does not cause any harm since the resulting time when
569	* timer will fire and what is in writeout_period_time will be
570	* roughly the same.
571	*/
572	dom->period_time = wp_next_time(cur_time: jiffies);
573	mod_timer(timer: &dom->period_timer, expires: dom->period_time);
574	}
575	}
576
577	/*
578	* Increment @wb's writeout completion count and the global writeout
579	* completion count. Called from __folio_end_writeback().
580	*/
581	static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr)
582	{
583	struct wb_domain *cgdom;
584
585	wb_stat_mod(wb, item: WB_WRITTEN, amount: nr);
586	wb_domain_writeout_add(dom: &global_wb_domain, completions: &wb->completions,
587	max_prop_frac: wb->bdi->max_prop_frac, nr);
588
589	cgdom = mem_cgroup_wb_domain(wb);
590	if (cgdom)
591	wb_domain_writeout_add(dom: cgdom, completions: wb_memcg_completions(wb),
592	max_prop_frac: wb->bdi->max_prop_frac, nr);
593	}
594
595	void wb_writeout_inc(struct bdi_writeback *wb)
596	{
597	unsigned long flags;
598
599	local_irq_save(flags);
600	__wb_writeout_add(wb, nr: 1);
601	local_irq_restore(flags);
602	}
603	EXPORT_SYMBOL_GPL(wb_writeout_inc);
604
605	/*
606	* On idle system, we can be called long after we scheduled because we use
607	* deferred timers so count with missed periods.
608	*/
609	static void writeout_period(struct timer_list *t)
610	{
611	struct wb_domain *dom = timer_container_of(dom, t, period_timer);
612	int miss_periods = (jiffies - dom->period_time) /
613	VM_COMPLETIONS_PERIOD_LEN;
614
615	if (fprop_new_period(p: &dom->completions, periods: miss_periods + 1)) {
616	dom->period_time = wp_next_time(cur_time: dom->period_time +
617	miss_periods * VM_COMPLETIONS_PERIOD_LEN);
618	mod_timer(timer: &dom->period_timer, expires: dom->period_time);
619	} else {
620	/*
621	* Aging has zeroed all fractions. Stop wasting CPU on period
622	* updates.
623	*/
624	dom->period_time = 0;
625	}
626	}
627
628	int wb_domain_init(struct wb_domain *dom, gfp_t gfp)
629	{
630	memset(dom, 0, sizeof(*dom));
631
632	spin_lock_init(&dom->lock);
633
634	timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE);
635
636	dom->dirty_limit_tstamp = jiffies;
637
638	return fprop_global_init(p: &dom->completions, gfp);
639	}
640
641	#ifdef CONFIG_CGROUP_WRITEBACK
642	void wb_domain_exit(struct wb_domain *dom)
643	{
644	timer_delete_sync(timer: &dom->period_timer);
645	fprop_global_destroy(p: &dom->completions);
646	}
647	#endif
648
649	/*
650	* bdi_min_ratio keeps the sum of the minimum dirty shares of all
651	* registered backing devices, which, for obvious reasons, can not
652	* exceed 100%.
653	*/
654	static unsigned int bdi_min_ratio;
655
656	static int bdi_check_pages_limit(unsigned long pages)
657	{
658	unsigned long max_dirty_pages = global_dirtyable_memory();
659
660	if (pages > max_dirty_pages)
661	return -EINVAL;
662
663	return 0;
664	}
665
666	static unsigned long bdi_ratio_from_pages(unsigned long pages)
667	{
668	unsigned long background_thresh;
669	unsigned long dirty_thresh;
670	unsigned long ratio;
671
672	global_dirty_limits(pbackground: &background_thresh, pdirty: &dirty_thresh);
673	if (!dirty_thresh)
674	return -EINVAL;
675	ratio = div64_u64(dividend: pages * 100ULL * BDI_RATIO_SCALE, divisor: dirty_thresh);
676
677	return ratio;
678	}
679
680	static u64 bdi_get_bytes(unsigned int ratio)
681	{
682	unsigned long background_thresh;
683	unsigned long dirty_thresh;
684	u64 bytes;
685
686	global_dirty_limits(pbackground: &background_thresh, pdirty: &dirty_thresh);
687	bytes = (dirty_thresh * PAGE_SIZE * ratio) / BDI_RATIO_SCALE / 100;
688
689	return bytes;
690	}
691
692	static int __bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
693	{
694	unsigned int delta;
695	int ret = 0;
696
697	if (min_ratio > 100 * BDI_RATIO_SCALE)
698	return -EINVAL;
699
700	spin_lock_bh(lock: &bdi_lock);
701	if (min_ratio > bdi->max_ratio) {
702	ret = -EINVAL;
703	} else {
704	if (min_ratio < bdi->min_ratio) {
705	delta = bdi->min_ratio - min_ratio;
706	bdi_min_ratio -= delta;
707	bdi->min_ratio = min_ratio;
708	} else {
709	delta = min_ratio - bdi->min_ratio;
710	if (bdi_min_ratio + delta < 100 * BDI_RATIO_SCALE) {
711	bdi_min_ratio += delta;
712	bdi->min_ratio = min_ratio;
713	} else {
714	ret = -EINVAL;
715	}
716	}
717	}
718	spin_unlock_bh(lock: &bdi_lock);
719
720	return ret;
721	}
722
723	static int __bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio)
724	{
725	int ret = 0;
726
727	if (max_ratio > 100 * BDI_RATIO_SCALE)
728	return -EINVAL;
729
730	spin_lock_bh(lock: &bdi_lock);
731	if (bdi->min_ratio > max_ratio) {
732	ret = -EINVAL;
733	} else {
734	bdi->max_ratio = max_ratio;
735	bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) /
736	(100 * BDI_RATIO_SCALE);
737	}
738	spin_unlock_bh(lock: &bdi_lock);
739
740	return ret;
741	}
742
743	int bdi_set_min_ratio_no_scale(struct backing_dev_info *bdi, unsigned int min_ratio)
744	{
745	return __bdi_set_min_ratio(bdi, min_ratio);
746	}
747
748	int bdi_set_max_ratio_no_scale(struct backing_dev_info *bdi, unsigned int max_ratio)
749	{
750	return __bdi_set_max_ratio(bdi, max_ratio);
751	}
752
753	int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
754	{
755	return __bdi_set_min_ratio(bdi, min_ratio: min_ratio * BDI_RATIO_SCALE);
756	}
757
758	int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio)
759	{
760	return __bdi_set_max_ratio(bdi, max_ratio: max_ratio * BDI_RATIO_SCALE);
761	}
762	EXPORT_SYMBOL(bdi_set_max_ratio);
763
764	u64 bdi_get_min_bytes(struct backing_dev_info *bdi)
765	{
766	return bdi_get_bytes(ratio: bdi->min_ratio);
767	}
768
769	int bdi_set_min_bytes(struct backing_dev_info *bdi, u64 min_bytes)
770	{
771	int ret;
772	unsigned long pages = min_bytes >> PAGE_SHIFT;
773	long min_ratio;
774
775	ret = bdi_check_pages_limit(pages);
776	if (ret)
777	return ret;
778
779	min_ratio = bdi_ratio_from_pages(pages);
780	if (min_ratio < 0)
781	return min_ratio;
782	return __bdi_set_min_ratio(bdi, min_ratio);
783	}
784
785	u64 bdi_get_max_bytes(struct backing_dev_info *bdi)
786	{
787	return bdi_get_bytes(ratio: bdi->max_ratio);
788	}
789
790	int bdi_set_max_bytes(struct backing_dev_info *bdi, u64 max_bytes)
791	{
792	int ret;
793	unsigned long pages = max_bytes >> PAGE_SHIFT;
794	long max_ratio;
795
796	ret = bdi_check_pages_limit(pages);
797	if (ret)
798	return ret;
799
800	max_ratio = bdi_ratio_from_pages(pages);
801	if (max_ratio < 0)
802	return max_ratio;
803	return __bdi_set_max_ratio(bdi, max_ratio);
804	}
805
806	int bdi_set_strict_limit(struct backing_dev_info *bdi, unsigned int strict_limit)
807	{
808	if (strict_limit > 1)
809	return -EINVAL;
810
811	spin_lock_bh(lock: &bdi_lock);
812	if (strict_limit)
813	bdi->capabilities |= BDI_CAP_STRICTLIMIT;
814	else
815	bdi->capabilities &= ~BDI_CAP_STRICTLIMIT;
816	spin_unlock_bh(lock: &bdi_lock);
817
818	return 0;
819	}
820
821	static unsigned long dirty_freerun_ceiling(unsigned long thresh,
822	unsigned long bg_thresh)
823	{
824	return (thresh + bg_thresh) / 2;
825	}
826
827	static unsigned long hard_dirty_limit(struct wb_domain *dom,
828	unsigned long thresh)
829	{
830	return max(thresh, dom->dirty_limit);
831	}
832
833	/*
834	* Memory which can be further allocated to a memcg domain is capped by
835	* system-wide clean memory excluding the amount being used in the domain.
836	*/
837	static void mdtc_calc_avail(struct dirty_throttle_control *mdtc,
838	unsigned long filepages, unsigned long headroom)
839	{
840	struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc);
841	unsigned long clean = filepages - min(filepages, mdtc->dirty);
842	unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty);
843	unsigned long other_clean = global_clean - min(global_clean, clean);
844
845	mdtc->avail = filepages + min(headroom, other_clean);
846	}
847
848	static inline bool dtc_is_global(struct dirty_throttle_control *dtc)
849	{
850	return mdtc_gdtc(mdtc: dtc) == NULL;
851	}
852
853	/*
854	* Dirty background will ignore pages being written as we're trying to
855	* decide whether to put more under writeback.
856	*/
857	static void domain_dirty_avail(struct dirty_throttle_control *dtc,
858	bool include_writeback)
859	{
860	if (dtc_is_global(dtc)) {
861	dtc->avail = global_dirtyable_memory();
862	dtc->dirty = global_node_page_state(item: NR_FILE_DIRTY);
863	if (include_writeback)
864	dtc->dirty += global_node_page_state(item: NR_WRITEBACK);
865	} else {
866	unsigned long filepages = 0, headroom = 0, writeback = 0;
867
868	mem_cgroup_wb_stats(wb: dtc->wb, pfilepages: &filepages, pheadroom: &headroom, pdirty: &dtc->dirty,
869	pwriteback: &writeback);
870	if (include_writeback)
871	dtc->dirty += writeback;
872	mdtc_calc_avail(mdtc: dtc, filepages, headroom);
873	}
874	}
875
876	/**
877	* __wb_calc_thresh - @wb's share of dirty threshold
878	* @dtc: dirty_throttle_context of interest
879	* @thresh: dirty throttling or dirty background threshold of wb_domain in @dtc
880	*
881	* Note that balance_dirty_pages() will only seriously take dirty throttling
882	* threshold as a hard limit when sleeping max_pause per page is not enough
883	* to keep the dirty pages under control. For example, when the device is
884	* completely stalled due to some error conditions, or when there are 1000
885	* dd tasks writing to a slow 10MB/s USB key.
886	* In the other normal situations, it acts more gently by throttling the tasks
887	* more (rather than completely block them) when the wb dirty pages go high.
888	*
889	* It allocates high/low dirty limits to fast/slow devices, in order to prevent
890	* - starving fast devices
891	* - piling up dirty pages (that will take long time to sync) on slow devices
892	*
893	* The wb's share of dirty limit will be adapting to its throughput and
894	* bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
895	*
896	* Return: @wb's dirty limit in pages. For dirty throttling limit, the term
897	* "dirty" in the context of dirty balancing includes all PG_dirty and
898	* PG_writeback pages.
899	*/
900	static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc,
901	unsigned long thresh)
902	{
903	struct wb_domain *dom = dtc_dom(dtc);
904	struct bdi_writeback *wb = dtc->wb;
905	u64 wb_thresh;
906	u64 wb_max_thresh;
907	unsigned long numerator, denominator;
908	unsigned long wb_min_ratio, wb_max_ratio;
909
910	/*
911	* Calculate this wb's share of the thresh ratio.
912	*/
913	fprop_fraction_percpu(p: &dom->completions, pl: dtc->wb_completions,
914	numerator: &numerator, denominator: &denominator);
915
916	wb_thresh = (thresh * (100 * BDI_RATIO_SCALE - bdi_min_ratio)) / (100 * BDI_RATIO_SCALE);
917	wb_thresh *= numerator;
918	wb_thresh = div64_ul(wb_thresh, denominator);
919
920	wb_min_max_ratio(wb, minp: &wb_min_ratio, maxp: &wb_max_ratio);
921
922	wb_thresh += (thresh * wb_min_ratio) / (100 * BDI_RATIO_SCALE);
923
924	/*
925	* It's very possible that wb_thresh is close to 0 not because the
926	* device is slow, but that it has remained inactive for long time.
927	* Honour such devices a reasonable good (hopefully IO efficient)
928	* threshold, so that the occasional writes won't be blocked and active
929	* writes can rampup the threshold quickly.
930	*/
931	if (thresh > dtc->dirty) {
932	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT))
933	wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 100);
934	else
935	wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 8);
936	}
937
938	wb_max_thresh = thresh * wb_max_ratio / (100 * BDI_RATIO_SCALE);
939	if (wb_thresh > wb_max_thresh)
940	wb_thresh = wb_max_thresh;
941
942	return wb_thresh;
943	}
944
945	unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)
946	{
947	struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };
948
949	domain_dirty_avail(dtc: &gdtc, include_writeback: true);
950	return __wb_calc_thresh(dtc: &gdtc, thresh);
951	}
952
953	unsigned long cgwb_calc_thresh(struct bdi_writeback *wb)
954	{
955	struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };
956	struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) };
957
958	domain_dirty_avail(dtc: &gdtc, include_writeback: true);
959	domain_dirty_avail(dtc: &mdtc, include_writeback: true);
960	domain_dirty_limits(dtc: &mdtc);
961
962	return __wb_calc_thresh(dtc: &mdtc, thresh: mdtc.thresh);
963	}
964
965	/*
966	* setpoint - dirty 3
967	* f(dirty) := 1.0 + (----------------)
968	* limit - setpoint
969	*
970	* it's a 3rd order polynomial that subjects to
971	*
972	* (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast
973	* (2) f(setpoint) = 1.0 => the balance point
974	* (3) f(limit) = 0 => the hard limit
975	* (4) df/dx <= 0 => negative feedback control
976	* (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
977	* => fast response on large errors; small oscillation near setpoint
978	*/
979	static long long pos_ratio_polynom(unsigned long setpoint,
980	unsigned long dirty,
981	unsigned long limit)
982	{
983	long long pos_ratio;
984	long x;
985
986	x = div64_s64(dividend: ((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT,
987	divisor: (limit - setpoint) | 1);
988	pos_ratio = x;
989	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
990	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
991	pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
992
993	return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT);
994	}
995
996	/*
997	* Dirty position control.
998	*
999	* (o) global/bdi setpoints
1000	*
1001	* We want the dirty pages be balanced around the global/wb setpoints.
1002	* When the number of dirty pages is higher/lower than the setpoint, the
1003	* dirty position control ratio (and hence task dirty ratelimit) will be
1004	* decreased/increased to bring the dirty pages back to the setpoint.
1005	*
1006	* pos_ratio = 1 << RATELIMIT_CALC_SHIFT
1007	*
1008	* if (dirty < setpoint) scale up pos_ratio
1009	* if (dirty > setpoint) scale down pos_ratio
1010	*
1011	* if (wb_dirty < wb_setpoint) scale up pos_ratio
1012	* if (wb_dirty > wb_setpoint) scale down pos_ratio
1013	*
1014	* task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
1015	*
1016	* (o) global control line
1017	*
1018	* ^ pos_ratio
1019	* |
1020	* | |<===== global dirty control scope ======>|
1021	* 2.0 * * * * * * *
1022	* | .*
1023	* | . *
1024	* | . *
1025	* | . *
1026	* | . *
1027	* | . *
1028	* 1.0 ................................*
1029	* | . . *
1030	* | . . *
1031	* | . . *
1032	* | . . *
1033	* | . . *
1034	* 0 +------------.------------------.----------------------*------------->
1035	* freerun^ setpoint^ limit^ dirty pages
1036	*
1037	* (o) wb control line
1038	*
1039	* ^ pos_ratio
1040	* |
1041	* | *
1042	* | *
1043	* | *
1044	* | *
1045	* | * |<=========== span ============>|
1046	* 1.0 .......................*
1047	* | . *
1048	* | . *
1049	* | . *
1050	* | . *
1051	* | . *
1052	* | . *
1053	* | . *
1054	* | . *
1055	* | . *
1056	* | . *
1057	* | . *
1058	* 1/4 ...............................................* * * * * * * * * * * *
1059	* | . .
1060	* | . .
1061	* | . .
1062	* 0 +----------------------.-------------------------------.------------->
1063	* wb_setpoint^ x_intercept^
1064	*
1065	* The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can
1066	* be smoothly throttled down to normal if it starts high in situations like
1067	* - start writing to a slow SD card and a fast disk at the same time. The SD
1068	* card's wb_dirty may rush to many times higher than wb_setpoint.
1069	* - the wb dirty thresh drops quickly due to change of JBOD workload
1070	*/
1071	static void wb_position_ratio(struct dirty_throttle_control *dtc)
1072	{
1073	struct bdi_writeback *wb = dtc->wb;
1074	unsigned long write_bw = READ_ONCE(wb->avg_write_bandwidth);
1075	unsigned long freerun = dirty_freerun_ceiling(thresh: dtc->thresh, bg_thresh: dtc->bg_thresh);
1076	unsigned long limit = dtc->limit = hard_dirty_limit(dom: dtc_dom(dtc), thresh: dtc->thresh);
1077	unsigned long wb_thresh = dtc->wb_thresh;
1078	unsigned long x_intercept;
1079	unsigned long setpoint; /* dirty pages' target balance point */
1080	unsigned long wb_setpoint;
1081	unsigned long span;
1082	long long pos_ratio; /* for scaling up/down the rate limit */
1083	long x;
1084
1085	dtc->pos_ratio = 0;
1086
1087	if (unlikely(dtc->dirty >= limit))
1088	return;
1089
1090	/*
1091	* global setpoint
1092	*
1093	* See comment for pos_ratio_polynom().
1094	*/
1095	setpoint = (freerun + limit) / 2;
1096	pos_ratio = pos_ratio_polynom(setpoint, dirty: dtc->dirty, limit);
1097
1098	/*
1099	* The strictlimit feature is a tool preventing mistrusted filesystems
1100	* from growing a large number of dirty pages before throttling. For
1101	* such filesystems balance_dirty_pages always checks wb counters
1102	* against wb limits. Even if global "nr_dirty" is under "freerun".
1103	* This is especially important for fuse which sets bdi->max_ratio to
1104	* 1% by default. Without strictlimit feature, fuse writeback may
1105	* consume arbitrary amount of RAM because it is accounted in
1106	* NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty".
1107	*
1108	* Here, in wb_position_ratio(), we calculate pos_ratio based on
1109	* two values: wb_dirty and wb_thresh. Let's consider an example:
1110	* total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global
1111	* limits are set by default to 10% and 20% (background and throttle).
1112	* Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages.
1113	* wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is
1114	* about ~6K pages (as the average of background and throttle wb
1115	* limits). The 3rd order polynomial will provide positive feedback if
1116	* wb_dirty is under wb_setpoint and vice versa.
1117	*
1118	* Note, that we cannot use global counters in these calculations
1119	* because we want to throttle process writing to a strictlimit wb
1120	* much earlier than global "freerun" is reached (~23MB vs. ~2.3GB
1121	* in the example above).
1122	*/
1123	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
1124	long long wb_pos_ratio;
1125
1126	if (dtc->wb_dirty >= wb_thresh)
1127	return;
1128
1129	wb_setpoint = dirty_freerun_ceiling(thresh: wb_thresh,
1130	bg_thresh: dtc->wb_bg_thresh);
1131
1132	if (wb_setpoint == 0 || wb_setpoint == wb_thresh)
1133	return;
1134
1135	wb_pos_ratio = pos_ratio_polynom(setpoint: wb_setpoint, dirty: dtc->wb_dirty,
1136	limit: wb_thresh);
1137
1138	/*
1139	* Typically, for strictlimit case, wb_setpoint << setpoint
1140	* and pos_ratio >> wb_pos_ratio. In the other words global
1141	* state ("dirty") is not limiting factor and we have to
1142	* make decision based on wb counters. But there is an
1143	* important case when global pos_ratio should get precedence:
1144	* global limits are exceeded (e.g. due to activities on other
1145	* wb's) while given strictlimit wb is below limit.
1146	*
1147	* "pos_ratio * wb_pos_ratio" would work for the case above,
1148	* but it would look too non-natural for the case of all
1149	* activity in the system coming from a single strictlimit wb
1150	* with bdi->max_ratio == 100%.
1151	*
1152	* Note that min() below somewhat changes the dynamics of the
1153	* control system. Normally, pos_ratio value can be well over 3
1154	* (when globally we are at freerun and wb is well below wb
1155	* setpoint). Now the maximum pos_ratio in the same situation
1156	* is 2. We might want to tweak this if we observe the control
1157	* system is too slow to adapt.
1158	*/
1159	dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);
1160	return;
1161	}
1162
1163	/*
1164	* We have computed basic pos_ratio above based on global situation. If
1165	* the wb is over/under its share of dirty pages, we want to scale
1166	* pos_ratio further down/up. That is done by the following mechanism.
1167	*/
1168
1169	/*
1170	* wb setpoint
1171	*
1172	* f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint)
1173	*
1174	* x_intercept - wb_dirty
1175	* := --------------------------
1176	* x_intercept - wb_setpoint
1177	*
1178	* The main wb control line is a linear function that subjects to
1179	*
1180	* (1) f(wb_setpoint) = 1.0
1181	* (2) k = - 1 / (8 * write_bw) (in single wb case)
1182	* or equally: x_intercept = wb_setpoint + 8 * write_bw
1183	*
1184	* For single wb case, the dirty pages are observed to fluctuate
1185	* regularly within range
1186	* [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2]
1187	* for various filesystems, where (2) can yield in a reasonable 12.5%
1188	* fluctuation range for pos_ratio.
1189	*
1190	* For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its
1191	* own size, so move the slope over accordingly and choose a slope that
1192	* yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh.
1193	*/
1194	if (unlikely(wb_thresh > dtc->thresh))
1195	wb_thresh = dtc->thresh;
1196	/*
1197	* scale global setpoint to wb's:
1198	* wb_setpoint = setpoint * wb_thresh / thresh
1199	*/
1200	x = div_u64(dividend: (u64)wb_thresh << 16, divisor: dtc->thresh | 1);
1201	wb_setpoint = setpoint * (u64)x >> 16;
1202	/*
1203	* Use span=(8*write_bw) in single wb case as indicated by
1204	* (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case.
1205	*
1206	* wb_thresh thresh - wb_thresh
1207	* span = --------- * (8 * write_bw) + ------------------ * wb_thresh
1208	* thresh thresh
1209	*/
1210	span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;
1211	x_intercept = wb_setpoint + span;
1212
1213	if (dtc->wb_dirty < x_intercept - span / 4) {
1214	pos_ratio = div64_u64(dividend: pos_ratio * (x_intercept - dtc->wb_dirty),
1215	divisor: (x_intercept - wb_setpoint) | 1);
1216	} else
1217	pos_ratio /= 4;
1218
1219	/*
1220	* wb reserve area, safeguard against dirty pool underrun and disk idle
1221	* It may push the desired control point of global dirty pages higher
1222	* than setpoint.
1223	*/
1224	x_intercept = wb_thresh / 2;
1225	if (dtc->wb_dirty < x_intercept) {
1226	if (dtc->wb_dirty > x_intercept / 8)
1227	pos_ratio = div_u64(dividend: pos_ratio * x_intercept,
1228	divisor: dtc->wb_dirty);
1229	else
1230	pos_ratio *= 8;
1231	}
1232
1233	dtc->pos_ratio = pos_ratio;
1234	}
1235
1236	static void wb_update_write_bandwidth(struct bdi_writeback *wb,
1237	unsigned long elapsed,
1238	unsigned long written)
1239	{
1240	const unsigned long period = roundup_pow_of_two(3 * HZ);
1241	unsigned long avg = wb->avg_write_bandwidth;
1242	unsigned long old = wb->write_bandwidth;
1243	u64 bw;
1244
1245	/*
1246	* bw = written * HZ / elapsed
1247	*
1248	* bw * elapsed + write_bandwidth * (period - elapsed)
1249	* write_bandwidth = ---------------------------------------------------
1250	* period
1251	*
1252	* @written may have decreased due to folio_redirty_for_writepage().
1253	* Avoid underflowing @bw calculation.
1254	*/
1255	bw = written - min(written, wb->written_stamp);
1256	bw *= HZ;
1257	if (unlikely(elapsed > period)) {
1258	bw = div64_ul(bw, elapsed);
1259	avg = bw;
1260	goto out;
1261	}
1262	bw += (u64)wb->write_bandwidth * (period - elapsed);
1263	bw >>= ilog2(period);
1264
1265	/*
1266	* one more level of smoothing, for filtering out sudden spikes
1267	*/
1268	if (avg > old && old >= (unsigned long)bw)
1269	avg -= (avg - old) >> 3;
1270
1271	if (avg < old && old <= (unsigned long)bw)
1272	avg += (old - avg) >> 3;
1273
1274	out:
1275	/* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */
1276	avg = max(avg, 1LU);
1277	if (wb_has_dirty_io(wb)) {
1278	long delta = avg - wb->avg_write_bandwidth;
1279	WARN_ON_ONCE(atomic_long_add_return(delta,
1280	&wb->bdi->tot_write_bandwidth) <= 0);
1281	}
1282	wb->write_bandwidth = bw;
1283	WRITE_ONCE(wb->avg_write_bandwidth, avg);
1284	}
1285
1286	static void update_dirty_limit(struct dirty_throttle_control *dtc)
1287	{
1288	struct wb_domain *dom = dtc_dom(dtc);
1289	unsigned long thresh = dtc->thresh;
1290	unsigned long limit = dom->dirty_limit;
1291
1292	/*
1293	* Follow up in one step.
1294	*/
1295	if (limit < thresh) {
1296	limit = thresh;
1297	goto update;
1298	}
1299
1300	/*
1301	* Follow down slowly. Use the higher one as the target, because thresh
1302	* may drop below dirty. This is exactly the reason to introduce
1303	* dom->dirty_limit which is guaranteed to lie above the dirty pages.
1304	*/
1305	thresh = max(thresh, dtc->dirty);
1306	if (limit > thresh) {
1307	limit -= (limit - thresh) >> 5;
1308	goto update;
1309	}
1310	return;
1311	update:
1312	dom->dirty_limit = limit;
1313	}
1314
1315	static void domain_update_dirty_limit(struct dirty_throttle_control *dtc,
1316	unsigned long now)
1317	{
1318	struct wb_domain *dom = dtc_dom(dtc);
1319
1320	/*
1321	* check locklessly first to optimize away locking for the most time
1322	*/
1323	if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL))
1324	return;
1325
1326	spin_lock(lock: &dom->lock);
1327	if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) {
1328	update_dirty_limit(dtc);
1329	dom->dirty_limit_tstamp = now;
1330	}
1331	spin_unlock(lock: &dom->lock);
1332	}
1333
1334	/*
1335	* Maintain wb->dirty_ratelimit, the base dirty throttle rate.
1336	*
1337	* Normal wb tasks will be curbed at or below it in long term.
1338	* Obviously it should be around (write_bw / N) when there are N dd tasks.
1339	*/
1340	static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc,
1341	unsigned long dirtied,
1342	unsigned long elapsed)
1343	{
1344	struct bdi_writeback *wb = dtc->wb;
1345	unsigned long dirty = dtc->dirty;
1346	unsigned long freerun = dirty_freerun_ceiling(thresh: dtc->thresh, bg_thresh: dtc->bg_thresh);
1347	unsigned long limit = hard_dirty_limit(dom: dtc_dom(dtc), thresh: dtc->thresh);
1348	unsigned long setpoint = (freerun + limit) / 2;
1349	unsigned long write_bw = wb->avg_write_bandwidth;
1350	unsigned long dirty_ratelimit = wb->dirty_ratelimit;
1351	unsigned long dirty_rate;
1352	unsigned long task_ratelimit;
1353	unsigned long balanced_dirty_ratelimit;
1354	unsigned long step;
1355	unsigned long x;
1356	unsigned long shift;
1357
1358	/*
1359	* The dirty rate will match the writeout rate in long term, except
1360	* when dirty pages are truncated by userspace or re-dirtied by FS.
1361	*/
1362	dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed;
1363
1364	/*
1365	* task_ratelimit reflects each dd's dirty rate for the past 200ms.
1366	*/
1367	task_ratelimit = (u64)dirty_ratelimit *
1368	dtc->pos_ratio >> RATELIMIT_CALC_SHIFT;
1369	task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */
1370
1371	/*
1372	* A linear estimation of the "balanced" throttle rate. The theory is,
1373	* if there are N dd tasks, each throttled at task_ratelimit, the wb's
1374	* dirty_rate will be measured to be (N * task_ratelimit). So the below
1375	* formula will yield the balanced rate limit (write_bw / N).
1376	*
1377	* Note that the expanded form is not a pure rate feedback:
1378	* rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1)
1379	* but also takes pos_ratio into account:
1380	* rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2)
1381	*
1382	* (1) is not realistic because pos_ratio also takes part in balancing
1383	* the dirty rate. Consider the state
1384	* pos_ratio = 0.5 (3)
1385	* rate = 2 * (write_bw / N) (4)
1386	* If (1) is used, it will stuck in that state! Because each dd will
1387	* be throttled at
1388	* task_ratelimit = pos_ratio * rate = (write_bw / N) (5)
1389	* yielding
1390	* dirty_rate = N * task_ratelimit = write_bw (6)
1391	* put (6) into (1) we get
1392	* rate_(i+1) = rate_(i) (7)
1393	*
1394	* So we end up using (2) to always keep
1395	* rate_(i+1) ~= (write_bw / N) (8)
1396	* regardless of the value of pos_ratio. As long as (8) is satisfied,
1397	* pos_ratio is able to drive itself to 1.0, which is not only where
1398	* the dirty count meet the setpoint, but also where the slope of
1399	* pos_ratio is most flat and hence task_ratelimit is least fluctuated.
1400	*/
1401	balanced_dirty_ratelimit = div_u64(dividend: (u64)task_ratelimit * write_bw,
1402	divisor: dirty_rate | 1);
1403	/*
1404	* balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw
1405	*/
1406	if (unlikely(balanced_dirty_ratelimit > write_bw))
1407	balanced_dirty_ratelimit = write_bw;
1408
1409	/*
1410	* We could safely do this and return immediately:
1411	*
1412	* wb->dirty_ratelimit = balanced_dirty_ratelimit;
1413	*
1414	* However to get a more stable dirty_ratelimit, the below elaborated
1415	* code makes use of task_ratelimit to filter out singular points and
1416	* limit the step size.
1417	*
1418	* The below code essentially only uses the relative value of
1419	*
1420	* task_ratelimit - dirty_ratelimit
1421	* = (pos_ratio - 1) * dirty_ratelimit
1422	*
1423	* which reflects the direction and size of dirty position error.
1424	*/
1425
1426	/*
1427	* dirty_ratelimit will follow balanced_dirty_ratelimit iff
1428	* task_ratelimit is on the same side of dirty_ratelimit, too.
1429	* For example, when
1430	* - dirty_ratelimit > balanced_dirty_ratelimit
1431	* - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)
1432	* lowering dirty_ratelimit will help meet both the position and rate
1433	* control targets. Otherwise, don't update dirty_ratelimit if it will
1434	* only help meet the rate target. After all, what the users ultimately
1435	* feel and care are stable dirty rate and small position error.
1436	*
1437	* |task_ratelimit - dirty_ratelimit| is used to limit the step size
1438	* and filter out the singular points of balanced_dirty_ratelimit. Which
1439	* keeps jumping around randomly and can even leap far away at times
1440	* due to the small 200ms estimation period of dirty_rate (we want to
1441	* keep that period small to reduce time lags).
1442	*/
1443	step = 0;
1444
1445	/*
1446	* For strictlimit case, calculations above were based on wb counters
1447	* and limits (starting from pos_ratio = wb_position_ratio() and up to
1448	* balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate).
1449	* Hence, to calculate "step" properly, we have to use wb_dirty as
1450	* "dirty" and wb_setpoint as "setpoint".
1451	*/
1452	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
1453	dirty = dtc->wb_dirty;
1454	setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2;
1455	}
1456
1457	if (dirty < setpoint) {
1458	x = min3(wb->balanced_dirty_ratelimit,
1459	balanced_dirty_ratelimit, task_ratelimit);
1460	if (dirty_ratelimit < x)
1461	step = x - dirty_ratelimit;
1462	} else {
1463	x = max3(wb->balanced_dirty_ratelimit,
1464	balanced_dirty_ratelimit, task_ratelimit);
1465	if (dirty_ratelimit > x)
1466	step = dirty_ratelimit - x;
1467	}
1468
1469	/*
1470	* Don't pursue 100% rate matching. It's impossible since the balanced
1471	* rate itself is constantly fluctuating. So decrease the track speed
1472	* when it gets close to the target. Helps eliminate pointless tremors.
1473	*/
1474	shift = dirty_ratelimit / (2 * step + 1);
1475	if (shift < BITS_PER_LONG)
1476	step = DIV_ROUND_UP(step >> shift, 8);
1477	else
1478	step = 0;
1479
1480	if (dirty_ratelimit < balanced_dirty_ratelimit)
1481	dirty_ratelimit += step;
1482	else
1483	dirty_ratelimit -= step;
1484
1485	WRITE_ONCE(wb->dirty_ratelimit, max(dirty_ratelimit, 1UL));
1486	wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit;
1487
1488	trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit);
1489	}
1490
1491	static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc,
1492	struct dirty_throttle_control *mdtc,
1493	bool update_ratelimit)
1494	{
1495	struct bdi_writeback *wb = gdtc->wb;
1496	unsigned long now = jiffies;
1497	unsigned long elapsed;
1498	unsigned long dirtied;
1499	unsigned long written;
1500
1501	spin_lock(lock: &wb->list_lock);
1502
1503	/*
1504	* Lockless checks for elapsed time are racy and delayed update after
1505	* IO completion doesn't do it at all (to make sure written pages are
1506	* accounted reasonably quickly). Make sure elapsed >= 1 to avoid
1507	* division errors.
1508	*/
1509	elapsed = max(now - wb->bw_time_stamp, 1UL);
1510	dirtied = percpu_counter_read(fbc: &wb->stat[WB_DIRTIED]);
1511	written = percpu_counter_read(fbc: &wb->stat[WB_WRITTEN]);
1512
1513	if (update_ratelimit) {
1514	domain_update_dirty_limit(dtc: gdtc, now);
1515	wb_update_dirty_ratelimit(dtc: gdtc, dirtied, elapsed);
1516
1517	/*
1518	* @mdtc is always NULL if !CGROUP_WRITEBACK but the
1519	* compiler has no way to figure that out. Help it.
1520	*/
1521	if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) {
1522	domain_update_dirty_limit(dtc: mdtc, now);
1523	wb_update_dirty_ratelimit(dtc: mdtc, dirtied, elapsed);
1524	}
1525	}
1526	wb_update_write_bandwidth(wb, elapsed, written);
1527
1528	wb->dirtied_stamp = dirtied;
1529	wb->written_stamp = written;
1530	WRITE_ONCE(wb->bw_time_stamp, now);
1531	spin_unlock(lock: &wb->list_lock);
1532	}
1533
1534	void wb_update_bandwidth(struct bdi_writeback *wb)
1535	{
1536	struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };
1537
1538	__wb_update_bandwidth(gdtc: &gdtc, NULL, update_ratelimit: false);
1539	}
1540
1541	/* Interval after which we consider wb idle and don't estimate bandwidth */
1542	#define WB_BANDWIDTH_IDLE_JIF (HZ)
1543
1544	static void wb_bandwidth_estimate_start(struct bdi_writeback *wb)
1545	{
1546	unsigned long now = jiffies;
1547	unsigned long elapsed = now - READ_ONCE(wb->bw_time_stamp);
1548
1549	if (elapsed > WB_BANDWIDTH_IDLE_JIF &&
1550	!atomic_read(v: &wb->writeback_inodes)) {
1551	spin_lock(lock: &wb->list_lock);
1552	wb->dirtied_stamp = wb_stat(wb, item: WB_DIRTIED);
1553	wb->written_stamp = wb_stat(wb, item: WB_WRITTEN);
1554	WRITE_ONCE(wb->bw_time_stamp, now);
1555	spin_unlock(lock: &wb->list_lock);
1556	}
1557	}
1558
1559	/*
1560	* After a task dirtied this many pages, balance_dirty_pages_ratelimited()
1561	* will look to see if it needs to start dirty throttling.
1562	*
1563	* If dirty_poll_interval is too low, big NUMA machines will call the expensive
1564	* global_zone_page_state() too often. So scale it near-sqrt to the safety margin
1565	* (the number of pages we may dirty without exceeding the dirty limits).
1566	*/
1567	static unsigned long dirty_poll_interval(unsigned long dirty,
1568	unsigned long thresh)
1569	{
1570	if (thresh > dirty)
1571	return 1UL << (ilog2(thresh - dirty) >> 1);
1572
1573	return 1;
1574	}
1575
1576	static unsigned long wb_max_pause(struct bdi_writeback *wb,
1577	unsigned long wb_dirty)
1578	{
1579	unsigned long bw = READ_ONCE(wb->avg_write_bandwidth);
1580	unsigned long t;
1581
1582	/*
1583	* Limit pause time for small memory systems. If sleeping for too long
1584	* time, a small pool of dirty/writeback pages may go empty and disk go
1585	* idle.
1586	*
1587	* 8 serves as the safety ratio.
1588	*/
1589	t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));
1590	t++;
1591
1592	return min_t(unsigned long, t, MAX_PAUSE);
1593	}
1594
1595	static long wb_min_pause(struct bdi_writeback *wb,
1596	long max_pause,
1597	unsigned long task_ratelimit,
1598	unsigned long dirty_ratelimit,
1599	int *nr_dirtied_pause)
1600	{
1601	long hi = ilog2(READ_ONCE(wb->avg_write_bandwidth));
1602	long lo = ilog2(READ_ONCE(wb->dirty_ratelimit));
1603	long t; /* target pause */
1604	long pause; /* estimated next pause */
1605	int pages; /* target nr_dirtied_pause */
1606
1607	/* target for 10ms pause on 1-dd case */
1608	t = max(1, HZ / 100);
1609
1610	/*
1611	* Scale up pause time for concurrent dirtiers in order to reduce CPU
1612	* overheads.
1613	*
1614	* (N * 10ms) on 2^N concurrent tasks.
1615	*/
1616	if (hi > lo)
1617	t += (hi - lo) * (10 * HZ) / 1024;
1618
1619	/*
1620	* This is a bit convoluted. We try to base the next nr_dirtied_pause
1621	* on the much more stable dirty_ratelimit. However the next pause time
1622	* will be computed based on task_ratelimit and the two rate limits may
1623	* depart considerably at some time. Especially if task_ratelimit goes
1624	* below dirty_ratelimit/2 and the target pause is max_pause, the next
1625	* pause time will be max_pause*2 _trimmed down_ to max_pause. As a
1626	* result task_ratelimit won't be executed faithfully, which could
1627	* eventually bring down dirty_ratelimit.
1628	*
1629	* We apply two rules to fix it up:
1630	* 1) try to estimate the next pause time and if necessary, use a lower
1631	* nr_dirtied_pause so as not to exceed max_pause. When this happens,
1632	* nr_dirtied_pause will be "dancing" with task_ratelimit.
1633	* 2) limit the target pause time to max_pause/2, so that the normal
1634	* small fluctuations of task_ratelimit won't trigger rule (1) and
1635	* nr_dirtied_pause will remain as stable as dirty_ratelimit.
1636	*/
1637	t = min(t, 1 + max_pause / 2);
1638	pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1639
1640	/*
1641	* Tiny nr_dirtied_pause is found to hurt I/O performance in the test
1642	* case fio-mmap-randwrite-64k, which does 16*{sync read, async write}.
1643	* When the 16 consecutive reads are often interrupted by some dirty
1644	* throttling pause during the async writes, cfq will go into idles
1645	* (deadline is fine). So push nr_dirtied_pause as high as possible
1646	* until reaches DIRTY_POLL_THRESH=32 pages.
1647	*/
1648	if (pages < DIRTY_POLL_THRESH) {
1649	t = max_pause;
1650	pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1651	if (pages > DIRTY_POLL_THRESH) {
1652	pages = DIRTY_POLL_THRESH;
1653	t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
1654	}
1655	}
1656
1657	pause = HZ * pages / (task_ratelimit + 1);
1658	if (pause > max_pause) {
1659	t = max_pause;
1660	pages = task_ratelimit * t / roundup_pow_of_two(HZ);
1661	}
1662
1663	*nr_dirtied_pause = pages;
1664	/*
1665	* The minimal pause time will normally be half the target pause time.
1666	*/
1667	return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;
1668	}
1669
1670	static inline void wb_dirty_limits(struct dirty_throttle_control *dtc)
1671	{
1672	struct bdi_writeback *wb = dtc->wb;
1673	unsigned long wb_reclaimable;
1674
1675	/*
1676	* wb_thresh is not treated as some limiting factor as
1677	* dirty_thresh, due to reasons
1678	* - in JBOD setup, wb_thresh can fluctuate a lot
1679	* - in a system with HDD and USB key, the USB key may somehow
1680	* go into state (wb_dirty >> wb_thresh) either because
1681	* wb_dirty starts high, or because wb_thresh drops low.
1682	* In this case we don't want to hard throttle the USB key
1683	* dirtiers for 100 seconds until wb_dirty drops under
1684	* wb_thresh. Instead the auxiliary wb control line in
1685	* wb_position_ratio() will let the dirtier task progress
1686	* at some rate <= (write_bw / 2) for bringing down wb_dirty.
1687	*/
1688	dtc->wb_thresh = __wb_calc_thresh(dtc, thresh: dtc->thresh);
1689	dtc->wb_bg_thresh = dtc->thresh ?
1690	div_u64(dividend: (u64)dtc->wb_thresh * dtc->bg_thresh, divisor: dtc->thresh) : 0;
1691
1692	/*
1693	* In order to avoid the stacked BDI deadlock we need
1694	* to ensure we accurately count the 'dirty' pages when
1695	* the threshold is low.
1696	*
1697	* Otherwise it would be possible to get thresh+n pages
1698	* reported dirty, even though there are thresh-m pages
1699	* actually dirty; with m+n sitting in the percpu
1700	* deltas.
1701	*/
1702	if (dtc->wb_thresh < 2 * wb_stat_error()) {
1703	wb_reclaimable = wb_stat_sum(wb, item: WB_RECLAIMABLE);
1704	dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, item: WB_WRITEBACK);
1705	} else {
1706	wb_reclaimable = wb_stat(wb, item: WB_RECLAIMABLE);
1707	dtc->wb_dirty = wb_reclaimable + wb_stat(wb, item: WB_WRITEBACK);
1708	}
1709	}
1710
1711	static unsigned long domain_poll_intv(struct dirty_throttle_control *dtc,
1712	bool strictlimit)
1713	{
1714	unsigned long dirty, thresh;
1715
1716	if (strictlimit) {
1717	dirty = dtc->wb_dirty;
1718	thresh = dtc->wb_thresh;
1719	} else {
1720	dirty = dtc->dirty;
1721	thresh = dtc->thresh;
1722	}
1723
1724	return dirty_poll_interval(dirty, thresh);
1725	}
1726
1727	/*
1728	* Throttle it only when the background writeback cannot catch-up. This avoids
1729	* (excessively) small writeouts when the wb limits are ramping up in case of
1730	* !strictlimit.
1731	*
1732	* In strictlimit case make decision based on the wb counters and limits. Small
1733	* writeouts when the wb limits are ramping up are the price we consciously pay
1734	* for strictlimit-ing.
1735	*/
1736	static void domain_dirty_freerun(struct dirty_throttle_control *dtc,
1737	bool strictlimit)
1738	{
1739	unsigned long dirty, thresh, bg_thresh;
1740
1741	if (unlikely(strictlimit)) {
1742	wb_dirty_limits(dtc);
1743	dirty = dtc->wb_dirty;
1744	thresh = dtc->wb_thresh;
1745	bg_thresh = dtc->wb_bg_thresh;
1746	} else {
1747	dirty = dtc->dirty;
1748	thresh = dtc->thresh;
1749	bg_thresh = dtc->bg_thresh;
1750	}
1751	dtc->freerun = dirty <= dirty_freerun_ceiling(thresh, bg_thresh);
1752	}
1753
1754	static void balance_domain_limits(struct dirty_throttle_control *dtc,
1755	bool strictlimit)
1756	{
1757	domain_dirty_avail(dtc, include_writeback: true);
1758	domain_dirty_limits(dtc);
1759	domain_dirty_freerun(dtc, strictlimit);
1760	}
1761
1762	static void wb_dirty_freerun(struct dirty_throttle_control *dtc,
1763	bool strictlimit)
1764	{
1765	dtc->freerun = false;
1766
1767	/* was already handled in domain_dirty_freerun */
1768	if (strictlimit)
1769	return;
1770
1771	wb_dirty_limits(dtc);
1772	/*
1773	* LOCAL_THROTTLE tasks must not be throttled when below the per-wb
1774	* freerun ceiling.
1775	*/
1776	if (!(current->flags & PF_LOCAL_THROTTLE))
1777	return;
1778
1779	dtc->freerun = dtc->wb_dirty <
1780	dirty_freerun_ceiling(thresh: dtc->wb_thresh, bg_thresh: dtc->wb_bg_thresh);
1781	}
1782
1783	static inline void wb_dirty_exceeded(struct dirty_throttle_control *dtc,
1784	bool strictlimit)
1785	{
1786	dtc->dirty_exceeded = (dtc->wb_dirty > dtc->wb_thresh) &&
1787	((dtc->dirty > dtc->thresh) || strictlimit);
1788	}
1789
1790	/*
1791	* The limits fields dirty_exceeded and pos_ratio won't be updated if wb is
1792	* in freerun state. Please don't use these invalid fields in freerun case.
1793	*/
1794	static void balance_wb_limits(struct dirty_throttle_control *dtc,
1795	bool strictlimit)
1796	{
1797	wb_dirty_freerun(dtc, strictlimit);
1798	if (dtc->freerun)
1799	return;
1800
1801	wb_dirty_exceeded(dtc, strictlimit);
1802	wb_position_ratio(dtc);
1803	}
1804
1805	/*
1806	* balance_dirty_pages() must be called by processes which are generating dirty
1807	* data. It looks at the number of dirty pages in the machine and will force
1808	* the caller to wait once crossing the (background_thresh + dirty_thresh) / 2.
1809	* If we're over `background_thresh' then the writeback threads are woken to
1810	* perform some writeout.
1811	*/
1812	static int balance_dirty_pages(struct bdi_writeback *wb,
1813	unsigned long pages_dirtied, unsigned int flags)
1814	{
1815	struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
1816	struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
1817	struct dirty_throttle_control * const gdtc = &gdtc_stor;
1818	struct dirty_throttle_control * const mdtc = mdtc_valid(dtc: &mdtc_stor) ?
1819	&mdtc_stor : NULL;
1820	struct dirty_throttle_control *sdtc;
1821	unsigned long nr_dirty;
1822	long period;
1823	long pause;
1824	long max_pause;
1825	long min_pause;
1826	int nr_dirtied_pause;
1827	unsigned long task_ratelimit;
1828	unsigned long dirty_ratelimit;
1829	struct backing_dev_info *bdi = wb->bdi;
1830	bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT;
1831	unsigned long start_time = jiffies;
1832	int ret = 0;
1833
1834	for (;;) {
1835	unsigned long now = jiffies;
1836
1837	nr_dirty = global_node_page_state(item: NR_FILE_DIRTY);
1838
1839	balance_domain_limits(dtc: gdtc, strictlimit);
1840	if (mdtc) {
1841	/*
1842	* If @wb belongs to !root memcg, repeat the same
1843	* basic calculations for the memcg domain.
1844	*/
1845	balance_domain_limits(dtc: mdtc, strictlimit);
1846	}
1847
1848	/*
1849	* In laptop mode, we wait until hitting the higher threshold
1850	* before starting background writeout, and then write out all
1851	* the way down to the lower threshold. So slow writers cause
1852	* minimal disk activity.
1853	*
1854	* In normal mode, we start background writeout at the lower
1855	* background_thresh, to keep the amount of dirty memory low.
1856	*/
1857	if (!laptop_mode && nr_dirty > gdtc->bg_thresh &&
1858	!writeback_in_progress(wb))
1859	wb_start_background_writeback(wb);
1860
1861	/*
1862	* If memcg domain is in effect, @dirty should be under
1863	* both global and memcg freerun ceilings.
1864	*/
1865	if (gdtc->freerun && (!mdtc || mdtc->freerun)) {
1866	unsigned long intv;
1867	unsigned long m_intv;
1868
1869	free_running:
1870	intv = domain_poll_intv(dtc: gdtc, strictlimit);
1871	m_intv = ULONG_MAX;
1872
1873	current->dirty_paused_when = now;
1874	current->nr_dirtied = 0;
1875	if (mdtc)
1876	m_intv = domain_poll_intv(dtc: mdtc, strictlimit);
1877	current->nr_dirtied_pause = min(intv, m_intv);
1878	break;
1879	}
1880
1881	/* Start writeback even when in laptop mode */
1882	if (unlikely(!writeback_in_progress(wb)))
1883	wb_start_background_writeback(wb);
1884
1885	mem_cgroup_flush_foreign(wb);
1886
1887	/*
1888	* Calculate global domain's pos_ratio and select the
1889	* global dtc by default.
1890	*/
1891	balance_wb_limits(dtc: gdtc, strictlimit);
1892	if (gdtc->freerun)
1893	goto free_running;
1894	sdtc = gdtc;
1895
1896	if (mdtc) {
1897	/*
1898	* If memcg domain is in effect, calculate its
1899	* pos_ratio. @wb should satisfy constraints from
1900	* both global and memcg domains. Choose the one
1901	* w/ lower pos_ratio.
1902	*/
1903	balance_wb_limits(dtc: mdtc, strictlimit);
1904	if (mdtc->freerun)
1905	goto free_running;
1906	if (mdtc->pos_ratio < gdtc->pos_ratio)
1907	sdtc = mdtc;
1908	}
1909
1910	wb->dirty_exceeded = gdtc->dirty_exceeded ||
1911	(mdtc && mdtc->dirty_exceeded);
1912	if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
1913	BANDWIDTH_INTERVAL))
1914	__wb_update_bandwidth(gdtc, mdtc, update_ratelimit: true);
1915
1916	/* throttle according to the chosen dtc */
1917	dirty_ratelimit = READ_ONCE(wb->dirty_ratelimit);
1918	task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >>
1919	RATELIMIT_CALC_SHIFT;
1920	max_pause = wb_max_pause(wb, wb_dirty: sdtc->wb_dirty);
1921	min_pause = wb_min_pause(wb, max_pause,
1922	task_ratelimit, dirty_ratelimit,
1923	nr_dirtied_pause: &nr_dirtied_pause);
1924
1925	if (unlikely(task_ratelimit == 0)) {
1926	period = max_pause;
1927	pause = max_pause;
1928	goto pause;
1929	}
1930	period = HZ * pages_dirtied / task_ratelimit;
1931	pause = period;
1932	if (current->dirty_paused_when)
1933	pause -= now - current->dirty_paused_when;
1934	/*
1935	* For less than 1s think time (ext3/4 may block the dirtier
1936	* for up to 800ms from time to time on 1-HDD; so does xfs,
1937	* however at much less frequency), try to compensate it in
1938	* future periods by updating the virtual time; otherwise just
1939	* do a reset, as it may be a light dirtier.
1940	*/
1941	if (pause < min_pause) {
1942	trace_balance_dirty_pages(wb,
1943	dtc: sdtc,
1944	dirty_ratelimit,
1945	task_ratelimit,
1946	dirtied: pages_dirtied,
1947	period,
1948	min(pause, 0L),
1949	start_time);
1950	if (pause < -HZ) {
1951	current->dirty_paused_when = now;
1952	current->nr_dirtied = 0;
1953	} else if (period) {
1954	current->dirty_paused_when += period;
1955	current->nr_dirtied = 0;
1956	} else if (current->nr_dirtied_pause <= pages_dirtied)
1957	current->nr_dirtied_pause += pages_dirtied;
1958	break;
1959	}
1960	if (unlikely(pause > max_pause)) {
1961	/* for occasional dropped task_ratelimit */
1962	now += min(pause - max_pause, max_pause);
1963	pause = max_pause;
1964	}
1965
1966	pause:
1967	trace_balance_dirty_pages(wb,
1968	dtc: sdtc,
1969	dirty_ratelimit,
1970	task_ratelimit,
1971	dirtied: pages_dirtied,
1972	period,
1973	pause,
1974	start_time);
1975	if (flags & BDP_ASYNC) {
1976	ret = -EAGAIN;
1977	break;
1978	}
1979	__set_current_state(TASK_KILLABLE);
1980	bdi->last_bdp_sleep = jiffies;
1981	io_schedule_timeout(timeout: pause);
1982
1983	current->dirty_paused_when = now + pause;
1984	current->nr_dirtied = 0;
1985	current->nr_dirtied_pause = nr_dirtied_pause;
1986
1987	/*
1988	* This is typically equal to (dirty < thresh) and can also
1989	* keep "1000+ dd on a slow USB stick" under control.
1990	*/
1991	if (task_ratelimit)
1992	break;
1993
1994	/*
1995	* In the case of an unresponsive NFS server and the NFS dirty
1996	* pages exceeds dirty_thresh, give the other good wb's a pipe
1997	* to go through, so that tasks on them still remain responsive.
1998	*
1999	* In theory 1 page is enough to keep the consumer-producer
2000	* pipe going: the flusher cleans 1 page => the task dirties 1
2001	* more page. However wb_dirty has accounting errors. So use
2002	* the larger and more IO friendly wb_stat_error.
2003	*/
2004	if (sdtc->wb_dirty <= wb_stat_error())
2005	break;
2006
2007	if (fatal_signal_pending(current))
2008	break;
2009	}
2010	return ret;
2011	}
2012
2013	static DEFINE_PER_CPU(int, bdp_ratelimits);
2014
2015	/*
2016	* Normal tasks are throttled by
2017	* loop {
2018	* dirty tsk->nr_dirtied_pause pages;
2019	* take a snap in balance_dirty_pages();
2020	* }
2021	* However there is a worst case. If every task exit immediately when dirtied
2022	* (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
2023	* called to throttle the page dirties. The solution is to save the not yet
2024	* throttled page dirties in dirty_throttle_leaks on task exit and charge them
2025	* randomly into the running tasks. This works well for the above worst case,
2026	* as the new task will pick up and accumulate the old task's leaked dirty
2027	* count and eventually get throttled.
2028	*/
2029	DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;
2030
2031	/**
2032	* balance_dirty_pages_ratelimited_flags - Balance dirty memory state.
2033	* @mapping: address_space which was dirtied.
2034	* @flags: BDP flags.
2035	*
2036	* Processes which are dirtying memory should call in here once for each page
2037	* which was newly dirtied. The function will periodically check the system's
2038	* dirty state and will initiate writeback if needed.
2039	*
2040	* See balance_dirty_pages_ratelimited() for details.
2041	*
2042	* Return: If @flags contains BDP_ASYNC, it may return -EAGAIN to
2043	* indicate that memory is out of balance and the caller must wait
2044	* for I/O to complete. Otherwise, it will return 0 to indicate
2045	* that either memory was already in balance, or it was able to sleep
2046	* until the amount of dirty memory returned to balance.
2047	*/
2048	int balance_dirty_pages_ratelimited_flags(struct address_space *mapping,
2049	unsigned int flags)
2050	{
2051	struct inode *inode = mapping->host;
2052	struct backing_dev_info *bdi = inode_to_bdi(inode);
2053	struct bdi_writeback *wb = NULL;
2054	int ratelimit;
2055	int ret = 0;
2056	int *p;
2057
2058	if (!(bdi->capabilities & BDI_CAP_WRITEBACK))
2059	return ret;
2060
2061	if (inode_cgwb_enabled(inode))
2062	wb = wb_get_create_current(bdi, GFP_KERNEL);
2063	if (!wb)
2064	wb = &bdi->wb;
2065
2066	ratelimit = current->nr_dirtied_pause;
2067	if (wb->dirty_exceeded)
2068	ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10));
2069
2070	preempt_disable();
2071	/*
2072	* This prevents one CPU to accumulate too many dirtied pages without
2073	* calling into balance_dirty_pages(), which can happen when there are
2074	* 1000+ tasks, all of them start dirtying pages at exactly the same
2075	* time, hence all honoured too large initial task->nr_dirtied_pause.
2076	*/
2077	p = this_cpu_ptr(&bdp_ratelimits);
2078	if (unlikely(current->nr_dirtied >= ratelimit))
2079	*p = 0;
2080	else if (unlikely(*p >= ratelimit_pages)) {
2081	*p = 0;
2082	ratelimit = 0;
2083	}
2084	/*
2085	* Pick up the dirtied pages by the exited tasks. This avoids lots of
2086	* short-lived tasks (eg. gcc invocations in a kernel build) escaping
2087	* the dirty throttling and livelock other long-run dirtiers.
2088	*/
2089	p = this_cpu_ptr(&dirty_throttle_leaks);
2090	if (*p > 0 && current->nr_dirtied < ratelimit) {
2091	unsigned long nr_pages_dirtied;
2092	nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
2093	*p -= nr_pages_dirtied;
2094	current->nr_dirtied += nr_pages_dirtied;
2095	}
2096	preempt_enable();
2097
2098	if (unlikely(current->nr_dirtied >= ratelimit))
2099	ret = balance_dirty_pages(wb, current->nr_dirtied, flags);
2100
2101	wb_put(wb);
2102	return ret;
2103	}
2104	EXPORT_SYMBOL_GPL(balance_dirty_pages_ratelimited_flags);
2105
2106	/**
2107	* balance_dirty_pages_ratelimited - balance dirty memory state.
2108	* @mapping: address_space which was dirtied.
2109	*
2110	* Processes which are dirtying memory should call in here once for each page
2111	* which was newly dirtied. The function will periodically check the system's
2112	* dirty state and will initiate writeback if needed.
2113	*
2114	* Once we're over the dirty memory limit we decrease the ratelimiting
2115	* by a lot, to prevent individual processes from overshooting the limit
2116	* by (ratelimit_pages) each.
2117	*/
2118	void balance_dirty_pages_ratelimited(struct address_space *mapping)
2119	{
2120	balance_dirty_pages_ratelimited_flags(mapping, 0);
2121	}
2122	EXPORT_SYMBOL(balance_dirty_pages_ratelimited);
2123
2124	/*
2125	* Similar to wb_dirty_limits, wb_bg_dirty_limits also calculates dirty
2126	* and thresh, but it's for background writeback.
2127	*/
2128	static void wb_bg_dirty_limits(struct dirty_throttle_control *dtc)
2129	{
2130	struct bdi_writeback *wb = dtc->wb;
2131
2132	dtc->wb_bg_thresh = __wb_calc_thresh(dtc, thresh: dtc->bg_thresh);
2133	if (dtc->wb_bg_thresh < 2 * wb_stat_error())
2134	dtc->wb_dirty = wb_stat_sum(wb, item: WB_RECLAIMABLE);
2135	else
2136	dtc->wb_dirty = wb_stat(wb, item: WB_RECLAIMABLE);
2137	}
2138
2139	static bool domain_over_bg_thresh(struct dirty_throttle_control *dtc)
2140	{
2141	domain_dirty_avail(dtc, include_writeback: false);
2142	domain_dirty_limits(dtc);
2143	if (dtc->dirty > dtc->bg_thresh)
2144	return true;
2145
2146	wb_bg_dirty_limits(dtc);
2147	if (dtc->wb_dirty > dtc->wb_bg_thresh)
2148	return true;
2149
2150	return false;
2151	}
2152
2153	/**
2154	* wb_over_bg_thresh - does @wb need to be written back?
2155	* @wb: bdi_writeback of interest
2156	*
2157	* Determines whether background writeback should keep writing @wb or it's
2158	* clean enough.
2159	*
2160	* Return: %true if writeback should continue.
2161	*/
2162	bool wb_over_bg_thresh(struct bdi_writeback *wb)
2163	{
2164	struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };
2165	struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) };
2166
2167	if (domain_over_bg_thresh(dtc: &gdtc))
2168	return true;
2169
2170	if (mdtc_valid(dtc: &mdtc))
2171	return domain_over_bg_thresh(dtc: &mdtc);
2172
2173	return false;
2174	}
2175
2176	#ifdef CONFIG_SYSCTL
2177	/*
2178	* sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
2179	*/
2180	static int dirty_writeback_centisecs_handler(const struct ctl_table *table, int write,
2181	void *buffer, size_t *length, loff_t *ppos)
2182	{
2183	unsigned int old_interval = dirty_writeback_interval;
2184	int ret;
2185
2186	ret = proc_dointvec(table, write, buffer, length, ppos);
2187
2188	/*
2189	* Writing 0 to dirty_writeback_interval will disable periodic writeback
2190	* and a different non-zero value will wakeup the writeback threads.
2191	* wb_wakeup_delayed() would be more appropriate, but it's a pain to
2192	* iterate over all bdis and wbs.
2193	* The reason we do this is to make the change take effect immediately.
2194	*/
2195	if (!ret && write && dirty_writeback_interval &&
2196	dirty_writeback_interval != old_interval)
2197	wakeup_flusher_threads(reason: WB_REASON_PERIODIC);
2198
2199	return ret;
2200	}
2201	#endif
2202
2203	void laptop_mode_timer_fn(struct timer_list *t)
2204	{
2205	struct backing_dev_info *backing_dev_info =
2206	timer_container_of(backing_dev_info, t, laptop_mode_wb_timer);
2207
2208	wakeup_flusher_threads_bdi(bdi: backing_dev_info, reason: WB_REASON_LAPTOP_TIMER);
2209	}
2210
2211	/*
2212	* We've spun up the disk and we're in laptop mode: schedule writeback
2213	* of all dirty data a few seconds from now. If the flush is already scheduled
2214	* then push it back - the user is still using the disk.
2215	*/
2216	void laptop_io_completion(struct backing_dev_info *info)
2217	{
2218	mod_timer(timer: &info->laptop_mode_wb_timer, expires: jiffies + laptop_mode);
2219	}
2220
2221	/*
2222	* We're in laptop mode and we've just synced. The sync's writes will have
2223	* caused another writeback to be scheduled by laptop_io_completion.
2224	* Nothing needs to be written back anymore, so we unschedule the writeback.
2225	*/
2226	void laptop_sync_completion(void)
2227	{
2228	struct backing_dev_info *bdi;
2229
2230	rcu_read_lock();
2231
2232	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
2233	timer_delete(timer: &bdi->laptop_mode_wb_timer);
2234
2235	rcu_read_unlock();
2236	}
2237
2238	/*
2239	* If ratelimit_pages is too high then we can get into dirty-data overload
2240	* if a large number of processes all perform writes at the same time.
2241	*
2242	* Here we set ratelimit_pages to a level which ensures that when all CPUs are
2243	* dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
2244	* thresholds.
2245	*/
2246
2247	void writeback_set_ratelimit(void)
2248	{
2249	struct wb_domain *dom = &global_wb_domain;
2250	unsigned long background_thresh;
2251	unsigned long dirty_thresh;
2252
2253	global_dirty_limits(pbackground: &background_thresh, pdirty: &dirty_thresh);
2254	dom->dirty_limit = dirty_thresh;
2255	ratelimit_pages = dirty_thresh / (num_online_cpus() * 32);
2256	if (ratelimit_pages < 16)
2257	ratelimit_pages = 16;
2258	}
2259
2260	static int page_writeback_cpu_online(unsigned int cpu)
2261	{
2262	writeback_set_ratelimit();
2263	return 0;
2264	}
2265
2266	#ifdef CONFIG_SYSCTL
2267
2268	/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */
2269	static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE;
2270
2271	static const struct ctl_table vm_page_writeback_sysctls[] = {
2272	{
2273	.procname = "dirty_background_ratio",
2274	.data = &dirty_background_ratio,
2275	.maxlen = sizeof(dirty_background_ratio),
2276	.mode = 0644,
2277	.proc_handler = dirty_background_ratio_handler,
2278	.extra1 = SYSCTL_ZERO,
2279	.extra2 = SYSCTL_ONE_HUNDRED,
2280	},
2281	{
2282	.procname = "dirty_background_bytes",
2283	.data = &dirty_background_bytes,
2284	.maxlen = sizeof(dirty_background_bytes),
2285	.mode = 0644,
2286	.proc_handler = dirty_background_bytes_handler,
2287	.extra1 = SYSCTL_LONG_ONE,
2288	},
2289	{
2290	.procname = "dirty_ratio",
2291	.data = &vm_dirty_ratio,
2292	.maxlen = sizeof(vm_dirty_ratio),
2293	.mode = 0644,
2294	.proc_handler = dirty_ratio_handler,
2295	.extra1 = SYSCTL_ZERO,
2296	.extra2 = SYSCTL_ONE_HUNDRED,
2297	},
2298	{
2299	.procname = "dirty_bytes",
2300	.data = &vm_dirty_bytes,
2301	.maxlen = sizeof(vm_dirty_bytes),
2302	.mode = 0644,
2303	.proc_handler = dirty_bytes_handler,
2304	.extra1 = (void *)&dirty_bytes_min,
2305	},
2306	{
2307	.procname = "dirty_writeback_centisecs",
2308	.data = &dirty_writeback_interval,
2309	.maxlen = sizeof(dirty_writeback_interval),
2310	.mode = 0644,
2311	.proc_handler = dirty_writeback_centisecs_handler,
2312	},
2313	{
2314	.procname = "dirty_expire_centisecs",
2315	.data = &dirty_expire_interval,
2316	.maxlen = sizeof(dirty_expire_interval),
2317	.mode = 0644,
2318	.proc_handler = proc_dointvec_minmax,
2319	.extra1 = SYSCTL_ZERO,
2320	},
2321	#ifdef CONFIG_HIGHMEM
2322	{
2323	.procname = "highmem_is_dirtyable",
2324	.data = &vm_highmem_is_dirtyable,
2325	.maxlen = sizeof(vm_highmem_is_dirtyable),
2326	.mode = 0644,
2327	.proc_handler = proc_dointvec_minmax,
2328	.extra1 = SYSCTL_ZERO,
2329	.extra2 = SYSCTL_ONE,
2330	},
2331	#endif
2332	{
2333	.procname = "laptop_mode",
2334	.data = &laptop_mode,
2335	.maxlen = sizeof(laptop_mode),
2336	.mode = 0644,
2337	.proc_handler = proc_dointvec_jiffies,
2338	},
2339	};
2340	#endif
2341
2342	/*
2343	* Called early on to tune the page writeback dirty limits.
2344	*
2345	* We used to scale dirty pages according to how total memory
2346	* related to pages that could be allocated for buffers.
2347	*
2348	* However, that was when we used "dirty_ratio" to scale with
2349	* all memory, and we don't do that any more. "dirty_ratio"
2350	* is now applied to total non-HIGHPAGE memory, and as such we can't
2351	* get into the old insane situation any more where we had
2352	* large amounts of dirty pages compared to a small amount of
2353	* non-HIGHMEM memory.
2354	*
2355	* But we might still want to scale the dirty_ratio by how
2356	* much memory the box has..
2357	*/
2358	void __init page_writeback_init(void)
2359	{
2360	BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL));
2361
2362	cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "mm/writeback:online",
2363	startup: page_writeback_cpu_online, NULL);
2364	cpuhp_setup_state(state: CPUHP_MM_WRITEBACK_DEAD, name: "mm/writeback:dead", NULL,
2365	teardown: page_writeback_cpu_online);
2366	#ifdef CONFIG_SYSCTL
2367	register_sysctl_init("vm", vm_page_writeback_sysctls);
2368	#endif
2369	}
2370
2371	/**
2372	* tag_pages_for_writeback - tag pages to be written by writeback
2373	* @mapping: address space structure to write
2374	* @start: starting page index
2375	* @end: ending page index (inclusive)
2376	*
2377	* This function scans the page range from @start to @end (inclusive) and tags
2378	* all pages that have DIRTY tag set with a special TOWRITE tag. The caller
2379	* can then use the TOWRITE tag to identify pages eligible for writeback.
2380	* This mechanism is used to avoid livelocking of writeback by a process
2381	* steadily creating new dirty pages in the file (thus it is important for this
2382	* function to be quick so that it can tag pages faster than a dirtying process
2383	* can create them).
2384	*/
2385	void tag_pages_for_writeback(struct address_space *mapping,
2386	pgoff_t start, pgoff_t end)
2387	{
2388	XA_STATE(xas, &mapping->i_pages, start);
2389	unsigned int tagged = 0;
2390	void *page;
2391
2392	xas_lock_irq(&xas);
2393	xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) {
2394	xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
2395	if (++tagged % XA_CHECK_SCHED)
2396	continue;
2397
2398	xas_pause(&xas);
2399	xas_unlock_irq(&xas);
2400	cond_resched();
2401	xas_lock_irq(&xas);
2402	}
2403	xas_unlock_irq(&xas);
2404	}
2405	EXPORT_SYMBOL(tag_pages_for_writeback);
2406
2407	static bool folio_prepare_writeback(struct address_space *mapping,
2408	struct writeback_control *wbc, struct folio *folio)
2409	{
2410	/*
2411	* Folio truncated or invalidated. We can freely skip it then,
2412	* even for data integrity operations: the folio has disappeared
2413	* concurrently, so there could be no real expectation of this
2414	* data integrity operation even if there is now a new, dirty
2415	* folio at the same pagecache index.
2416	*/
2417	if (unlikely(folio->mapping != mapping))
2418	return false;
2419
2420	/*
2421	* Did somebody else write it for us?
2422	*/
2423	if (!folio_test_dirty(folio))
2424	return false;
2425
2426	if (folio_test_writeback(folio)) {
2427	if (wbc->sync_mode == WB_SYNC_NONE)
2428	return false;
2429	folio_wait_writeback(folio);
2430	}
2431	BUG_ON(folio_test_writeback(folio));
2432
2433	if (!folio_clear_dirty_for_io(folio))
2434	return false;
2435
2436	return true;
2437	}
2438
2439	static xa_mark_t wbc_to_tag(struct writeback_control *wbc)
2440	{
2441	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2442	return PAGECACHE_TAG_TOWRITE;
2443	return PAGECACHE_TAG_DIRTY;
2444	}
2445
2446	static pgoff_t wbc_end(struct writeback_control *wbc)
2447	{
2448	if (wbc->range_cyclic)
2449	return -1;
2450	return wbc->range_end >> PAGE_SHIFT;
2451	}
2452
2453	static struct folio *writeback_get_folio(struct address_space *mapping,
2454	struct writeback_control *wbc)
2455	{
2456	struct folio *folio;
2457
2458	retry:
2459	folio = folio_batch_next(fbatch: &wbc->fbatch);
2460	if (!folio) {
2461	folio_batch_release(fbatch: &wbc->fbatch);
2462	cond_resched();
2463	filemap_get_folios_tag(mapping, start: &wbc->index, end: wbc_end(wbc),
2464	tag: wbc_to_tag(wbc), fbatch: &wbc->fbatch);
2465	folio = folio_batch_next(fbatch: &wbc->fbatch);
2466	if (!folio)
2467	return NULL;
2468	}
2469
2470	folio_lock(folio);
2471	if (unlikely(!folio_prepare_writeback(mapping, wbc, folio))) {
2472	folio_unlock(folio);
2473	goto retry;
2474	}
2475
2476	trace_wbc_writepage(wbc, bdi: inode_to_bdi(inode: mapping->host));
2477	return folio;
2478	}
2479
2480	/**
2481	* writeback_iter - iterate folio of a mapping for writeback
2482	* @mapping: address space structure to write
2483	* @wbc: writeback context
2484	* @folio: previously iterated folio (%NULL to start)
2485	* @error: in-out pointer for writeback errors (see below)
2486	*
2487	* This function returns the next folio for the writeback operation described by
2488	* @wbc on @mapping and should be called in a while loop in the ->writepages
2489	* implementation.
2490	*
2491	* To start the writeback operation, %NULL is passed in the @folio argument, and
2492	* for every subsequent iteration the folio returned previously should be passed
2493	* back in.
2494	*
2495	* If there was an error in the per-folio writeback inside the writeback_iter()
2496	* loop, @error should be set to the error value.
2497	*
2498	* Once the writeback described in @wbc has finished, this function will return
2499	* %NULL and if there was an error in any iteration restore it to @error.
2500	*
2501	* Note: callers should not manually break out of the loop using break or goto
2502	* but must keep calling writeback_iter() until it returns %NULL.
2503	*
2504	* Return: the folio to write or %NULL if the loop is done.
2505	*/
2506	struct folio *writeback_iter(struct address_space *mapping,
2507	struct writeback_control *wbc, struct folio *folio, int *error)
2508	{
2509	if (!folio) {
2510	folio_batch_init(fbatch: &wbc->fbatch);
2511	wbc->saved_err = *error = 0;
2512
2513	/*
2514	* For range cyclic writeback we remember where we stopped so
2515	* that we can continue where we stopped.
2516	*
2517	* For non-cyclic writeback we always start at the beginning of
2518	* the passed in range.
2519	*/
2520	if (wbc->range_cyclic)
2521	wbc->index = mapping->writeback_index;
2522	else
2523	wbc->index = wbc->range_start >> PAGE_SHIFT;
2524
2525	/*
2526	* To avoid livelocks when other processes dirty new pages, we
2527	* first tag pages which should be written back and only then
2528	* start writing them.
2529	*
2530	* For data-integrity writeback we have to be careful so that we
2531	* do not miss some pages (e.g., because some other process has
2532	* cleared the TOWRITE tag we set). The rule we follow is that
2533	* TOWRITE tag can be cleared only by the process clearing the
2534	* DIRTY tag (and submitting the page for I/O).
2535	*/
2536	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2537	tag_pages_for_writeback(mapping, wbc->index,
2538	wbc_end(wbc));
2539	} else {
2540	wbc->nr_to_write -= folio_nr_pages(folio);
2541
2542	WARN_ON_ONCE(*error > 0);
2543
2544	/*
2545	* For integrity writeback we have to keep going until we have
2546	* written all the folios we tagged for writeback above, even if
2547	* we run past wbc->nr_to_write or encounter errors.
2548	* We stash away the first error we encounter in wbc->saved_err
2549	* so that it can be retrieved when we're done. This is because
2550	* the file system may still have state to clear for each folio.
2551	*
2552	* For background writeback we exit as soon as we run past
2553	* wbc->nr_to_write or encounter the first error.
2554	*/
2555	if (wbc->sync_mode == WB_SYNC_ALL) {
2556	if (*error && !wbc->saved_err)
2557	wbc->saved_err = *error;
2558	} else {
2559	if (*error || wbc->nr_to_write <= 0)
2560	goto done;
2561	}
2562	}
2563
2564	folio = writeback_get_folio(mapping, wbc);
2565	if (!folio) {
2566	/*
2567	* To avoid deadlocks between range_cyclic writeback and callers
2568	* that hold folios in writeback to aggregate I/O until
2569	* the writeback iteration finishes, we do not loop back to the
2570	* start of the file. Doing so causes a folio lock/folio
2571	* writeback access order inversion - we should only ever lock
2572	* multiple folios in ascending folio->index order, and looping
2573	* back to the start of the file violates that rule and causes
2574	* deadlocks.
2575	*/
2576	if (wbc->range_cyclic)
2577	mapping->writeback_index = 0;
2578
2579	/*
2580	* Return the first error we encountered (if there was any) to
2581	* the caller.
2582	*/
2583	*error = wbc->saved_err;
2584	}
2585	return folio;
2586
2587	done:
2588	if (wbc->range_cyclic)
2589	mapping->writeback_index = folio_next_index(folio);
2590	folio_batch_release(fbatch: &wbc->fbatch);
2591	return NULL;
2592	}
2593	EXPORT_SYMBOL_GPL(writeback_iter);
2594
2595	/**
2596	* write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
2597	* @mapping: address space structure to write
2598	* @wbc: subtract the number of written pages from *@wbc->nr_to_write
2599	* @writepage: function called for each page
2600	* @data: data passed to writepage function
2601	*
2602	* Return: %0 on success, negative error code otherwise
2603	*
2604	* Note: please use writeback_iter() instead.
2605	*/
2606	int write_cache_pages(struct address_space *mapping,
2607	struct writeback_control *wbc, writepage_t writepage,
2608	void *data)
2609	{
2610	struct folio *folio = NULL;
2611	int error;
2612
2613	while ((folio = writeback_iter(mapping, wbc, folio, &error))) {
2614	error = writepage(folio, wbc, data);
2615	if (error == AOP_WRITEPAGE_ACTIVATE) {
2616	folio_unlock(folio);
2617	error = 0;
2618	}
2619	}
2620
2621	return error;
2622	}
2623	EXPORT_SYMBOL(write_cache_pages);
2624
2625	int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
2626	{
2627	int ret;
2628	struct bdi_writeback *wb;
2629
2630	if (wbc->nr_to_write <= 0)
2631	return 0;
2632	wb = inode_to_wb_wbc(inode: mapping->host, wbc);
2633	wb_bandwidth_estimate_start(wb);
2634	while (1) {
2635	if (mapping->a_ops->writepages)
2636	ret = mapping->a_ops->writepages(mapping, wbc);
2637	else
2638	/* deal with chardevs and other special files */
2639	ret = 0;
2640	if (ret != -ENOMEM || wbc->sync_mode != WB_SYNC_ALL)
2641	break;
2642
2643	/*
2644	* Lacking an allocation context or the locality or writeback
2645	* state of any of the inode's pages, throttle based on
2646	* writeback activity on the local node. It's as good a
2647	* guess as any.
2648	*/
2649	reclaim_throttle(NODE_DATA(numa_node_id()),
2650	reason: VMSCAN_THROTTLE_WRITEBACK);
2651	}
2652	/*
2653	* Usually few pages are written by now from those we've just submitted
2654	* but if there's constant writeback being submitted, this makes sure
2655	* writeback bandwidth is updated once in a while.
2656	*/
2657	if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
2658	BANDWIDTH_INTERVAL))
2659	wb_update_bandwidth(wb);
2660	return ret;
2661	}
2662
2663	/*
2664	* For address_spaces which do not use buffers nor write back.
2665	*/
2666	bool noop_dirty_folio(struct address_space *mapping, struct folio *folio)
2667	{
2668	if (!folio_test_dirty(folio))
2669	return !folio_test_set_dirty(folio);
2670	return false;
2671	}
2672	EXPORT_SYMBOL(noop_dirty_folio);
2673
2674	/*
2675	* Helper function for set_page_dirty family.
2676	*
2677	* NOTE: This relies on being atomic wrt interrupts.
2678	*/
2679	static void folio_account_dirtied(struct folio *folio,
2680	struct address_space *mapping)
2681	{
2682	struct inode *inode = mapping->host;
2683
2684	trace_writeback_dirty_folio(folio, mapping);
2685
2686	if (mapping_can_writeback(mapping)) {
2687	struct bdi_writeback *wb;
2688	long nr = folio_nr_pages(folio);
2689
2690	inode_attach_wb(inode, folio);
2691	wb = inode_to_wb(inode);
2692
2693	__lruvec_stat_mod_folio(folio, idx: NR_FILE_DIRTY, val: nr);
2694	__zone_stat_mod_folio(folio, item: NR_ZONE_WRITE_PENDING, nr);
2695	__node_stat_mod_folio(folio, item: NR_DIRTIED, nr);
2696	wb_stat_mod(wb, item: WB_RECLAIMABLE, amount: nr);
2697	wb_stat_mod(wb, item: WB_DIRTIED, amount: nr);
2698	task_io_account_write(bytes: nr * PAGE_SIZE);
2699	current->nr_dirtied += nr;
2700	__this_cpu_add(bdp_ratelimits, nr);
2701
2702	mem_cgroup_track_foreign_dirty(folio, wb);
2703	}
2704	}
2705
2706	/*
2707	* Helper function for deaccounting dirty page without writeback.
2708	*
2709	*/
2710	void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb)
2711	{
2712	long nr = folio_nr_pages(folio);
2713
2714	lruvec_stat_mod_folio(folio, idx: NR_FILE_DIRTY, val: -nr);
2715	zone_stat_mod_folio(folio, item: NR_ZONE_WRITE_PENDING, nr: -nr);
2716	wb_stat_mod(wb, item: WB_RECLAIMABLE, amount: -nr);
2717	task_io_account_cancelled_write(bytes: nr * PAGE_SIZE);
2718	}
2719
2720	/*
2721	* Mark the folio dirty, and set it dirty in the page cache.
2722	*
2723	* If warn is true, then emit a warning if the folio is not uptodate and has
2724	* not been truncated.
2725	*
2726	* It is the caller's responsibility to prevent the folio from being truncated
2727	* while this function is in progress, although it may have been truncated
2728	* before this function is called. Most callers have the folio locked.
2729	* A few have the folio blocked from truncation through other means (e.g.
2730	* zap_vma_pages() has it mapped and is holding the page table lock).
2731	* When called from mark_buffer_dirty(), the filesystem should hold a
2732	* reference to the buffer_head that is being marked dirty, which causes
2733	* try_to_free_buffers() to fail.
2734	*/
2735	void __folio_mark_dirty(struct folio *folio, struct address_space *mapping,
2736	int warn)
2737	{
2738	unsigned long flags;
2739
2740	xa_lock_irqsave(&mapping->i_pages, flags);
2741	if (folio->mapping) { /* Race with truncate? */
2742	WARN_ON_ONCE(warn && !folio_test_uptodate(folio));
2743	folio_account_dirtied(folio, mapping);
2744	__xa_set_mark(&mapping->i_pages, index: folio_index(folio),
2745	PAGECACHE_TAG_DIRTY);
2746	}
2747	xa_unlock_irqrestore(&mapping->i_pages, flags);
2748	}
2749
2750	/**
2751	* filemap_dirty_folio - Mark a folio dirty for filesystems which do not use buffer_heads.
2752	* @mapping: Address space this folio belongs to.
2753	* @folio: Folio to be marked as dirty.
2754	*
2755	* Filesystems which do not use buffer heads should call this function
2756	* from their dirty_folio address space operation. It ignores the
2757	* contents of folio_get_private(), so if the filesystem marks individual
2758	* blocks as dirty, the filesystem should handle that itself.
2759	*
2760	* This is also sometimes used by filesystems which use buffer_heads when
2761	* a single buffer is being dirtied: we want to set the folio dirty in
2762	* that case, but not all the buffers. This is a "bottom-up" dirtying,
2763	* whereas block_dirty_folio() is a "top-down" dirtying.
2764	*
2765	* The caller must ensure this doesn't race with truncation. Most will
2766	* simply hold the folio lock, but e.g. zap_pte_range() calls with the
2767	* folio mapped and the pte lock held, which also locks out truncation.
2768	*/
2769	bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio)
2770	{
2771	if (folio_test_set_dirty(folio))
2772	return false;
2773
2774	__folio_mark_dirty(folio, mapping, warn: !folio_test_private(folio));
2775
2776	if (mapping->host) {
2777	/* !PageAnon && !swapper_space */
2778	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2779	}
2780	return true;
2781	}
2782	EXPORT_SYMBOL(filemap_dirty_folio);
2783
2784	/**
2785	* folio_redirty_for_writepage - Decline to write a dirty folio.
2786	* @wbc: The writeback control.
2787	* @folio: The folio.
2788	*
2789	* When a writepage implementation decides that it doesn't want to write
2790	* @folio for some reason, it should call this function, unlock @folio and
2791	* return 0.
2792	*
2793	* Return: True if we redirtied the folio. False if someone else dirtied
2794	* it first.
2795	*/
2796	bool folio_redirty_for_writepage(struct writeback_control *wbc,
2797	struct folio *folio)
2798	{
2799	struct address_space *mapping = folio->mapping;
2800	long nr = folio_nr_pages(folio);
2801	bool ret;
2802
2803	wbc->pages_skipped += nr;
2804	ret = filemap_dirty_folio(mapping, folio);
2805	if (mapping && mapping_can_writeback(mapping)) {
2806	struct inode *inode = mapping->host;
2807	struct bdi_writeback *wb;
2808	struct wb_lock_cookie cookie = {};
2809
2810	wb = unlocked_inode_to_wb_begin(inode, cookie: &cookie);
2811	current->nr_dirtied -= nr;
2812	node_stat_mod_folio(folio, item: NR_DIRTIED, nr: -nr);
2813	wb_stat_mod(wb, item: WB_DIRTIED, amount: -nr);
2814	unlocked_inode_to_wb_end(inode, cookie: &cookie);
2815	}
2816	return ret;
2817	}
2818	EXPORT_SYMBOL(folio_redirty_for_writepage);
2819
2820	/**
2821	* folio_mark_dirty - Mark a folio as being modified.
2822	* @folio: The folio.
2823	*
2824	* The folio may not be truncated while this function is running.
2825	* Holding the folio lock is sufficient to prevent truncation, but some
2826	* callers cannot acquire a sleeping lock. These callers instead hold
2827	* the page table lock for a page table which contains at least one page
2828	* in this folio. Truncation will block on the page table lock as it
2829	* unmaps pages before removing the folio from its mapping.
2830	*
2831	* Return: True if the folio was newly dirtied, false if it was already dirty.
2832	*/
2833	bool folio_mark_dirty(struct folio *folio)
2834	{
2835	struct address_space *mapping = folio_mapping(folio);
2836
2837	if (likely(mapping)) {
2838	/*
2839	* readahead/folio_deactivate could remain
2840	* PG_readahead/PG_reclaim due to race with folio_end_writeback
2841	* About readahead, if the folio is written, the flags would be
2842	* reset. So no problem.
2843	* About folio_deactivate, if the folio is redirtied,
2844	* the flag will be reset. So no problem. but if the
2845	* folio is used by readahead it will confuse readahead
2846	* and make it restart the size rampup process. But it's
2847	* a trivial problem.
2848	*/
2849	if (folio_test_reclaim(folio))
2850	folio_clear_reclaim(folio);
2851	return mapping->a_ops->dirty_folio(mapping, folio);
2852	}
2853
2854	return noop_dirty_folio(mapping, folio);
2855	}
2856	EXPORT_SYMBOL(folio_mark_dirty);
2857
2858	/*
2859	* folio_mark_dirty() is racy if the caller has no reference against
2860	* folio->mapping->host, and if the folio is unlocked. This is because another
2861	* CPU could truncate the folio off the mapping and then free the mapping.
2862	*
2863	* Usually, the folio _is_ locked, or the caller is a user-space process which
2864	* holds a reference on the inode by having an open file.
2865	*
2866	* In other cases, the folio should be locked before running folio_mark_dirty().
2867	*/
2868	bool folio_mark_dirty_lock(struct folio *folio)
2869	{
2870	bool ret;
2871
2872	folio_lock(folio);
2873	ret = folio_mark_dirty(folio);
2874	folio_unlock(folio);
2875	return ret;
2876	}
2877	EXPORT_SYMBOL(folio_mark_dirty_lock);
2878
2879	/*
2880	* This cancels just the dirty bit on the kernel page itself, it does NOT
2881	* actually remove dirty bits on any mmap's that may be around. It also
2882	* leaves the page tagged dirty, so any sync activity will still find it on
2883	* the dirty lists, and in particular, clear_page_dirty_for_io() will still
2884	* look at the dirty bits in the VM.
2885	*
2886	* Doing this should *normally* only ever be done when a page is truncated,
2887	* and is not actually mapped anywhere at all. However, fs/buffer.c does
2888	* this when it notices that somebody has cleaned out all the buffers on a
2889	* page without actually doing it through the VM. Can you say "ext3 is
2890	* horribly ugly"? Thought you could.
2891	*/
2892	void __folio_cancel_dirty(struct folio *folio)
2893	{
2894	struct address_space *mapping = folio_mapping(folio);
2895
2896	if (mapping_can_writeback(mapping)) {
2897	struct inode *inode = mapping->host;
2898	struct bdi_writeback *wb;
2899	struct wb_lock_cookie cookie = {};
2900
2901	wb = unlocked_inode_to_wb_begin(inode, cookie: &cookie);
2902
2903	if (folio_test_clear_dirty(folio))
2904	folio_account_cleaned(folio, wb);
2905
2906	unlocked_inode_to_wb_end(inode, cookie: &cookie);
2907	} else {
2908	folio_clear_dirty(folio);
2909	}
2910	}
2911	EXPORT_SYMBOL(__folio_cancel_dirty);
2912
2913	/*
2914	* Clear a folio's dirty flag, while caring for dirty memory accounting.
2915	* Returns true if the folio was previously dirty.
2916	*
2917	* This is for preparing to put the folio under writeout. We leave
2918	* the folio tagged as dirty in the xarray so that a concurrent
2919	* write-for-sync can discover it via a PAGECACHE_TAG_DIRTY walk.
2920	* The ->writepage implementation will run either folio_start_writeback()
2921	* or folio_mark_dirty(), at which stage we bring the folio's dirty flag
2922	* and xarray dirty tag back into sync.
2923	*
2924	* This incoherency between the folio's dirty flag and xarray tag is
2925	* unfortunate, but it only exists while the folio is locked.
2926	*/
2927	bool folio_clear_dirty_for_io(struct folio *folio)
2928	{
2929	struct address_space *mapping = folio_mapping(folio);
2930	bool ret = false;
2931
2932	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2933
2934	if (mapping && mapping_can_writeback(mapping)) {
2935	struct inode *inode = mapping->host;
2936	struct bdi_writeback *wb;
2937	struct wb_lock_cookie cookie = {};
2938
2939	/*
2940	* Yes, Virginia, this is indeed insane.
2941	*
2942	* We use this sequence to make sure that
2943	* (a) we account for dirty stats properly
2944	* (b) we tell the low-level filesystem to
2945	* mark the whole folio dirty if it was
2946	* dirty in a pagetable. Only to then
2947	* (c) clean the folio again and return 1 to
2948	* cause the writeback.
2949	*
2950	* This way we avoid all nasty races with the
2951	* dirty bit in multiple places and clearing
2952	* them concurrently from different threads.
2953	*
2954	* Note! Normally the "folio_mark_dirty(folio)"
2955	* has no effect on the actual dirty bit - since
2956	* that will already usually be set. But we
2957	* need the side effects, and it can help us
2958	* avoid races.
2959	*
2960	* We basically use the folio "master dirty bit"
2961	* as a serialization point for all the different
2962	* threads doing their things.
2963	*/
2964	if (folio_mkclean(folio))
2965	folio_mark_dirty(folio);
2966	/*
2967	* We carefully synchronise fault handlers against
2968	* installing a dirty pte and marking the folio dirty
2969	* at this point. We do this by having them hold the
2970	* page lock while dirtying the folio, and folios are
2971	* always locked coming in here, so we get the desired
2972	* exclusion.
2973	*/
2974	wb = unlocked_inode_to_wb_begin(inode, cookie: &cookie);
2975	if (folio_test_clear_dirty(folio)) {
2976	long nr = folio_nr_pages(folio);
2977	lruvec_stat_mod_folio(folio, idx: NR_FILE_DIRTY, val: -nr);
2978	zone_stat_mod_folio(folio, item: NR_ZONE_WRITE_PENDING, nr: -nr);
2979	wb_stat_mod(wb, item: WB_RECLAIMABLE, amount: -nr);
2980	ret = true;
2981	}
2982	unlocked_inode_to_wb_end(inode, cookie: &cookie);
2983	return ret;
2984	}
2985	return folio_test_clear_dirty(folio);
2986	}
2987	EXPORT_SYMBOL(folio_clear_dirty_for_io);
2988
2989	static void wb_inode_writeback_start(struct bdi_writeback *wb)
2990	{
2991	atomic_inc(v: &wb->writeback_inodes);
2992	}
2993
2994	static void wb_inode_writeback_end(struct bdi_writeback *wb)
2995	{
2996	unsigned long flags;
2997	atomic_dec(v: &wb->writeback_inodes);
2998	/*
2999	* Make sure estimate of writeback throughput gets updated after
3000	* writeback completed. We delay the update by BANDWIDTH_INTERVAL
3001	* (which is the interval other bandwidth updates use for batching) so
3002	* that if multiple inodes end writeback at a similar time, they get
3003	* batched into one bandwidth update.
3004	*/
3005	spin_lock_irqsave(&wb->work_lock, flags);
3006	if (test_bit(WB_registered, &wb->state))
3007	queue_delayed_work(wq: bdi_wq, dwork: &wb->bw_dwork, BANDWIDTH_INTERVAL);
3008	spin_unlock_irqrestore(lock: &wb->work_lock, flags);
3009	}
3010
3011	bool __folio_end_writeback(struct folio *folio)
3012	{
3013	long nr = folio_nr_pages(folio);
3014	struct address_space *mapping = folio_mapping(folio);
3015	bool ret;
3016
3017	if (mapping && mapping_use_writeback_tags(mapping)) {
3018	struct inode *inode = mapping->host;
3019	struct backing_dev_info *bdi = inode_to_bdi(inode);
3020	unsigned long flags;
3021
3022	xa_lock_irqsave(&mapping->i_pages, flags);
3023	ret = folio_xor_flags_has_waiters(folio, mask: 1 << PG_writeback);
3024	__xa_clear_mark(&mapping->i_pages, index: folio_index(folio),
3025	PAGECACHE_TAG_WRITEBACK);
3026	if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
3027	struct bdi_writeback *wb = inode_to_wb(inode);
3028
3029	wb_stat_mod(wb, item: WB_WRITEBACK, amount: -nr);
3030	__wb_writeout_add(wb, nr);
3031	if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
3032	wb_inode_writeback_end(wb);
3033	}
3034
3035	if (mapping->host && !mapping_tagged(mapping,
3036	PAGECACHE_TAG_WRITEBACK))
3037	sb_clear_inode_writeback(inode: mapping->host);
3038
3039	xa_unlock_irqrestore(&mapping->i_pages, flags);
3040	} else {
3041	ret = folio_xor_flags_has_waiters(folio, mask: 1 << PG_writeback);
3042	}
3043
3044	lruvec_stat_mod_folio(folio, idx: NR_WRITEBACK, val: -nr);
3045	zone_stat_mod_folio(folio, item: NR_ZONE_WRITE_PENDING, nr: -nr);
3046	node_stat_mod_folio(folio, item: NR_WRITTEN, nr);
3047
3048	return ret;
3049	}
3050
3051	void __folio_start_writeback(struct folio *folio, bool keep_write)
3052	{
3053	long nr = folio_nr_pages(folio);
3054	struct address_space *mapping = folio_mapping(folio);
3055	int access_ret;
3056
3057	VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
3058	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
3059
3060	if (mapping && mapping_use_writeback_tags(mapping)) {
3061	XA_STATE(xas, &mapping->i_pages, folio_index(folio));
3062	struct inode *inode = mapping->host;
3063	struct backing_dev_info *bdi = inode_to_bdi(inode);
3064	unsigned long flags;
3065	bool on_wblist;
3066
3067	xas_lock_irqsave(&xas, flags);
3068	xas_load(&xas);
3069	folio_test_set_writeback(folio);
3070
3071	on_wblist = mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
3072
3073	xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
3074	if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
3075	struct bdi_writeback *wb = inode_to_wb(inode);
3076
3077	wb_stat_mod(wb, item: WB_WRITEBACK, amount: nr);
3078	if (!on_wblist)
3079	wb_inode_writeback_start(wb);
3080	}
3081
3082	/*
3083	* We can come through here when swapping anonymous
3084	* folios, so we don't necessarily have an inode to
3085	* track for sync.
3086	*/
3087	if (mapping->host && !on_wblist)
3088	sb_mark_inode_writeback(inode: mapping->host);
3089	if (!folio_test_dirty(folio))
3090	xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
3091	if (!keep_write)
3092	xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
3093	xas_unlock_irqrestore(&xas, flags);
3094	} else {
3095	folio_test_set_writeback(folio);
3096	}
3097
3098	lruvec_stat_mod_folio(folio, idx: NR_WRITEBACK, val: nr);
3099	zone_stat_mod_folio(folio, item: NR_ZONE_WRITE_PENDING, nr);
3100
3101	access_ret = arch_make_folio_accessible(folio);
3102	/*
3103	* If writeback has been triggered on a page that cannot be made
3104	* accessible, it is too late to recover here.
3105	*/
3106	VM_BUG_ON_FOLIO(access_ret != 0, folio);
3107	}
3108	EXPORT_SYMBOL(__folio_start_writeback);
3109
3110	/**
3111	* folio_wait_writeback - Wait for a folio to finish writeback.
3112	* @folio: The folio to wait for.
3113	*
3114	* If the folio is currently being written back to storage, wait for the
3115	* I/O to complete.
3116	*
3117	* Context: Sleeps. Must be called in process context and with
3118	* no spinlocks held. Caller should hold a reference on the folio.
3119	* If the folio is not locked, writeback may start again after writeback
3120	* has finished.
3121	*/
3122	void folio_wait_writeback(struct folio *folio)
3123	{
3124	while (folio_test_writeback(folio)) {
3125	trace_folio_wait_writeback(folio, mapping: folio_mapping(folio));
3126	folio_wait_bit(folio, bit_nr: PG_writeback);
3127	}
3128	}
3129	EXPORT_SYMBOL_GPL(folio_wait_writeback);
3130
3131	/**
3132	* folio_wait_writeback_killable - Wait for a folio to finish writeback.
3133	* @folio: The folio to wait for.
3134	*
3135	* If the folio is currently being written back to storage, wait for the
3136	* I/O to complete or a fatal signal to arrive.
3137	*
3138	* Context: Sleeps. Must be called in process context and with
3139	* no spinlocks held. Caller should hold a reference on the folio.
3140	* If the folio is not locked, writeback may start again after writeback
3141	* has finished.
3142	* Return: 0 on success, -EINTR if we get a fatal signal while waiting.
3143	*/
3144	int folio_wait_writeback_killable(struct folio *folio)
3145	{
3146	while (folio_test_writeback(folio)) {
3147	trace_folio_wait_writeback(folio, mapping: folio_mapping(folio));
3148	if (folio_wait_bit_killable(folio, bit_nr: PG_writeback))
3149	return -EINTR;
3150	}
3151
3152	return 0;
3153	}
3154	EXPORT_SYMBOL_GPL(folio_wait_writeback_killable);
3155
3156	/**
3157	* folio_wait_stable() - wait for writeback to finish, if necessary.
3158	* @folio: The folio to wait on.
3159	*
3160	* This function determines if the given folio is related to a backing
3161	* device that requires folio contents to be held stable during writeback.
3162	* If so, then it will wait for any pending writeback to complete.
3163	*
3164	* Context: Sleeps. Must be called in process context and with
3165	* no spinlocks held. Caller should hold a reference on the folio.
3166	* If the folio is not locked, writeback may start again after writeback
3167	* has finished.
3168	*/
3169	void folio_wait_stable(struct folio *folio)
3170	{
3171	if (mapping_stable_writes(mapping: folio_mapping(folio)))
3172	folio_wait_writeback(folio);
3173	}
3174	EXPORT_SYMBOL_GPL(folio_wait_stable);
3175