1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_VMSTAT_H
3#define _LINUX_VMSTAT_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7#include <linux/mmzone.h>
8#include <linux/vm_event_item.h>
9#include <linux/atomic.h>
10#include <linux/static_key.h>
11#include <linux/mmdebug.h>
12
13extern int sysctl_stat_interval;
14
15#ifdef CONFIG_NUMA
16#define ENABLE_NUMA_STAT 1
17#define DISABLE_NUMA_STAT 0
18extern int sysctl_vm_numa_stat;
19DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
20int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
21 void *buffer, size_t *length, loff_t *ppos);
22#endif
23
24struct reclaim_stat {
25 unsigned nr_dirty;
26 unsigned nr_unqueued_dirty;
27 unsigned nr_congested;
28 unsigned nr_writeback;
29 unsigned nr_immediate;
30 unsigned nr_pageout;
31 unsigned nr_activate[ANON_AND_FILE];
32 unsigned nr_ref_keep;
33 unsigned nr_unmap_fail;
34 unsigned nr_lazyfree_fail;
35};
36
37enum writeback_stat_item {
38 NR_DIRTY_THRESHOLD,
39 NR_DIRTY_BG_THRESHOLD,
40 NR_VM_WRITEBACK_STAT_ITEMS,
41};
42
43#ifdef CONFIG_VM_EVENT_COUNTERS
44/*
45 * Light weight per cpu counter implementation.
46 *
47 * Counters should only be incremented and no critical kernel component
48 * should rely on the counter values.
49 *
50 * Counters are handled completely inline. On many platforms the code
51 * generated will simply be the increment of a global address.
52 */
53
54struct vm_event_state {
55 unsigned long event[NR_VM_EVENT_ITEMS];
56};
57
58DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
59
60/*
61 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
62 * local_irq_disable overhead.
63 */
64static inline void __count_vm_event(enum vm_event_item item)
65{
66 raw_cpu_inc(vm_event_states.event[item]);
67}
68
69static inline void count_vm_event(enum vm_event_item item)
70{
71 this_cpu_inc(vm_event_states.event[item]);
72}
73
74static inline void __count_vm_events(enum vm_event_item item, long delta)
75{
76 raw_cpu_add(vm_event_states.event[item], delta);
77}
78
79static inline void count_vm_events(enum vm_event_item item, long delta)
80{
81 this_cpu_add(vm_event_states.event[item], delta);
82}
83
84extern void all_vm_events(unsigned long *);
85
86extern void vm_events_fold_cpu(int cpu);
87
88#else
89
90/* Disable counters */
91static inline void count_vm_event(enum vm_event_item item)
92{
93}
94static inline void count_vm_events(enum vm_event_item item, long delta)
95{
96}
97static inline void __count_vm_event(enum vm_event_item item)
98{
99}
100static inline void __count_vm_events(enum vm_event_item item, long delta)
101{
102}
103static inline void all_vm_events(unsigned long *ret)
104{
105}
106static inline void vm_events_fold_cpu(int cpu)
107{
108}
109
110#endif /* CONFIG_VM_EVENT_COUNTERS */
111
112#ifdef CONFIG_NUMA_BALANCING
113#define count_vm_numa_event(x) count_vm_event(x)
114#define count_vm_numa_events(x, y) count_vm_events(x, y)
115#else
116#define count_vm_numa_event(x) do {} while (0)
117#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
118#endif /* CONFIG_NUMA_BALANCING */
119
120#ifdef CONFIG_DEBUG_TLBFLUSH
121#define count_vm_tlb_event(x) count_vm_event(x)
122#define count_vm_tlb_events(x, y) count_vm_events(x, y)
123#else
124#define count_vm_tlb_event(x) do {} while (0)
125#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
126#endif
127
128#ifdef CONFIG_PER_VMA_LOCK_STATS
129#define count_vm_vma_lock_event(x) count_vm_event(x)
130#else
131#define count_vm_vma_lock_event(x) do {} while (0)
132#endif
133
134#define __count_zid_vm_events(item, zid, delta) \
135 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
136
137/*
138 * Zone and node-based page accounting with per cpu differentials.
139 */
140extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
141extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
142extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
143
144#ifdef CONFIG_NUMA
145static inline void zone_numa_event_add(long x, struct zone *zone,
146 enum numa_stat_item item)
147{
148 atomic_long_add(i: x, v: &zone->vm_numa_event[item]);
149 atomic_long_add(i: x, v: &vm_numa_event[item]);
150}
151
152static inline unsigned long zone_numa_event_state(struct zone *zone,
153 enum numa_stat_item item)
154{
155 return atomic_long_read(v: &zone->vm_numa_event[item]);
156}
157
158static inline unsigned long
159global_numa_event_state(enum numa_stat_item item)
160{
161 return atomic_long_read(v: &vm_numa_event[item]);
162}
163#endif /* CONFIG_NUMA */
164
165static inline void zone_page_state_add(long x, struct zone *zone,
166 enum zone_stat_item item)
167{
168 atomic_long_add(i: x, v: &zone->vm_stat[item]);
169 atomic_long_add(i: x, v: &vm_zone_stat[item]);
170}
171
172static inline void node_page_state_add(long x, struct pglist_data *pgdat,
173 enum node_stat_item item)
174{
175 atomic_long_add(i: x, v: &pgdat->vm_stat[item]);
176 atomic_long_add(i: x, v: &vm_node_stat[item]);
177}
178
179static inline unsigned long global_zone_page_state(enum zone_stat_item item)
180{
181 long x = atomic_long_read(v: &vm_zone_stat[item]);
182#ifdef CONFIG_SMP
183 if (x < 0)
184 x = 0;
185#endif
186 return x;
187}
188
189static inline
190unsigned long global_node_page_state_pages(enum node_stat_item item)
191{
192 long x = atomic_long_read(v: &vm_node_stat[item]);
193#ifdef CONFIG_SMP
194 if (x < 0)
195 x = 0;
196#endif
197 return x;
198}
199
200static inline unsigned long global_node_page_state(enum node_stat_item item)
201{
202 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
203
204 return global_node_page_state_pages(item);
205}
206
207static inline unsigned long zone_page_state(struct zone *zone,
208 enum zone_stat_item item)
209{
210 long x = atomic_long_read(v: &zone->vm_stat[item]);
211#ifdef CONFIG_SMP
212 if (x < 0)
213 x = 0;
214#endif
215 return x;
216}
217
218/*
219 * More accurate version that also considers the currently pending
220 * deltas. For that we need to loop over all cpus to find the current
221 * deltas. There is no synchronization so the result cannot be
222 * exactly accurate either.
223 */
224static inline unsigned long zone_page_state_snapshot(struct zone *zone,
225 enum zone_stat_item item)
226{
227 long x = atomic_long_read(v: &zone->vm_stat[item]);
228
229#ifdef CONFIG_SMP
230 int cpu;
231 for_each_online_cpu(cpu)
232 x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];
233
234 if (x < 0)
235 x = 0;
236#endif
237 return x;
238}
239
240#ifdef CONFIG_NUMA
241/* See __count_vm_event comment on why raw_cpu_inc is used. */
242static inline void
243__count_numa_event(struct zone *zone, enum numa_stat_item item)
244{
245 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
246
247 raw_cpu_inc(pzstats->vm_numa_event[item]);
248}
249
250static inline void
251__count_numa_events(struct zone *zone, enum numa_stat_item item, long delta)
252{
253 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
254
255 raw_cpu_add(pzstats->vm_numa_event[item], delta);
256}
257
258extern unsigned long sum_zone_node_page_state(int node,
259 enum zone_stat_item item);
260extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item);
261extern unsigned long node_page_state(struct pglist_data *pgdat,
262 enum node_stat_item item);
263extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
264 enum node_stat_item item);
265extern void fold_vm_numa_events(void);
266#else
267#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
268#define node_page_state(node, item) global_node_page_state(item)
269#define node_page_state_pages(node, item) global_node_page_state_pages(item)
270static inline void fold_vm_numa_events(void)
271{
272}
273#endif /* CONFIG_NUMA */
274
275#ifdef CONFIG_SMP
276void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
277void __inc_zone_page_state(struct page *, enum zone_stat_item);
278void __dec_zone_page_state(struct page *, enum zone_stat_item);
279
280void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
281void __inc_node_page_state(struct page *, enum node_stat_item);
282void __dec_node_page_state(struct page *, enum node_stat_item);
283
284void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
285void inc_zone_page_state(struct page *, enum zone_stat_item);
286void dec_zone_page_state(struct page *, enum zone_stat_item);
287
288void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
289void inc_node_page_state(struct page *, enum node_stat_item);
290void dec_node_page_state(struct page *, enum node_stat_item);
291
292extern void inc_node_state(struct pglist_data *, enum node_stat_item);
293extern void __inc_zone_state(struct zone *, enum zone_stat_item);
294extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
295extern void dec_zone_state(struct zone *, enum zone_stat_item);
296extern void __dec_zone_state(struct zone *, enum zone_stat_item);
297extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
298
299void quiet_vmstat(void);
300void cpu_vm_stats_fold(int cpu);
301void refresh_zone_stat_thresholds(void);
302
303struct ctl_table;
304int vmstat_refresh(struct ctl_table *, int write, void *buffer, size_t *lenp,
305 loff_t *ppos);
306
307void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *);
308
309int calculate_pressure_threshold(struct zone *zone);
310int calculate_normal_threshold(struct zone *zone);
311void set_pgdat_percpu_threshold(pg_data_t *pgdat,
312 int (*calculate_pressure)(struct zone *));
313#else /* CONFIG_SMP */
314
315/*
316 * We do not maintain differentials in a single processor configuration.
317 * The functions directly modify the zone and global counters.
318 */
319static inline void __mod_zone_page_state(struct zone *zone,
320 enum zone_stat_item item, long delta)
321{
322 zone_page_state_add(delta, zone, item);
323}
324
325static inline void __mod_node_page_state(struct pglist_data *pgdat,
326 enum node_stat_item item, int delta)
327{
328 if (vmstat_item_in_bytes(item)) {
329 /*
330 * Only cgroups use subpage accounting right now; at
331 * the global level, these items still change in
332 * multiples of whole pages. Store them as pages
333 * internally to keep the per-cpu counters compact.
334 */
335 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
336 delta >>= PAGE_SHIFT;
337 }
338
339 node_page_state_add(delta, pgdat, item);
340}
341
342static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
343{
344 atomic_long_inc(&zone->vm_stat[item]);
345 atomic_long_inc(&vm_zone_stat[item]);
346}
347
348static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
349{
350 atomic_long_inc(&pgdat->vm_stat[item]);
351 atomic_long_inc(&vm_node_stat[item]);
352}
353
354static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
355{
356 atomic_long_dec(&zone->vm_stat[item]);
357 atomic_long_dec(&vm_zone_stat[item]);
358}
359
360static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
361{
362 atomic_long_dec(&pgdat->vm_stat[item]);
363 atomic_long_dec(&vm_node_stat[item]);
364}
365
366static inline void __inc_zone_page_state(struct page *page,
367 enum zone_stat_item item)
368{
369 __inc_zone_state(page_zone(page), item);
370}
371
372static inline void __inc_node_page_state(struct page *page,
373 enum node_stat_item item)
374{
375 __inc_node_state(page_pgdat(page), item);
376}
377
378
379static inline void __dec_zone_page_state(struct page *page,
380 enum zone_stat_item item)
381{
382 __dec_zone_state(page_zone(page), item);
383}
384
385static inline void __dec_node_page_state(struct page *page,
386 enum node_stat_item item)
387{
388 __dec_node_state(page_pgdat(page), item);
389}
390
391
392/*
393 * We only use atomic operations to update counters. So there is no need to
394 * disable interrupts.
395 */
396#define inc_zone_page_state __inc_zone_page_state
397#define dec_zone_page_state __dec_zone_page_state
398#define mod_zone_page_state __mod_zone_page_state
399
400#define inc_node_page_state __inc_node_page_state
401#define dec_node_page_state __dec_node_page_state
402#define mod_node_page_state __mod_node_page_state
403
404#define inc_zone_state __inc_zone_state
405#define inc_node_state __inc_node_state
406#define dec_zone_state __dec_zone_state
407
408#define set_pgdat_percpu_threshold(pgdat, callback) { }
409
410static inline void refresh_zone_stat_thresholds(void) { }
411static inline void cpu_vm_stats_fold(int cpu) { }
412static inline void quiet_vmstat(void) { }
413
414static inline void drain_zonestat(struct zone *zone,
415 struct per_cpu_zonestat *pzstats) { }
416#endif /* CONFIG_SMP */
417
418static inline void __zone_stat_mod_folio(struct folio *folio,
419 enum zone_stat_item item, long nr)
420{
421 __mod_zone_page_state(folio_zone(folio), item, nr);
422}
423
424static inline void __zone_stat_add_folio(struct folio *folio,
425 enum zone_stat_item item)
426{
427 __mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
428}
429
430static inline void __zone_stat_sub_folio(struct folio *folio,
431 enum zone_stat_item item)
432{
433 __mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
434}
435
436static inline void zone_stat_mod_folio(struct folio *folio,
437 enum zone_stat_item item, long nr)
438{
439 mod_zone_page_state(folio_zone(folio), item, nr);
440}
441
442static inline void zone_stat_add_folio(struct folio *folio,
443 enum zone_stat_item item)
444{
445 mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
446}
447
448static inline void zone_stat_sub_folio(struct folio *folio,
449 enum zone_stat_item item)
450{
451 mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
452}
453
454static inline void __node_stat_mod_folio(struct folio *folio,
455 enum node_stat_item item, long nr)
456{
457 __mod_node_page_state(folio_pgdat(folio), item, nr);
458}
459
460static inline void __node_stat_add_folio(struct folio *folio,
461 enum node_stat_item item)
462{
463 __mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
464}
465
466static inline void __node_stat_sub_folio(struct folio *folio,
467 enum node_stat_item item)
468{
469 __mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
470}
471
472static inline void node_stat_mod_folio(struct folio *folio,
473 enum node_stat_item item, long nr)
474{
475 mod_node_page_state(folio_pgdat(folio), item, nr);
476}
477
478static inline void node_stat_add_folio(struct folio *folio,
479 enum node_stat_item item)
480{
481 mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
482}
483
484static inline void node_stat_sub_folio(struct folio *folio,
485 enum node_stat_item item)
486{
487 mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
488}
489
490static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
491 int migratetype)
492{
493 __mod_zone_page_state(zone, item: NR_FREE_PAGES, nr_pages);
494 if (is_migrate_cma(migratetype))
495 __mod_zone_page_state(zone, item: NR_FREE_CMA_PAGES, nr_pages);
496}
497
498extern const char * const vmstat_text[];
499
500static inline const char *zone_stat_name(enum zone_stat_item item)
501{
502 return vmstat_text[item];
503}
504
505#ifdef CONFIG_NUMA
506static inline const char *numa_stat_name(enum numa_stat_item item)
507{
508 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
509 item];
510}
511#endif /* CONFIG_NUMA */
512
513static inline const char *node_stat_name(enum node_stat_item item)
514{
515 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
516 NR_VM_NUMA_EVENT_ITEMS +
517 item];
518}
519
520static inline const char *lru_list_name(enum lru_list lru)
521{
522 return node_stat_name(item: NR_LRU_BASE + lru) + 3; // skip "nr_"
523}
524
525static inline const char *writeback_stat_name(enum writeback_stat_item item)
526{
527 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
528 NR_VM_NUMA_EVENT_ITEMS +
529 NR_VM_NODE_STAT_ITEMS +
530 item];
531}
532
533#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
534static inline const char *vm_event_name(enum vm_event_item item)
535{
536 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
537 NR_VM_NUMA_EVENT_ITEMS +
538 NR_VM_NODE_STAT_ITEMS +
539 NR_VM_WRITEBACK_STAT_ITEMS +
540 item];
541}
542#endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
543
544#ifdef CONFIG_MEMCG
545
546void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
547 int val);
548
549static inline void mod_lruvec_state(struct lruvec *lruvec,
550 enum node_stat_item idx, int val)
551{
552 unsigned long flags;
553
554 local_irq_save(flags);
555 __mod_lruvec_state(lruvec, idx, val);
556 local_irq_restore(flags);
557}
558
559void __lruvec_stat_mod_folio(struct folio *folio,
560 enum node_stat_item idx, int val);
561
562static inline void lruvec_stat_mod_folio(struct folio *folio,
563 enum node_stat_item idx, int val)
564{
565 unsigned long flags;
566
567 local_irq_save(flags);
568 __lruvec_stat_mod_folio(folio, idx, val);
569 local_irq_restore(flags);
570}
571
572static inline void mod_lruvec_page_state(struct page *page,
573 enum node_stat_item idx, int val)
574{
575 lruvec_stat_mod_folio(page_folio(page), idx, val);
576}
577
578#else
579
580static inline void __mod_lruvec_state(struct lruvec *lruvec,
581 enum node_stat_item idx, int val)
582{
583 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
584}
585
586static inline void mod_lruvec_state(struct lruvec *lruvec,
587 enum node_stat_item idx, int val)
588{
589 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
590}
591
592static inline void __lruvec_stat_mod_folio(struct folio *folio,
593 enum node_stat_item idx, int val)
594{
595 __mod_node_page_state(folio_pgdat(folio), idx, val);
596}
597
598static inline void lruvec_stat_mod_folio(struct folio *folio,
599 enum node_stat_item idx, int val)
600{
601 mod_node_page_state(folio_pgdat(folio), idx, val);
602}
603
604static inline void mod_lruvec_page_state(struct page *page,
605 enum node_stat_item idx, int val)
606{
607 mod_node_page_state(page_pgdat(page), idx, val);
608}
609
610#endif /* CONFIG_MEMCG */
611
612static inline void __lruvec_stat_add_folio(struct folio *folio,
613 enum node_stat_item idx)
614{
615 __lruvec_stat_mod_folio(folio, idx, val: folio_nr_pages(folio));
616}
617
618static inline void __lruvec_stat_sub_folio(struct folio *folio,
619 enum node_stat_item idx)
620{
621 __lruvec_stat_mod_folio(folio, idx, val: -folio_nr_pages(folio));
622}
623
624static inline void lruvec_stat_add_folio(struct folio *folio,
625 enum node_stat_item idx)
626{
627 lruvec_stat_mod_folio(folio, idx, val: folio_nr_pages(folio));
628}
629
630static inline void lruvec_stat_sub_folio(struct folio *folio,
631 enum node_stat_item idx)
632{
633 lruvec_stat_mod_folio(folio, idx, val: -folio_nr_pages(folio));
634}
635#endif /* _LINUX_VMSTAT_H */
636

source code of linux/include/linux/vmstat.h