1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_MM_H |
3 | #define _LINUX_MM_H |
4 | |
5 | #include <linux/errno.h> |
6 | #include <linux/mmdebug.h> |
7 | #include <linux/gfp.h> |
8 | #include <linux/bug.h> |
9 | #include <linux/list.h> |
10 | #include <linux/mmzone.h> |
11 | #include <linux/rbtree.h> |
12 | #include <linux/atomic.h> |
13 | #include <linux/debug_locks.h> |
14 | #include <linux/mm_types.h> |
15 | #include <linux/mmap_lock.h> |
16 | #include <linux/range.h> |
17 | #include <linux/pfn.h> |
18 | #include <linux/percpu-refcount.h> |
19 | #include <linux/bit_spinlock.h> |
20 | #include <linux/shrinker.h> |
21 | #include <linux/resource.h> |
22 | #include <linux/page_ext.h> |
23 | #include <linux/err.h> |
24 | #include <linux/page-flags.h> |
25 | #include <linux/page_ref.h> |
26 | #include <linux/overflow.h> |
27 | #include <linux/sizes.h> |
28 | #include <linux/sched.h> |
29 | #include <linux/pgtable.h> |
30 | #include <linux/kasan.h> |
31 | #include <linux/memremap.h> |
32 | #include <linux/slab.h> |
33 | |
34 | struct mempolicy; |
35 | struct anon_vma; |
36 | struct anon_vma_chain; |
37 | struct user_struct; |
38 | struct pt_regs; |
39 | |
40 | extern int sysctl_page_lock_unfairness; |
41 | |
42 | void mm_core_init(void); |
43 | void init_mm_internals(void); |
44 | |
45 | #ifndef CONFIG_NUMA /* Don't use mapnrs, do it properly */ |
46 | extern unsigned long max_mapnr; |
47 | |
48 | static inline void set_max_mapnr(unsigned long limit) |
49 | { |
50 | max_mapnr = limit; |
51 | } |
52 | #else |
53 | static inline void set_max_mapnr(unsigned long limit) { } |
54 | #endif |
55 | |
56 | extern atomic_long_t _totalram_pages; |
57 | static inline unsigned long totalram_pages(void) |
58 | { |
59 | return (unsigned long)atomic_long_read(v: &_totalram_pages); |
60 | } |
61 | |
62 | static inline void totalram_pages_inc(void) |
63 | { |
64 | atomic_long_inc(v: &_totalram_pages); |
65 | } |
66 | |
67 | static inline void totalram_pages_dec(void) |
68 | { |
69 | atomic_long_dec(v: &_totalram_pages); |
70 | } |
71 | |
72 | static inline void totalram_pages_add(long count) |
73 | { |
74 | atomic_long_add(i: count, v: &_totalram_pages); |
75 | } |
76 | |
77 | extern void * high_memory; |
78 | extern int page_cluster; |
79 | extern const int page_cluster_max; |
80 | |
81 | #ifdef CONFIG_SYSCTL |
82 | extern int sysctl_legacy_va_layout; |
83 | #else |
84 | #define sysctl_legacy_va_layout 0 |
85 | #endif |
86 | |
87 | #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS |
88 | extern const int mmap_rnd_bits_min; |
89 | extern const int mmap_rnd_bits_max; |
90 | extern int mmap_rnd_bits __read_mostly; |
91 | #endif |
92 | #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS |
93 | extern const int mmap_rnd_compat_bits_min; |
94 | extern const int mmap_rnd_compat_bits_max; |
95 | extern int mmap_rnd_compat_bits __read_mostly; |
96 | #endif |
97 | |
98 | #include <asm/page.h> |
99 | #include <asm/processor.h> |
100 | |
101 | #ifndef __pa_symbol |
102 | #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) |
103 | #endif |
104 | |
105 | #ifndef page_to_virt |
106 | #define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x))) |
107 | #endif |
108 | |
109 | #ifndef lm_alias |
110 | #define lm_alias(x) __va(__pa_symbol(x)) |
111 | #endif |
112 | |
113 | /* |
114 | * To prevent common memory management code establishing |
115 | * a zero page mapping on a read fault. |
116 | * This macro should be defined within <asm/pgtable.h>. |
117 | * s390 does this to prevent multiplexing of hardware bits |
118 | * related to the physical page in case of virtualization. |
119 | */ |
120 | #ifndef mm_forbids_zeropage |
121 | #define mm_forbids_zeropage(X) (0) |
122 | #endif |
123 | |
124 | /* |
125 | * On some architectures it is expensive to call memset() for small sizes. |
126 | * If an architecture decides to implement their own version of |
127 | * mm_zero_struct_page they should wrap the defines below in a #ifndef and |
128 | * define their own version of this macro in <asm/pgtable.h> |
129 | */ |
130 | #if BITS_PER_LONG == 64 |
131 | /* This function must be updated when the size of struct page grows above 96 |
132 | * or reduces below 56. The idea that compiler optimizes out switch() |
133 | * statement, and only leaves move/store instructions. Also the compiler can |
134 | * combine write statements if they are both assignments and can be reordered, |
135 | * this can result in several of the writes here being dropped. |
136 | */ |
137 | #define mm_zero_struct_page(pp) __mm_zero_struct_page(pp) |
138 | static inline void __mm_zero_struct_page(struct page *page) |
139 | { |
140 | unsigned long *_pp = (void *)page; |
141 | |
142 | /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */ |
143 | BUILD_BUG_ON(sizeof(struct page) & 7); |
144 | BUILD_BUG_ON(sizeof(struct page) < 56); |
145 | BUILD_BUG_ON(sizeof(struct page) > 96); |
146 | |
147 | switch (sizeof(struct page)) { |
148 | case 96: |
149 | _pp[11] = 0; |
150 | fallthrough; |
151 | case 88: |
152 | _pp[10] = 0; |
153 | fallthrough; |
154 | case 80: |
155 | _pp[9] = 0; |
156 | fallthrough; |
157 | case 72: |
158 | _pp[8] = 0; |
159 | fallthrough; |
160 | case 64: |
161 | _pp[7] = 0; |
162 | fallthrough; |
163 | case 56: |
164 | _pp[6] = 0; |
165 | _pp[5] = 0; |
166 | _pp[4] = 0; |
167 | _pp[3] = 0; |
168 | _pp[2] = 0; |
169 | _pp[1] = 0; |
170 | _pp[0] = 0; |
171 | } |
172 | } |
173 | #else |
174 | #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page))) |
175 | #endif |
176 | |
177 | /* |
178 | * Default maximum number of active map areas, this limits the number of vmas |
179 | * per mm struct. Users can overwrite this number by sysctl but there is a |
180 | * problem. |
181 | * |
182 | * When a program's coredump is generated as ELF format, a section is created |
183 | * per a vma. In ELF, the number of sections is represented in unsigned short. |
184 | * This means the number of sections should be smaller than 65535 at coredump. |
185 | * Because the kernel adds some informative sections to a image of program at |
186 | * generating coredump, we need some margin. The number of extra sections is |
187 | * 1-3 now and depends on arch. We use "5" as safe margin, here. |
188 | * |
189 | * ELF extended numbering allows more than 65535 sections, so 16-bit bound is |
190 | * not a hard limit any more. Although some userspace tools can be surprised by |
191 | * that. |
192 | */ |
193 | #define MAPCOUNT_ELF_CORE_MARGIN (5) |
194 | #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) |
195 | |
196 | extern int sysctl_max_map_count; |
197 | |
198 | extern unsigned long sysctl_user_reserve_kbytes; |
199 | extern unsigned long sysctl_admin_reserve_kbytes; |
200 | |
201 | extern int sysctl_overcommit_memory; |
202 | extern int sysctl_overcommit_ratio; |
203 | extern unsigned long sysctl_overcommit_kbytes; |
204 | |
205 | int overcommit_ratio_handler(struct ctl_table *, int, void *, size_t *, |
206 | loff_t *); |
207 | int overcommit_kbytes_handler(struct ctl_table *, int, void *, size_t *, |
208 | loff_t *); |
209 | int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, |
210 | loff_t *); |
211 | |
212 | #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) |
213 | #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) |
214 | #define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio)) |
215 | #else |
216 | #define nth_page(page,n) ((page) + (n)) |
217 | #define folio_page_idx(folio, p) ((p) - &(folio)->page) |
218 | #endif |
219 | |
220 | /* to align the pointer to the (next) page boundary */ |
221 | #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) |
222 | |
223 | /* to align the pointer to the (prev) page boundary */ |
224 | #define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE) |
225 | |
226 | /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ |
227 | #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE) |
228 | |
229 | #define lru_to_page(head) (list_entry((head)->prev, struct page, lru)) |
230 | static inline struct folio *lru_to_folio(struct list_head *head) |
231 | { |
232 | return list_entry((head)->prev, struct folio, lru); |
233 | } |
234 | |
235 | void setup_initial_init_mm(void *start_code, void *end_code, |
236 | void *end_data, void *brk); |
237 | |
238 | /* |
239 | * Linux kernel virtual memory manager primitives. |
240 | * The idea being to have a "virtual" mm in the same way |
241 | * we have a virtual fs - giving a cleaner interface to the |
242 | * mm details, and allowing different kinds of memory mappings |
243 | * (from shared memory to executable loading to arbitrary |
244 | * mmap() functions). |
245 | */ |
246 | |
247 | struct vm_area_struct *vm_area_alloc(struct mm_struct *); |
248 | struct vm_area_struct *vm_area_dup(struct vm_area_struct *); |
249 | void vm_area_free(struct vm_area_struct *); |
250 | /* Use only if VMA has no other users */ |
251 | void __vm_area_free(struct vm_area_struct *vma); |
252 | |
253 | #ifndef CONFIG_MMU |
254 | extern struct rb_root nommu_region_tree; |
255 | extern struct rw_semaphore nommu_region_sem; |
256 | |
257 | extern unsigned int kobjsize(const void *objp); |
258 | #endif |
259 | |
260 | /* |
261 | * vm_flags in vm_area_struct, see mm_types.h. |
262 | * When changing, update also include/trace/events/mmflags.h |
263 | */ |
264 | #define VM_NONE 0x00000000 |
265 | |
266 | #define VM_READ 0x00000001 /* currently active flags */ |
267 | #define VM_WRITE 0x00000002 |
268 | #define VM_EXEC 0x00000004 |
269 | #define VM_SHARED 0x00000008 |
270 | |
271 | /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ |
272 | #define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */ |
273 | #define VM_MAYWRITE 0x00000020 |
274 | #define VM_MAYEXEC 0x00000040 |
275 | #define VM_MAYSHARE 0x00000080 |
276 | |
277 | #define VM_GROWSDOWN 0x00000100 /* general info on the segment */ |
278 | #ifdef CONFIG_MMU |
279 | #define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */ |
280 | #else /* CONFIG_MMU */ |
281 | #define VM_MAYOVERLAY 0x00000200 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */ |
282 | #define VM_UFFD_MISSING 0 |
283 | #endif /* CONFIG_MMU */ |
284 | #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ |
285 | #define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */ |
286 | |
287 | #define VM_LOCKED 0x00002000 |
288 | #define VM_IO 0x00004000 /* Memory mapped I/O or similar */ |
289 | |
290 | /* Used by sys_madvise() */ |
291 | #define VM_SEQ_READ 0x00008000 /* App will access data sequentially */ |
292 | #define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */ |
293 | |
294 | #define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */ |
295 | #define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */ |
296 | #define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */ |
297 | #define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */ |
298 | #define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */ |
299 | #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */ |
300 | #define VM_SYNC 0x00800000 /* Synchronous page faults */ |
301 | #define VM_ARCH_1 0x01000000 /* Architecture-specific flag */ |
302 | #define VM_WIPEONFORK 0x02000000 /* Wipe VMA contents in child. */ |
303 | #define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */ |
304 | |
305 | #ifdef CONFIG_MEM_SOFT_DIRTY |
306 | # define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */ |
307 | #else |
308 | # define VM_SOFTDIRTY 0 |
309 | #endif |
310 | |
311 | #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */ |
312 | #define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */ |
313 | #define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */ |
314 | #define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */ |
315 | |
316 | #ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS |
317 | #define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */ |
318 | #define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */ |
319 | #define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */ |
320 | #define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */ |
321 | #define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */ |
322 | #define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit architectures */ |
323 | #define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0) |
324 | #define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1) |
325 | #define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2) |
326 | #define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3) |
327 | #define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4) |
328 | #define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5) |
329 | #endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */ |
330 | |
331 | #ifdef CONFIG_ARCH_HAS_PKEYS |
332 | # define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0 |
333 | # define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */ |
334 | # define VM_PKEY_BIT1 VM_HIGH_ARCH_1 /* on x86 and 5-bit value on ppc64 */ |
335 | # define VM_PKEY_BIT2 VM_HIGH_ARCH_2 |
336 | # define VM_PKEY_BIT3 VM_HIGH_ARCH_3 |
337 | #ifdef CONFIG_PPC |
338 | # define VM_PKEY_BIT4 VM_HIGH_ARCH_4 |
339 | #else |
340 | # define VM_PKEY_BIT4 0 |
341 | #endif |
342 | #endif /* CONFIG_ARCH_HAS_PKEYS */ |
343 | |
344 | #ifdef CONFIG_X86_USER_SHADOW_STACK |
345 | /* |
346 | * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of |
347 | * support core mm. |
348 | * |
349 | * These VMAs will get a single end guard page. This helps userspace protect |
350 | * itself from attacks. A single page is enough for current shadow stack archs |
351 | * (x86). See the comments near alloc_shstk() in arch/x86/kernel/shstk.c |
352 | * for more details on the guard size. |
353 | */ |
354 | # define VM_SHADOW_STACK VM_HIGH_ARCH_5 |
355 | #else |
356 | # define VM_SHADOW_STACK VM_NONE |
357 | #endif |
358 | |
359 | #if defined(CONFIG_X86) |
360 | # define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */ |
361 | #elif defined(CONFIG_PPC) |
362 | # define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */ |
363 | #elif defined(CONFIG_PARISC) |
364 | # define VM_GROWSUP VM_ARCH_1 |
365 | #elif defined(CONFIG_SPARC64) |
366 | # define VM_SPARC_ADI VM_ARCH_1 /* Uses ADI tag for access control */ |
367 | # define VM_ARCH_CLEAR VM_SPARC_ADI |
368 | #elif defined(CONFIG_ARM64) |
369 | # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */ |
370 | # define VM_ARCH_CLEAR VM_ARM64_BTI |
371 | #elif !defined(CONFIG_MMU) |
372 | # define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */ |
373 | #endif |
374 | |
375 | #if defined(CONFIG_ARM64_MTE) |
376 | # define VM_MTE VM_HIGH_ARCH_0 /* Use Tagged memory for access control */ |
377 | # define VM_MTE_ALLOWED VM_HIGH_ARCH_1 /* Tagged memory permitted */ |
378 | #else |
379 | # define VM_MTE VM_NONE |
380 | # define VM_MTE_ALLOWED VM_NONE |
381 | #endif |
382 | |
383 | #ifndef VM_GROWSUP |
384 | # define VM_GROWSUP VM_NONE |
385 | #endif |
386 | |
387 | #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR |
388 | # define VM_UFFD_MINOR_BIT 38 |
389 | # define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */ |
390 | #else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ |
391 | # define VM_UFFD_MINOR VM_NONE |
392 | #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ |
393 | |
394 | /* Bits set in the VMA until the stack is in its final location */ |
395 | #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) |
396 | |
397 | #define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) |
398 | |
399 | /* Common data flag combinations */ |
400 | #define VM_DATA_FLAGS_TSK_EXEC (VM_READ | VM_WRITE | TASK_EXEC | \ |
401 | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) |
402 | #define VM_DATA_FLAGS_NON_EXEC (VM_READ | VM_WRITE | VM_MAYREAD | \ |
403 | VM_MAYWRITE | VM_MAYEXEC) |
404 | #define VM_DATA_FLAGS_EXEC (VM_READ | VM_WRITE | VM_EXEC | \ |
405 | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) |
406 | |
407 | #ifndef VM_DATA_DEFAULT_FLAGS /* arch can override this */ |
408 | #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_EXEC |
409 | #endif |
410 | |
411 | #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */ |
412 | #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS |
413 | #endif |
414 | |
415 | #define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK) |
416 | |
417 | #ifdef CONFIG_STACK_GROWSUP |
418 | #define VM_STACK VM_GROWSUP |
419 | #define VM_STACK_EARLY VM_GROWSDOWN |
420 | #else |
421 | #define VM_STACK VM_GROWSDOWN |
422 | #define VM_STACK_EARLY 0 |
423 | #endif |
424 | |
425 | #define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) |
426 | |
427 | /* VMA basic access permission flags */ |
428 | #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) |
429 | |
430 | |
431 | /* |
432 | * Special vmas that are non-mergable, non-mlock()able. |
433 | */ |
434 | #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP) |
435 | |
436 | /* This mask prevents VMA from being scanned with khugepaged */ |
437 | #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB) |
438 | |
439 | /* This mask defines which mm->def_flags a process can inherit its parent */ |
440 | #define VM_INIT_DEF_MASK VM_NOHUGEPAGE |
441 | |
442 | /* This mask represents all the VMA flag bits used by mlock */ |
443 | #define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT) |
444 | |
445 | /* Arch-specific flags to clear when updating VM flags on protection change */ |
446 | #ifndef VM_ARCH_CLEAR |
447 | # define VM_ARCH_CLEAR VM_NONE |
448 | #endif |
449 | #define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR) |
450 | |
451 | /* |
452 | * mapping from the currently active vm_flags protection bits (the |
453 | * low four bits) to a page protection mask.. |
454 | */ |
455 | |
456 | /* |
457 | * The default fault flags that should be used by most of the |
458 | * arch-specific page fault handlers. |
459 | */ |
460 | #define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ |
461 | FAULT_FLAG_KILLABLE | \ |
462 | FAULT_FLAG_INTERRUPTIBLE) |
463 | |
464 | /** |
465 | * fault_flag_allow_retry_first - check ALLOW_RETRY the first time |
466 | * @flags: Fault flags. |
467 | * |
468 | * This is mostly used for places where we want to try to avoid taking |
469 | * the mmap_lock for too long a time when waiting for another condition |
470 | * to change, in which case we can try to be polite to release the |
471 | * mmap_lock in the first round to avoid potential starvation of other |
472 | * processes that would also want the mmap_lock. |
473 | * |
474 | * Return: true if the page fault allows retry and this is the first |
475 | * attempt of the fault handling; false otherwise. |
476 | */ |
477 | static inline bool fault_flag_allow_retry_first(enum fault_flag flags) |
478 | { |
479 | return (flags & FAULT_FLAG_ALLOW_RETRY) && |
480 | (!(flags & FAULT_FLAG_TRIED)); |
481 | } |
482 | |
483 | #define FAULT_FLAG_TRACE \ |
484 | { FAULT_FLAG_WRITE, "WRITE" }, \ |
485 | { FAULT_FLAG_MKWRITE, "MKWRITE" }, \ |
486 | { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \ |
487 | { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \ |
488 | { FAULT_FLAG_KILLABLE, "KILLABLE" }, \ |
489 | { FAULT_FLAG_TRIED, "TRIED" }, \ |
490 | { FAULT_FLAG_USER, "USER" }, \ |
491 | { FAULT_FLAG_REMOTE, "REMOTE" }, \ |
492 | { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ |
493 | { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ |
494 | { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } |
495 | |
496 | /* |
497 | * vm_fault is filled by the pagefault handler and passed to the vma's |
498 | * ->fault function. The vma's ->fault is responsible for returning a bitmask |
499 | * of VM_FAULT_xxx flags that give details about how the fault was handled. |
500 | * |
501 | * MM layer fills up gfp_mask for page allocations but fault handler might |
502 | * alter it if its implementation requires a different allocation context. |
503 | * |
504 | * pgoff should be used in favour of virtual_address, if possible. |
505 | */ |
506 | struct vm_fault { |
507 | const struct { |
508 | struct vm_area_struct *vma; /* Target VMA */ |
509 | gfp_t gfp_mask; /* gfp mask to be used for allocations */ |
510 | pgoff_t pgoff; /* Logical page offset based on vma */ |
511 | unsigned long address; /* Faulting virtual address - masked */ |
512 | unsigned long real_address; /* Faulting virtual address - unmasked */ |
513 | }; |
514 | enum fault_flag flags; /* FAULT_FLAG_xxx flags |
515 | * XXX: should really be 'const' */ |
516 | pmd_t *pmd; /* Pointer to pmd entry matching |
517 | * the 'address' */ |
518 | pud_t *pud; /* Pointer to pud entry matching |
519 | * the 'address' |
520 | */ |
521 | union { |
522 | pte_t orig_pte; /* Value of PTE at the time of fault */ |
523 | pmd_t orig_pmd; /* Value of PMD at the time of fault, |
524 | * used by PMD fault only. |
525 | */ |
526 | }; |
527 | |
528 | struct page *cow_page; /* Page handler may use for COW fault */ |
529 | struct page *page; /* ->fault handlers should return a |
530 | * page here, unless VM_FAULT_NOPAGE |
531 | * is set (which is also implied by |
532 | * VM_FAULT_ERROR). |
533 | */ |
534 | /* These three entries are valid only while holding ptl lock */ |
535 | pte_t *pte; /* Pointer to pte entry matching |
536 | * the 'address'. NULL if the page |
537 | * table hasn't been allocated. |
538 | */ |
539 | spinlock_t *ptl; /* Page table lock. |
540 | * Protects pte page table if 'pte' |
541 | * is not NULL, otherwise pmd. |
542 | */ |
543 | pgtable_t prealloc_pte; /* Pre-allocated pte page table. |
544 | * vm_ops->map_pages() sets up a page |
545 | * table from atomic context. |
546 | * do_fault_around() pre-allocates |
547 | * page table to avoid allocation from |
548 | * atomic context. |
549 | */ |
550 | }; |
551 | |
552 | /* |
553 | * These are the virtual MM functions - opening of an area, closing and |
554 | * unmapping it (needed to keep files on disk up-to-date etc), pointer |
555 | * to the functions called when a no-page or a wp-page exception occurs. |
556 | */ |
557 | struct vm_operations_struct { |
558 | void (*open)(struct vm_area_struct * area); |
559 | /** |
560 | * @close: Called when the VMA is being removed from the MM. |
561 | * Context: User context. May sleep. Caller holds mmap_lock. |
562 | */ |
563 | void (*close)(struct vm_area_struct * area); |
564 | /* Called any time before splitting to check if it's allowed */ |
565 | int (*may_split)(struct vm_area_struct *area, unsigned long addr); |
566 | int (*mremap)(struct vm_area_struct *area); |
567 | /* |
568 | * Called by mprotect() to make driver-specific permission |
569 | * checks before mprotect() is finalised. The VMA must not |
570 | * be modified. Returns 0 if mprotect() can proceed. |
571 | */ |
572 | int (*mprotect)(struct vm_area_struct *vma, unsigned long start, |
573 | unsigned long end, unsigned long newflags); |
574 | vm_fault_t (*fault)(struct vm_fault *vmf); |
575 | vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); |
576 | vm_fault_t (*map_pages)(struct vm_fault *vmf, |
577 | pgoff_t start_pgoff, pgoff_t end_pgoff); |
578 | unsigned long (*pagesize)(struct vm_area_struct * area); |
579 | |
580 | /* notification that a previously read-only page is about to become |
581 | * writable, if an error is returned it will cause a SIGBUS */ |
582 | vm_fault_t (*page_mkwrite)(struct vm_fault *vmf); |
583 | |
584 | /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ |
585 | vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf); |
586 | |
587 | /* called by access_process_vm when get_user_pages() fails, typically |
588 | * for use by special VMAs. See also generic_access_phys() for a generic |
589 | * implementation useful for any iomem mapping. |
590 | */ |
591 | int (*access)(struct vm_area_struct *vma, unsigned long addr, |
592 | void *buf, int len, int write); |
593 | |
594 | /* Called by the /proc/PID/maps code to ask the vma whether it |
595 | * has a special name. Returning non-NULL will also cause this |
596 | * vma to be dumped unconditionally. */ |
597 | const char *(*name)(struct vm_area_struct *vma); |
598 | |
599 | #ifdef CONFIG_NUMA |
600 | /* |
601 | * set_policy() op must add a reference to any non-NULL @new mempolicy |
602 | * to hold the policy upon return. Caller should pass NULL @new to |
603 | * remove a policy and fall back to surrounding context--i.e. do not |
604 | * install a MPOL_DEFAULT policy, nor the task or system default |
605 | * mempolicy. |
606 | */ |
607 | int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new); |
608 | |
609 | /* |
610 | * get_policy() op must add reference [mpol_get()] to any policy at |
611 | * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure |
612 | * in mm/mempolicy.c will do this automatically. |
613 | * get_policy() must NOT add a ref if the policy at (vma,addr) is not |
614 | * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. |
615 | * If no [shared/vma] mempolicy exists at the addr, get_policy() op |
616 | * must return NULL--i.e., do not "fallback" to task or system default |
617 | * policy. |
618 | */ |
619 | struct mempolicy *(*get_policy)(struct vm_area_struct *vma, |
620 | unsigned long addr, pgoff_t *ilx); |
621 | #endif |
622 | /* |
623 | * Called by vm_normal_page() for special PTEs to find the |
624 | * page for @addr. This is useful if the default behavior |
625 | * (using pte_page()) would not find the correct page. |
626 | */ |
627 | struct page *(*find_special_page)(struct vm_area_struct *vma, |
628 | unsigned long addr); |
629 | }; |
630 | |
631 | #ifdef CONFIG_NUMA_BALANCING |
632 | static inline void vma_numab_state_init(struct vm_area_struct *vma) |
633 | { |
634 | vma->numab_state = NULL; |
635 | } |
636 | static inline void vma_numab_state_free(struct vm_area_struct *vma) |
637 | { |
638 | kfree(objp: vma->numab_state); |
639 | } |
640 | #else |
641 | static inline void vma_numab_state_init(struct vm_area_struct *vma) {} |
642 | static inline void vma_numab_state_free(struct vm_area_struct *vma) {} |
643 | #endif /* CONFIG_NUMA_BALANCING */ |
644 | |
645 | #ifdef CONFIG_PER_VMA_LOCK |
646 | /* |
647 | * Try to read-lock a vma. The function is allowed to occasionally yield false |
648 | * locked result to avoid performance overhead, in which case we fall back to |
649 | * using mmap_lock. The function should never yield false unlocked result. |
650 | */ |
651 | static inline bool vma_start_read(struct vm_area_struct *vma) |
652 | { |
653 | /* |
654 | * Check before locking. A race might cause false locked result. |
655 | * We can use READ_ONCE() for the mm_lock_seq here, and don't need |
656 | * ACQUIRE semantics, because this is just a lockless check whose result |
657 | * we don't rely on for anything - the mm_lock_seq read against which we |
658 | * need ordering is below. |
659 | */ |
660 | if (READ_ONCE(vma->vm_lock_seq) == READ_ONCE(vma->vm_mm->mm_lock_seq)) |
661 | return false; |
662 | |
663 | if (unlikely(down_read_trylock(&vma->vm_lock->lock) == 0)) |
664 | return false; |
665 | |
666 | /* |
667 | * Overflow might produce false locked result. |
668 | * False unlocked result is impossible because we modify and check |
669 | * vma->vm_lock_seq under vma->vm_lock protection and mm->mm_lock_seq |
670 | * modification invalidates all existing locks. |
671 | * |
672 | * We must use ACQUIRE semantics for the mm_lock_seq so that if we are |
673 | * racing with vma_end_write_all(), we only start reading from the VMA |
674 | * after it has been unlocked. |
675 | * This pairs with RELEASE semantics in vma_end_write_all(). |
676 | */ |
677 | if (unlikely(vma->vm_lock_seq == smp_load_acquire(&vma->vm_mm->mm_lock_seq))) { |
678 | up_read(sem: &vma->vm_lock->lock); |
679 | return false; |
680 | } |
681 | return true; |
682 | } |
683 | |
684 | static inline void vma_end_read(struct vm_area_struct *vma) |
685 | { |
686 | rcu_read_lock(); /* keeps vma alive till the end of up_read */ |
687 | up_read(sem: &vma->vm_lock->lock); |
688 | rcu_read_unlock(); |
689 | } |
690 | |
691 | /* WARNING! Can only be used if mmap_lock is expected to be write-locked */ |
692 | static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq) |
693 | { |
694 | mmap_assert_write_locked(mm: vma->vm_mm); |
695 | |
696 | /* |
697 | * current task is holding mmap_write_lock, both vma->vm_lock_seq and |
698 | * mm->mm_lock_seq can't be concurrently modified. |
699 | */ |
700 | *mm_lock_seq = vma->vm_mm->mm_lock_seq; |
701 | return (vma->vm_lock_seq == *mm_lock_seq); |
702 | } |
703 | |
704 | /* |
705 | * Begin writing to a VMA. |
706 | * Exclude concurrent readers under the per-VMA lock until the currently |
707 | * write-locked mmap_lock is dropped or downgraded. |
708 | */ |
709 | static inline void vma_start_write(struct vm_area_struct *vma) |
710 | { |
711 | int mm_lock_seq; |
712 | |
713 | if (__is_vma_write_locked(vma, mm_lock_seq: &mm_lock_seq)) |
714 | return; |
715 | |
716 | down_write(sem: &vma->vm_lock->lock); |
717 | /* |
718 | * We should use WRITE_ONCE() here because we can have concurrent reads |
719 | * from the early lockless pessimistic check in vma_start_read(). |
720 | * We don't really care about the correctness of that early check, but |
721 | * we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy. |
722 | */ |
723 | WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq); |
724 | up_write(sem: &vma->vm_lock->lock); |
725 | } |
726 | |
727 | static inline void vma_assert_write_locked(struct vm_area_struct *vma) |
728 | { |
729 | int mm_lock_seq; |
730 | |
731 | VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); |
732 | } |
733 | |
734 | static inline void vma_assert_locked(struct vm_area_struct *vma) |
735 | { |
736 | if (!rwsem_is_locked(sem: &vma->vm_lock->lock)) |
737 | vma_assert_write_locked(vma); |
738 | } |
739 | |
740 | static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached) |
741 | { |
742 | /* When detaching vma should be write-locked */ |
743 | if (detached) |
744 | vma_assert_write_locked(vma); |
745 | vma->detached = detached; |
746 | } |
747 | |
748 | static inline void release_fault_lock(struct vm_fault *vmf) |
749 | { |
750 | if (vmf->flags & FAULT_FLAG_VMA_LOCK) |
751 | vma_end_read(vma: vmf->vma); |
752 | else |
753 | mmap_read_unlock(mm: vmf->vma->vm_mm); |
754 | } |
755 | |
756 | static inline void assert_fault_locked(struct vm_fault *vmf) |
757 | { |
758 | if (vmf->flags & FAULT_FLAG_VMA_LOCK) |
759 | vma_assert_locked(vma: vmf->vma); |
760 | else |
761 | mmap_assert_locked(mm: vmf->vma->vm_mm); |
762 | } |
763 | |
764 | struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, |
765 | unsigned long address); |
766 | |
767 | #else /* CONFIG_PER_VMA_LOCK */ |
768 | |
769 | static inline bool vma_start_read(struct vm_area_struct *vma) |
770 | { return false; } |
771 | static inline void vma_end_read(struct vm_area_struct *vma) {} |
772 | static inline void vma_start_write(struct vm_area_struct *vma) {} |
773 | static inline void vma_assert_write_locked(struct vm_area_struct *vma) |
774 | { mmap_assert_write_locked(vma->vm_mm); } |
775 | static inline void vma_mark_detached(struct vm_area_struct *vma, |
776 | bool detached) {} |
777 | |
778 | static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, |
779 | unsigned long address) |
780 | { |
781 | return NULL; |
782 | } |
783 | |
784 | static inline void release_fault_lock(struct vm_fault *vmf) |
785 | { |
786 | mmap_read_unlock(vmf->vma->vm_mm); |
787 | } |
788 | |
789 | static inline void assert_fault_locked(struct vm_fault *vmf) |
790 | { |
791 | mmap_assert_locked(vmf->vma->vm_mm); |
792 | } |
793 | |
794 | #endif /* CONFIG_PER_VMA_LOCK */ |
795 | |
796 | extern const struct vm_operations_struct vma_dummy_vm_ops; |
797 | |
798 | /* |
799 | * WARNING: vma_init does not initialize vma->vm_lock. |
800 | * Use vm_area_alloc()/vm_area_free() if vma needs locking. |
801 | */ |
802 | static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) |
803 | { |
804 | memset(vma, 0, sizeof(*vma)); |
805 | vma->vm_mm = mm; |
806 | vma->vm_ops = &vma_dummy_vm_ops; |
807 | INIT_LIST_HEAD(list: &vma->anon_vma_chain); |
808 | vma_mark_detached(vma, detached: false); |
809 | vma_numab_state_init(vma); |
810 | } |
811 | |
812 | /* Use when VMA is not part of the VMA tree and needs no locking */ |
813 | static inline void vm_flags_init(struct vm_area_struct *vma, |
814 | vm_flags_t flags) |
815 | { |
816 | ACCESS_PRIVATE(vma, __vm_flags) = flags; |
817 | } |
818 | |
819 | /* |
820 | * Use when VMA is part of the VMA tree and modifications need coordination |
821 | * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and |
822 | * it should be locked explicitly beforehand. |
823 | */ |
824 | static inline void vm_flags_reset(struct vm_area_struct *vma, |
825 | vm_flags_t flags) |
826 | { |
827 | vma_assert_write_locked(vma); |
828 | vm_flags_init(vma, flags); |
829 | } |
830 | |
831 | static inline void vm_flags_reset_once(struct vm_area_struct *vma, |
832 | vm_flags_t flags) |
833 | { |
834 | vma_assert_write_locked(vma); |
835 | WRITE_ONCE(ACCESS_PRIVATE(vma, __vm_flags), flags); |
836 | } |
837 | |
838 | static inline void vm_flags_set(struct vm_area_struct *vma, |
839 | vm_flags_t flags) |
840 | { |
841 | vma_start_write(vma); |
842 | ACCESS_PRIVATE(vma, __vm_flags) |= flags; |
843 | } |
844 | |
845 | static inline void vm_flags_clear(struct vm_area_struct *vma, |
846 | vm_flags_t flags) |
847 | { |
848 | vma_start_write(vma); |
849 | ACCESS_PRIVATE(vma, __vm_flags) &= ~flags; |
850 | } |
851 | |
852 | /* |
853 | * Use only if VMA is not part of the VMA tree or has no other users and |
854 | * therefore needs no locking. |
855 | */ |
856 | static inline void __vm_flags_mod(struct vm_area_struct *vma, |
857 | vm_flags_t set, vm_flags_t clear) |
858 | { |
859 | vm_flags_init(vma, flags: (vma->vm_flags | set) & ~clear); |
860 | } |
861 | |
862 | /* |
863 | * Use only when the order of set/clear operations is unimportant, otherwise |
864 | * use vm_flags_{set|clear} explicitly. |
865 | */ |
866 | static inline void vm_flags_mod(struct vm_area_struct *vma, |
867 | vm_flags_t set, vm_flags_t clear) |
868 | { |
869 | vma_start_write(vma); |
870 | __vm_flags_mod(vma, set, clear); |
871 | } |
872 | |
873 | static inline void vma_set_anonymous(struct vm_area_struct *vma) |
874 | { |
875 | vma->vm_ops = NULL; |
876 | } |
877 | |
878 | static inline bool vma_is_anonymous(struct vm_area_struct *vma) |
879 | { |
880 | return !vma->vm_ops; |
881 | } |
882 | |
883 | /* |
884 | * Indicate if the VMA is a heap for the given task; for |
885 | * /proc/PID/maps that is the heap of the main task. |
886 | */ |
887 | static inline bool vma_is_initial_heap(const struct vm_area_struct *vma) |
888 | { |
889 | return vma->vm_start <= vma->vm_mm->brk && |
890 | vma->vm_end >= vma->vm_mm->start_brk; |
891 | } |
892 | |
893 | /* |
894 | * Indicate if the VMA is a stack for the given task; for |
895 | * /proc/PID/maps that is the stack of the main task. |
896 | */ |
897 | static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) |
898 | { |
899 | /* |
900 | * We make no effort to guess what a given thread considers to be |
901 | * its "stack". It's not even well-defined for programs written |
902 | * languages like Go. |
903 | */ |
904 | return vma->vm_start <= vma->vm_mm->start_stack && |
905 | vma->vm_end >= vma->vm_mm->start_stack; |
906 | } |
907 | |
908 | static inline bool vma_is_temporary_stack(struct vm_area_struct *vma) |
909 | { |
910 | int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); |
911 | |
912 | if (!maybe_stack) |
913 | return false; |
914 | |
915 | if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == |
916 | VM_STACK_INCOMPLETE_SETUP) |
917 | return true; |
918 | |
919 | return false; |
920 | } |
921 | |
922 | static inline bool vma_is_foreign(struct vm_area_struct *vma) |
923 | { |
924 | if (!current->mm) |
925 | return true; |
926 | |
927 | if (current->mm != vma->vm_mm) |
928 | return true; |
929 | |
930 | return false; |
931 | } |
932 | |
933 | static inline bool vma_is_accessible(struct vm_area_struct *vma) |
934 | { |
935 | return vma->vm_flags & VM_ACCESS_FLAGS; |
936 | } |
937 | |
938 | static inline bool is_shared_maywrite(vm_flags_t vm_flags) |
939 | { |
940 | return (vm_flags & (VM_SHARED | VM_MAYWRITE)) == |
941 | (VM_SHARED | VM_MAYWRITE); |
942 | } |
943 | |
944 | static inline bool vma_is_shared_maywrite(struct vm_area_struct *vma) |
945 | { |
946 | return is_shared_maywrite(vm_flags: vma->vm_flags); |
947 | } |
948 | |
949 | static inline |
950 | struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) |
951 | { |
952 | return mas_find(mas: &vmi->mas, max: max - 1); |
953 | } |
954 | |
955 | static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) |
956 | { |
957 | /* |
958 | * Uses mas_find() to get the first VMA when the iterator starts. |
959 | * Calling mas_next() could skip the first entry. |
960 | */ |
961 | return mas_find(mas: &vmi->mas, ULONG_MAX); |
962 | } |
963 | |
964 | static inline |
965 | struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) |
966 | { |
967 | return mas_next_range(mas: &vmi->mas, ULONG_MAX); |
968 | } |
969 | |
970 | |
971 | static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) |
972 | { |
973 | return mas_prev(mas: &vmi->mas, min: 0); |
974 | } |
975 | |
976 | static inline |
977 | struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi) |
978 | { |
979 | return mas_prev_range(mas: &vmi->mas, max: 0); |
980 | } |
981 | |
982 | static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) |
983 | { |
984 | return vmi->mas.index; |
985 | } |
986 | |
987 | static inline unsigned long vma_iter_end(struct vma_iterator *vmi) |
988 | { |
989 | return vmi->mas.last + 1; |
990 | } |
991 | static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi, |
992 | unsigned long count) |
993 | { |
994 | return mas_expected_entries(mas: &vmi->mas, nr_entries: count); |
995 | } |
996 | |
997 | /* Free any unused preallocations */ |
998 | static inline void vma_iter_free(struct vma_iterator *vmi) |
999 | { |
1000 | mas_destroy(mas: &vmi->mas); |
1001 | } |
1002 | |
1003 | static inline int vma_iter_bulk_store(struct vma_iterator *vmi, |
1004 | struct vm_area_struct *vma) |
1005 | { |
1006 | vmi->mas.index = vma->vm_start; |
1007 | vmi->mas.last = vma->vm_end - 1; |
1008 | mas_store(mas: &vmi->mas, entry: vma); |
1009 | if (unlikely(mas_is_err(&vmi->mas))) |
1010 | return -ENOMEM; |
1011 | |
1012 | return 0; |
1013 | } |
1014 | |
1015 | static inline void vma_iter_invalidate(struct vma_iterator *vmi) |
1016 | { |
1017 | mas_pause(mas: &vmi->mas); |
1018 | } |
1019 | |
1020 | static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) |
1021 | { |
1022 | mas_set(mas: &vmi->mas, index: addr); |
1023 | } |
1024 | |
1025 | #define for_each_vma(__vmi, __vma) \ |
1026 | while (((__vma) = vma_next(&(__vmi))) != NULL) |
1027 | |
1028 | /* The MM code likes to work with exclusive end addresses */ |
1029 | #define for_each_vma_range(__vmi, __vma, __end) \ |
1030 | while (((__vma) = vma_find(&(__vmi), (__end))) != NULL) |
1031 | |
1032 | #ifdef CONFIG_SHMEM |
1033 | /* |
1034 | * The vma_is_shmem is not inline because it is used only by slow |
1035 | * paths in userfault. |
1036 | */ |
1037 | bool vma_is_shmem(struct vm_area_struct *vma); |
1038 | bool vma_is_anon_shmem(struct vm_area_struct *vma); |
1039 | #else |
1040 | static inline bool vma_is_shmem(struct vm_area_struct *vma) { return false; } |
1041 | static inline bool vma_is_anon_shmem(struct vm_area_struct *vma) { return false; } |
1042 | #endif |
1043 | |
1044 | int vma_is_stack_for_current(struct vm_area_struct *vma); |
1045 | |
1046 | /* flush_tlb_range() takes a vma, not a mm, and can care about flags */ |
1047 | #define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) } |
1048 | |
1049 | struct mmu_gather; |
1050 | struct inode; |
1051 | |
1052 | /* |
1053 | * compound_order() can be called without holding a reference, which means |
1054 | * that niceties like page_folio() don't work. These callers should be |
1055 | * prepared to handle wild return values. For example, PG_head may be |
1056 | * set before the order is initialised, or this may be a tail page. |
1057 | * See compaction.c for some good examples. |
1058 | */ |
1059 | static inline unsigned int compound_order(struct page *page) |
1060 | { |
1061 | struct folio *folio = (struct folio *)page; |
1062 | |
1063 | if (!test_bit(PG_head, &folio->flags)) |
1064 | return 0; |
1065 | return folio->_flags_1 & 0xff; |
1066 | } |
1067 | |
1068 | /** |
1069 | * folio_order - The allocation order of a folio. |
1070 | * @folio: The folio. |
1071 | * |
1072 | * A folio is composed of 2^order pages. See get_order() for the definition |
1073 | * of order. |
1074 | * |
1075 | * Return: The order of the folio. |
1076 | */ |
1077 | static inline unsigned int folio_order(struct folio *folio) |
1078 | { |
1079 | if (!folio_test_large(folio)) |
1080 | return 0; |
1081 | return folio->_flags_1 & 0xff; |
1082 | } |
1083 | |
1084 | #include <linux/huge_mm.h> |
1085 | |
1086 | /* |
1087 | * Methods to modify the page usage count. |
1088 | * |
1089 | * What counts for a page usage: |
1090 | * - cache mapping (page->mapping) |
1091 | * - private data (page->private) |
1092 | * - page mapped in a task's page tables, each mapping |
1093 | * is counted separately |
1094 | * |
1095 | * Also, many kernel routines increase the page count before a critical |
1096 | * routine so they can be sure the page doesn't go away from under them. |
1097 | */ |
1098 | |
1099 | /* |
1100 | * Drop a ref, return true if the refcount fell to zero (the page has no users) |
1101 | */ |
1102 | static inline int put_page_testzero(struct page *page) |
1103 | { |
1104 | VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); |
1105 | return page_ref_dec_and_test(page); |
1106 | } |
1107 | |
1108 | static inline int folio_put_testzero(struct folio *folio) |
1109 | { |
1110 | return put_page_testzero(page: &folio->page); |
1111 | } |
1112 | |
1113 | /* |
1114 | * Try to grab a ref unless the page has a refcount of zero, return false if |
1115 | * that is the case. |
1116 | * This can be called when MMU is off so it must not access |
1117 | * any of the virtual mappings. |
1118 | */ |
1119 | static inline bool get_page_unless_zero(struct page *page) |
1120 | { |
1121 | return page_ref_add_unless(page, nr: 1, u: 0); |
1122 | } |
1123 | |
1124 | static inline struct folio *folio_get_nontail_page(struct page *page) |
1125 | { |
1126 | if (unlikely(!get_page_unless_zero(page))) |
1127 | return NULL; |
1128 | return (struct folio *)page; |
1129 | } |
1130 | |
1131 | extern int page_is_ram(unsigned long pfn); |
1132 | |
1133 | enum { |
1134 | REGION_INTERSECTS, |
1135 | REGION_DISJOINT, |
1136 | REGION_MIXED, |
1137 | }; |
1138 | |
1139 | int region_intersects(resource_size_t offset, size_t size, unsigned long flags, |
1140 | unsigned long desc); |
1141 | |
1142 | /* Support for virtually mapped pages */ |
1143 | struct page *vmalloc_to_page(const void *addr); |
1144 | unsigned long vmalloc_to_pfn(const void *addr); |
1145 | |
1146 | /* |
1147 | * Determine if an address is within the vmalloc range |
1148 | * |
1149 | * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there |
1150 | * is no special casing required. |
1151 | */ |
1152 | #ifdef CONFIG_MMU |
1153 | extern bool is_vmalloc_addr(const void *x); |
1154 | extern int is_vmalloc_or_module_addr(const void *x); |
1155 | #else |
1156 | static inline bool is_vmalloc_addr(const void *x) |
1157 | { |
1158 | return false; |
1159 | } |
1160 | static inline int is_vmalloc_or_module_addr(const void *x) |
1161 | { |
1162 | return 0; |
1163 | } |
1164 | #endif |
1165 | |
1166 | /* |
1167 | * How many times the entire folio is mapped as a single unit (eg by a |
1168 | * PMD or PUD entry). This is probably not what you want, except for |
1169 | * debugging purposes - it does not include PTE-mapped sub-pages; look |
1170 | * at folio_mapcount() or page_mapcount() or total_mapcount() instead. |
1171 | */ |
1172 | static inline int folio_entire_mapcount(struct folio *folio) |
1173 | { |
1174 | VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); |
1175 | return atomic_read(v: &folio->_entire_mapcount) + 1; |
1176 | } |
1177 | |
1178 | /* |
1179 | * The atomic page->_mapcount, starts from -1: so that transitions |
1180 | * both from it and to it can be tracked, using atomic_inc_and_test |
1181 | * and atomic_add_negative(-1). |
1182 | */ |
1183 | static inline void page_mapcount_reset(struct page *page) |
1184 | { |
1185 | atomic_set(v: &(page)->_mapcount, i: -1); |
1186 | } |
1187 | |
1188 | /** |
1189 | * page_mapcount() - Number of times this precise page is mapped. |
1190 | * @page: The page. |
1191 | * |
1192 | * The number of times this page is mapped. If this page is part of |
1193 | * a large folio, it includes the number of times this page is mapped |
1194 | * as part of that folio. |
1195 | * |
1196 | * The result is undefined for pages which cannot be mapped into userspace. |
1197 | * For example SLAB or special types of pages. See function page_has_type(). |
1198 | * They use this field in struct page differently. |
1199 | */ |
1200 | static inline int page_mapcount(struct page *page) |
1201 | { |
1202 | int mapcount = atomic_read(v: &page->_mapcount) + 1; |
1203 | |
1204 | if (unlikely(PageCompound(page))) |
1205 | mapcount += folio_entire_mapcount(page_folio(page)); |
1206 | |
1207 | return mapcount; |
1208 | } |
1209 | |
1210 | int folio_total_mapcount(struct folio *folio); |
1211 | |
1212 | /** |
1213 | * folio_mapcount() - Calculate the number of mappings of this folio. |
1214 | * @folio: The folio. |
1215 | * |
1216 | * A large folio tracks both how many times the entire folio is mapped, |
1217 | * and how many times each individual page in the folio is mapped. |
1218 | * This function calculates the total number of times the folio is |
1219 | * mapped. |
1220 | * |
1221 | * Return: The number of times this folio is mapped. |
1222 | */ |
1223 | static inline int folio_mapcount(struct folio *folio) |
1224 | { |
1225 | if (likely(!folio_test_large(folio))) |
1226 | return atomic_read(v: &folio->_mapcount) + 1; |
1227 | return folio_total_mapcount(folio); |
1228 | } |
1229 | |
1230 | static inline int total_mapcount(struct page *page) |
1231 | { |
1232 | if (likely(!PageCompound(page))) |
1233 | return atomic_read(v: &page->_mapcount) + 1; |
1234 | return folio_total_mapcount(page_folio(page)); |
1235 | } |
1236 | |
1237 | static inline bool folio_large_is_mapped(struct folio *folio) |
1238 | { |
1239 | /* |
1240 | * Reading _entire_mapcount below could be omitted if hugetlb |
1241 | * participated in incrementing nr_pages_mapped when compound mapped. |
1242 | */ |
1243 | return atomic_read(v: &folio->_nr_pages_mapped) > 0 || |
1244 | atomic_read(v: &folio->_entire_mapcount) >= 0; |
1245 | } |
1246 | |
1247 | /** |
1248 | * folio_mapped - Is this folio mapped into userspace? |
1249 | * @folio: The folio. |
1250 | * |
1251 | * Return: True if any page in this folio is referenced by user page tables. |
1252 | */ |
1253 | static inline bool folio_mapped(struct folio *folio) |
1254 | { |
1255 | if (likely(!folio_test_large(folio))) |
1256 | return atomic_read(v: &folio->_mapcount) >= 0; |
1257 | return folio_large_is_mapped(folio); |
1258 | } |
1259 | |
1260 | /* |
1261 | * Return true if this page is mapped into pagetables. |
1262 | * For compound page it returns true if any sub-page of compound page is mapped, |
1263 | * even if this particular sub-page is not itself mapped by any PTE or PMD. |
1264 | */ |
1265 | static inline bool page_mapped(struct page *page) |
1266 | { |
1267 | if (likely(!PageCompound(page))) |
1268 | return atomic_read(v: &page->_mapcount) >= 0; |
1269 | return folio_large_is_mapped(page_folio(page)); |
1270 | } |
1271 | |
1272 | static inline struct page *virt_to_head_page(const void *x) |
1273 | { |
1274 | struct page *page = virt_to_page(x); |
1275 | |
1276 | return compound_head(page); |
1277 | } |
1278 | |
1279 | static inline struct folio *virt_to_folio(const void *x) |
1280 | { |
1281 | struct page *page = virt_to_page(x); |
1282 | |
1283 | return page_folio(page); |
1284 | } |
1285 | |
1286 | void __folio_put(struct folio *folio); |
1287 | |
1288 | void put_pages_list(struct list_head *pages); |
1289 | |
1290 | void split_page(struct page *page, unsigned int order); |
1291 | void folio_copy(struct folio *dst, struct folio *src); |
1292 | |
1293 | unsigned long nr_free_buffer_pages(void); |
1294 | |
1295 | void destroy_large_folio(struct folio *folio); |
1296 | |
1297 | /* Returns the number of bytes in this potentially compound page. */ |
1298 | static inline unsigned long page_size(struct page *page) |
1299 | { |
1300 | return PAGE_SIZE << compound_order(page); |
1301 | } |
1302 | |
1303 | /* Returns the number of bits needed for the number of bytes in a page */ |
1304 | static inline unsigned int page_shift(struct page *page) |
1305 | { |
1306 | return PAGE_SHIFT + compound_order(page); |
1307 | } |
1308 | |
1309 | /** |
1310 | * thp_order - Order of a transparent huge page. |
1311 | * @page: Head page of a transparent huge page. |
1312 | */ |
1313 | static inline unsigned int thp_order(struct page *page) |
1314 | { |
1315 | VM_BUG_ON_PGFLAGS(PageTail(page), page); |
1316 | return compound_order(page); |
1317 | } |
1318 | |
1319 | /** |
1320 | * thp_size - Size of a transparent huge page. |
1321 | * @page: Head page of a transparent huge page. |
1322 | * |
1323 | * Return: Number of bytes in this page. |
1324 | */ |
1325 | static inline unsigned long thp_size(struct page *page) |
1326 | { |
1327 | return PAGE_SIZE << thp_order(page); |
1328 | } |
1329 | |
1330 | #ifdef CONFIG_MMU |
1331 | /* |
1332 | * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when |
1333 | * servicing faults for write access. In the normal case, do always want |
1334 | * pte_mkwrite. But get_user_pages can cause write faults for mappings |
1335 | * that do not have writing enabled, when used by access_process_vm. |
1336 | */ |
1337 | static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) |
1338 | { |
1339 | if (likely(vma->vm_flags & VM_WRITE)) |
1340 | pte = pte_mkwrite(pte, vma); |
1341 | return pte; |
1342 | } |
1343 | |
1344 | vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); |
1345 | void set_pte_range(struct vm_fault *vmf, struct folio *folio, |
1346 | struct page *page, unsigned int nr, unsigned long addr); |
1347 | |
1348 | vm_fault_t finish_fault(struct vm_fault *vmf); |
1349 | #endif |
1350 | |
1351 | /* |
1352 | * Multiple processes may "see" the same page. E.g. for untouched |
1353 | * mappings of /dev/null, all processes see the same page full of |
1354 | * zeroes, and text pages of executables and shared libraries have |
1355 | * only one copy in memory, at most, normally. |
1356 | * |
1357 | * For the non-reserved pages, page_count(page) denotes a reference count. |
1358 | * page_count() == 0 means the page is free. page->lru is then used for |
1359 | * freelist management in the buddy allocator. |
1360 | * page_count() > 0 means the page has been allocated. |
1361 | * |
1362 | * Pages are allocated by the slab allocator in order to provide memory |
1363 | * to kmalloc and kmem_cache_alloc. In this case, the management of the |
1364 | * page, and the fields in 'struct page' are the responsibility of mm/slab.c |
1365 | * unless a particular usage is carefully commented. (the responsibility of |
1366 | * freeing the kmalloc memory is the caller's, of course). |
1367 | * |
1368 | * A page may be used by anyone else who does a __get_free_page(). |
1369 | * In this case, page_count still tracks the references, and should only |
1370 | * be used through the normal accessor functions. The top bits of page->flags |
1371 | * and page->virtual store page management information, but all other fields |
1372 | * are unused and could be used privately, carefully. The management of this |
1373 | * page is the responsibility of the one who allocated it, and those who have |
1374 | * subsequently been given references to it. |
1375 | * |
1376 | * The other pages (we may call them "pagecache pages") are completely |
1377 | * managed by the Linux memory manager: I/O, buffers, swapping etc. |
1378 | * The following discussion applies only to them. |
1379 | * |
1380 | * A pagecache page contains an opaque `private' member, which belongs to the |
1381 | * page's address_space. Usually, this is the address of a circular list of |
1382 | * the page's disk buffers. PG_private must be set to tell the VM to call |
1383 | * into the filesystem to release these pages. |
1384 | * |
1385 | * A page may belong to an inode's memory mapping. In this case, page->mapping |
1386 | * is the pointer to the inode, and page->index is the file offset of the page, |
1387 | * in units of PAGE_SIZE. |
1388 | * |
1389 | * If pagecache pages are not associated with an inode, they are said to be |
1390 | * anonymous pages. These may become associated with the swapcache, and in that |
1391 | * case PG_swapcache is set, and page->private is an offset into the swapcache. |
1392 | * |
1393 | * In either case (swapcache or inode backed), the pagecache itself holds one |
1394 | * reference to the page. Setting PG_private should also increment the |
1395 | * refcount. The each user mapping also has a reference to the page. |
1396 | * |
1397 | * The pagecache pages are stored in a per-mapping radix tree, which is |
1398 | * rooted at mapping->i_pages, and indexed by offset. |
1399 | * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space |
1400 | * lists, we instead now tag pages as dirty/writeback in the radix tree. |
1401 | * |
1402 | * All pagecache pages may be subject to I/O: |
1403 | * - inode pages may need to be read from disk, |
1404 | * - inode pages which have been modified and are MAP_SHARED may need |
1405 | * to be written back to the inode on disk, |
1406 | * - anonymous pages (including MAP_PRIVATE file mappings) which have been |
1407 | * modified may need to be swapped out to swap space and (later) to be read |
1408 | * back into memory. |
1409 | */ |
1410 | |
1411 | #if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX) |
1412 | DECLARE_STATIC_KEY_FALSE(devmap_managed_key); |
1413 | |
1414 | bool __put_devmap_managed_page_refs(struct page *page, int refs); |
1415 | static inline bool put_devmap_managed_page_refs(struct page *page, int refs) |
1416 | { |
1417 | if (!static_branch_unlikely(&devmap_managed_key)) |
1418 | return false; |
1419 | if (!is_zone_device_page(page)) |
1420 | return false; |
1421 | return __put_devmap_managed_page_refs(page, refs); |
1422 | } |
1423 | #else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ |
1424 | static inline bool put_devmap_managed_page_refs(struct page *page, int refs) |
1425 | { |
1426 | return false; |
1427 | } |
1428 | #endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ |
1429 | |
1430 | static inline bool put_devmap_managed_page(struct page *page) |
1431 | { |
1432 | return put_devmap_managed_page_refs(page, refs: 1); |
1433 | } |
1434 | |
1435 | /* 127: arbitrary random number, small enough to assemble well */ |
1436 | #define folio_ref_zero_or_close_to_overflow(folio) \ |
1437 | ((unsigned int) folio_ref_count(folio) + 127u <= 127u) |
1438 | |
1439 | /** |
1440 | * folio_get - Increment the reference count on a folio. |
1441 | * @folio: The folio. |
1442 | * |
1443 | * Context: May be called in any context, as long as you know that |
1444 | * you have a refcount on the folio. If you do not already have one, |
1445 | * folio_try_get() may be the right interface for you to use. |
1446 | */ |
1447 | static inline void folio_get(struct folio *folio) |
1448 | { |
1449 | VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); |
1450 | folio_ref_inc(folio); |
1451 | } |
1452 | |
1453 | static inline void get_page(struct page *page) |
1454 | { |
1455 | folio_get(page_folio(page)); |
1456 | } |
1457 | |
1458 | static inline __must_check bool try_get_page(struct page *page) |
1459 | { |
1460 | page = compound_head(page); |
1461 | if (WARN_ON_ONCE(page_ref_count(page) <= 0)) |
1462 | return false; |
1463 | page_ref_inc(page); |
1464 | return true; |
1465 | } |
1466 | |
1467 | /** |
1468 | * folio_put - Decrement the reference count on a folio. |
1469 | * @folio: The folio. |
1470 | * |
1471 | * If the folio's reference count reaches zero, the memory will be |
1472 | * released back to the page allocator and may be used by another |
1473 | * allocation immediately. Do not access the memory or the struct folio |
1474 | * after calling folio_put() unless you can be sure that it wasn't the |
1475 | * last reference. |
1476 | * |
1477 | * Context: May be called in process or interrupt context, but not in NMI |
1478 | * context. May be called while holding a spinlock. |
1479 | */ |
1480 | static inline void folio_put(struct folio *folio) |
1481 | { |
1482 | if (folio_put_testzero(folio)) |
1483 | __folio_put(folio); |
1484 | } |
1485 | |
1486 | /** |
1487 | * folio_put_refs - Reduce the reference count on a folio. |
1488 | * @folio: The folio. |
1489 | * @refs: The amount to subtract from the folio's reference count. |
1490 | * |
1491 | * If the folio's reference count reaches zero, the memory will be |
1492 | * released back to the page allocator and may be used by another |
1493 | * allocation immediately. Do not access the memory or the struct folio |
1494 | * after calling folio_put_refs() unless you can be sure that these weren't |
1495 | * the last references. |
1496 | * |
1497 | * Context: May be called in process or interrupt context, but not in NMI |
1498 | * context. May be called while holding a spinlock. |
1499 | */ |
1500 | static inline void folio_put_refs(struct folio *folio, int refs) |
1501 | { |
1502 | if (folio_ref_sub_and_test(folio, nr: refs)) |
1503 | __folio_put(folio); |
1504 | } |
1505 | |
1506 | /* |
1507 | * union release_pages_arg - an array of pages or folios |
1508 | * |
1509 | * release_pages() releases a simple array of multiple pages, and |
1510 | * accepts various different forms of said page array: either |
1511 | * a regular old boring array of pages, an array of folios, or |
1512 | * an array of encoded page pointers. |
1513 | * |
1514 | * The transparent union syntax for this kind of "any of these |
1515 | * argument types" is all kinds of ugly, so look away. |
1516 | */ |
1517 | typedef union { |
1518 | struct page **pages; |
1519 | struct folio **folios; |
1520 | struct encoded_page **encoded_pages; |
1521 | } release_pages_arg __attribute__ ((__transparent_union__)); |
1522 | |
1523 | void release_pages(release_pages_arg, int nr); |
1524 | |
1525 | /** |
1526 | * folios_put - Decrement the reference count on an array of folios. |
1527 | * @folios: The folios. |
1528 | * @nr: How many folios there are. |
1529 | * |
1530 | * Like folio_put(), but for an array of folios. This is more efficient |
1531 | * than writing the loop yourself as it will optimise the locks which |
1532 | * need to be taken if the folios are freed. |
1533 | * |
1534 | * Context: May be called in process or interrupt context, but not in NMI |
1535 | * context. May be called while holding a spinlock. |
1536 | */ |
1537 | static inline void folios_put(struct folio **folios, unsigned int nr) |
1538 | { |
1539 | release_pages(folios, nr); |
1540 | } |
1541 | |
1542 | static inline void put_page(struct page *page) |
1543 | { |
1544 | struct folio *folio = page_folio(page); |
1545 | |
1546 | /* |
1547 | * For some devmap managed pages we need to catch refcount transition |
1548 | * from 2 to 1: |
1549 | */ |
1550 | if (put_devmap_managed_page(page: &folio->page)) |
1551 | return; |
1552 | folio_put(folio); |
1553 | } |
1554 | |
1555 | /* |
1556 | * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload |
1557 | * the page's refcount so that two separate items are tracked: the original page |
1558 | * reference count, and also a new count of how many pin_user_pages() calls were |
1559 | * made against the page. ("gup-pinned" is another term for the latter). |
1560 | * |
1561 | * With this scheme, pin_user_pages() becomes special: such pages are marked as |
1562 | * distinct from normal pages. As such, the unpin_user_page() call (and its |
1563 | * variants) must be used in order to release gup-pinned pages. |
1564 | * |
1565 | * Choice of value: |
1566 | * |
1567 | * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference |
1568 | * counts with respect to pin_user_pages() and unpin_user_page() becomes |
1569 | * simpler, due to the fact that adding an even power of two to the page |
1570 | * refcount has the effect of using only the upper N bits, for the code that |
1571 | * counts up using the bias value. This means that the lower bits are left for |
1572 | * the exclusive use of the original code that increments and decrements by one |
1573 | * (or at least, by much smaller values than the bias value). |
1574 | * |
1575 | * Of course, once the lower bits overflow into the upper bits (and this is |
1576 | * OK, because subtraction recovers the original values), then visual inspection |
1577 | * no longer suffices to directly view the separate counts. However, for normal |
1578 | * applications that don't have huge page reference counts, this won't be an |
1579 | * issue. |
1580 | * |
1581 | * Locking: the lockless algorithm described in folio_try_get_rcu() |
1582 | * provides safe operation for get_user_pages(), page_mkclean() and |
1583 | * other calls that race to set up page table entries. |
1584 | */ |
1585 | #define GUP_PIN_COUNTING_BIAS (1U << 10) |
1586 | |
1587 | void unpin_user_page(struct page *page); |
1588 | void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, |
1589 | bool make_dirty); |
1590 | void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, |
1591 | bool make_dirty); |
1592 | void unpin_user_pages(struct page **pages, unsigned long npages); |
1593 | |
1594 | static inline bool is_cow_mapping(vm_flags_t flags) |
1595 | { |
1596 | return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; |
1597 | } |
1598 | |
1599 | #ifndef CONFIG_MMU |
1600 | static inline bool is_nommu_shared_mapping(vm_flags_t flags) |
1601 | { |
1602 | /* |
1603 | * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected |
1604 | * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of |
1605 | * a file mapping. R/O MAP_PRIVATE mappings might still modify |
1606 | * underlying memory if ptrace is active, so this is only possible if |
1607 | * ptrace does not apply. Note that there is no mprotect() to upgrade |
1608 | * write permissions later. |
1609 | */ |
1610 | return flags & (VM_MAYSHARE | VM_MAYOVERLAY); |
1611 | } |
1612 | #endif |
1613 | |
1614 | #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) |
1615 | #define SECTION_IN_PAGE_FLAGS |
1616 | #endif |
1617 | |
1618 | /* |
1619 | * The identification function is mainly used by the buddy allocator for |
1620 | * determining if two pages could be buddies. We are not really identifying |
1621 | * the zone since we could be using the section number id if we do not have |
1622 | * node id available in page flags. |
1623 | * We only guarantee that it will return the same value for two combinable |
1624 | * pages in a zone. |
1625 | */ |
1626 | static inline int page_zone_id(struct page *page) |
1627 | { |
1628 | return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; |
1629 | } |
1630 | |
1631 | #ifdef NODE_NOT_IN_PAGE_FLAGS |
1632 | extern int page_to_nid(const struct page *page); |
1633 | #else |
1634 | static inline int page_to_nid(const struct page *page) |
1635 | { |
1636 | struct page *p = (struct page *)page; |
1637 | |
1638 | return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK; |
1639 | } |
1640 | #endif |
1641 | |
1642 | static inline int folio_nid(const struct folio *folio) |
1643 | { |
1644 | return page_to_nid(page: &folio->page); |
1645 | } |
1646 | |
1647 | #ifdef CONFIG_NUMA_BALANCING |
1648 | /* page access time bits needs to hold at least 4 seconds */ |
1649 | #define PAGE_ACCESS_TIME_MIN_BITS 12 |
1650 | #if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS |
1651 | #define PAGE_ACCESS_TIME_BUCKETS \ |
1652 | (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT) |
1653 | #else |
1654 | #define PAGE_ACCESS_TIME_BUCKETS 0 |
1655 | #endif |
1656 | |
1657 | #define PAGE_ACCESS_TIME_MASK \ |
1658 | (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS) |
1659 | |
1660 | static inline int cpu_pid_to_cpupid(int cpu, int pid) |
1661 | { |
1662 | return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); |
1663 | } |
1664 | |
1665 | static inline int cpupid_to_pid(int cpupid) |
1666 | { |
1667 | return cpupid & LAST__PID_MASK; |
1668 | } |
1669 | |
1670 | static inline int cpupid_to_cpu(int cpupid) |
1671 | { |
1672 | return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK; |
1673 | } |
1674 | |
1675 | static inline int cpupid_to_nid(int cpupid) |
1676 | { |
1677 | return cpu_to_node(cpu: cpupid_to_cpu(cpupid)); |
1678 | } |
1679 | |
1680 | static inline bool cpupid_pid_unset(int cpupid) |
1681 | { |
1682 | return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK); |
1683 | } |
1684 | |
1685 | static inline bool cpupid_cpu_unset(int cpupid) |
1686 | { |
1687 | return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK); |
1688 | } |
1689 | |
1690 | static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid) |
1691 | { |
1692 | return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid); |
1693 | } |
1694 | |
1695 | #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid) |
1696 | #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS |
1697 | static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) |
1698 | { |
1699 | return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK); |
1700 | } |
1701 | |
1702 | static inline int folio_last_cpupid(struct folio *folio) |
1703 | { |
1704 | return folio->_last_cpupid; |
1705 | } |
1706 | static inline void page_cpupid_reset_last(struct page *page) |
1707 | { |
1708 | page->_last_cpupid = -1 & LAST_CPUPID_MASK; |
1709 | } |
1710 | #else |
1711 | static inline int folio_last_cpupid(struct folio *folio) |
1712 | { |
1713 | return (folio->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; |
1714 | } |
1715 | |
1716 | int folio_xchg_last_cpupid(struct folio *folio, int cpupid); |
1717 | |
1718 | static inline void page_cpupid_reset_last(struct page *page) |
1719 | { |
1720 | page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; |
1721 | } |
1722 | #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ |
1723 | |
1724 | static inline int folio_xchg_access_time(struct folio *folio, int time) |
1725 | { |
1726 | int last_time; |
1727 | |
1728 | last_time = folio_xchg_last_cpupid(folio, |
1729 | time >> PAGE_ACCESS_TIME_BUCKETS); |
1730 | return last_time << PAGE_ACCESS_TIME_BUCKETS; |
1731 | } |
1732 | |
1733 | static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) |
1734 | { |
1735 | unsigned int pid_bit; |
1736 | |
1737 | pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); |
1738 | if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) { |
1739 | __set_bit(pid_bit, &vma->numab_state->pids_active[1]); |
1740 | } |
1741 | } |
1742 | #else /* !CONFIG_NUMA_BALANCING */ |
1743 | static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) |
1744 | { |
1745 | return folio_nid(folio); /* XXX */ |
1746 | } |
1747 | |
1748 | static inline int folio_xchg_access_time(struct folio *folio, int time) |
1749 | { |
1750 | return 0; |
1751 | } |
1752 | |
1753 | static inline int folio_last_cpupid(struct folio *folio) |
1754 | { |
1755 | return folio_nid(folio); /* XXX */ |
1756 | } |
1757 | |
1758 | static inline int cpupid_to_nid(int cpupid) |
1759 | { |
1760 | return -1; |
1761 | } |
1762 | |
1763 | static inline int cpupid_to_pid(int cpupid) |
1764 | { |
1765 | return -1; |
1766 | } |
1767 | |
1768 | static inline int cpupid_to_cpu(int cpupid) |
1769 | { |
1770 | return -1; |
1771 | } |
1772 | |
1773 | static inline int cpu_pid_to_cpupid(int nid, int pid) |
1774 | { |
1775 | return -1; |
1776 | } |
1777 | |
1778 | static inline bool cpupid_pid_unset(int cpupid) |
1779 | { |
1780 | return true; |
1781 | } |
1782 | |
1783 | static inline void page_cpupid_reset_last(struct page *page) |
1784 | { |
1785 | } |
1786 | |
1787 | static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) |
1788 | { |
1789 | return false; |
1790 | } |
1791 | |
1792 | static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) |
1793 | { |
1794 | } |
1795 | #endif /* CONFIG_NUMA_BALANCING */ |
1796 | |
1797 | #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) |
1798 | |
1799 | /* |
1800 | * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid |
1801 | * setting tags for all pages to native kernel tag value 0xff, as the default |
1802 | * value 0x00 maps to 0xff. |
1803 | */ |
1804 | |
1805 | static inline u8 page_kasan_tag(const struct page *page) |
1806 | { |
1807 | u8 tag = 0xff; |
1808 | |
1809 | if (kasan_enabled()) { |
1810 | tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; |
1811 | tag ^= 0xff; |
1812 | } |
1813 | |
1814 | return tag; |
1815 | } |
1816 | |
1817 | static inline void page_kasan_tag_set(struct page *page, u8 tag) |
1818 | { |
1819 | unsigned long old_flags, flags; |
1820 | |
1821 | if (!kasan_enabled()) |
1822 | return; |
1823 | |
1824 | tag ^= 0xff; |
1825 | old_flags = READ_ONCE(page->flags); |
1826 | do { |
1827 | flags = old_flags; |
1828 | flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); |
1829 | flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; |
1830 | } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags))); |
1831 | } |
1832 | |
1833 | static inline void page_kasan_tag_reset(struct page *page) |
1834 | { |
1835 | if (kasan_enabled()) |
1836 | page_kasan_tag_set(page, 0xff); |
1837 | } |
1838 | |
1839 | #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ |
1840 | |
1841 | static inline u8 page_kasan_tag(const struct page *page) |
1842 | { |
1843 | return 0xff; |
1844 | } |
1845 | |
1846 | static inline void page_kasan_tag_set(struct page *page, u8 tag) { } |
1847 | static inline void page_kasan_tag_reset(struct page *page) { } |
1848 | |
1849 | #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ |
1850 | |
1851 | static inline struct zone *page_zone(const struct page *page) |
1852 | { |
1853 | return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; |
1854 | } |
1855 | |
1856 | static inline pg_data_t *page_pgdat(const struct page *page) |
1857 | { |
1858 | return NODE_DATA(page_to_nid(page)); |
1859 | } |
1860 | |
1861 | static inline struct zone *folio_zone(const struct folio *folio) |
1862 | { |
1863 | return page_zone(page: &folio->page); |
1864 | } |
1865 | |
1866 | static inline pg_data_t *folio_pgdat(const struct folio *folio) |
1867 | { |
1868 | return page_pgdat(page: &folio->page); |
1869 | } |
1870 | |
1871 | #ifdef SECTION_IN_PAGE_FLAGS |
1872 | static inline void set_page_section(struct page *page, unsigned long section) |
1873 | { |
1874 | page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); |
1875 | page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; |
1876 | } |
1877 | |
1878 | static inline unsigned long page_to_section(const struct page *page) |
1879 | { |
1880 | return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; |
1881 | } |
1882 | #endif |
1883 | |
1884 | /** |
1885 | * folio_pfn - Return the Page Frame Number of a folio. |
1886 | * @folio: The folio. |
1887 | * |
1888 | * A folio may contain multiple pages. The pages have consecutive |
1889 | * Page Frame Numbers. |
1890 | * |
1891 | * Return: The Page Frame Number of the first page in the folio. |
1892 | */ |
1893 | static inline unsigned long folio_pfn(struct folio *folio) |
1894 | { |
1895 | return page_to_pfn(&folio->page); |
1896 | } |
1897 | |
1898 | static inline struct folio *pfn_folio(unsigned long pfn) |
1899 | { |
1900 | return page_folio(pfn_to_page(pfn)); |
1901 | } |
1902 | |
1903 | /** |
1904 | * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA. |
1905 | * @folio: The folio. |
1906 | * |
1907 | * This function checks if a folio has been pinned via a call to |
1908 | * a function in the pin_user_pages() family. |
1909 | * |
1910 | * For small folios, the return value is partially fuzzy: false is not fuzzy, |
1911 | * because it means "definitely not pinned for DMA", but true means "probably |
1912 | * pinned for DMA, but possibly a false positive due to having at least |
1913 | * GUP_PIN_COUNTING_BIAS worth of normal folio references". |
1914 | * |
1915 | * False positives are OK, because: a) it's unlikely for a folio to |
1916 | * get that many refcounts, and b) all the callers of this routine are |
1917 | * expected to be able to deal gracefully with a false positive. |
1918 | * |
1919 | * For large folios, the result will be exactly correct. That's because |
1920 | * we have more tracking data available: the _pincount field is used |
1921 | * instead of the GUP_PIN_COUNTING_BIAS scheme. |
1922 | * |
1923 | * For more information, please see Documentation/core-api/pin_user_pages.rst. |
1924 | * |
1925 | * Return: True, if it is likely that the page has been "dma-pinned". |
1926 | * False, if the page is definitely not dma-pinned. |
1927 | */ |
1928 | static inline bool folio_maybe_dma_pinned(struct folio *folio) |
1929 | { |
1930 | if (folio_test_large(folio)) |
1931 | return atomic_read(v: &folio->_pincount) > 0; |
1932 | |
1933 | /* |
1934 | * folio_ref_count() is signed. If that refcount overflows, then |
1935 | * folio_ref_count() returns a negative value, and callers will avoid |
1936 | * further incrementing the refcount. |
1937 | * |
1938 | * Here, for that overflow case, use the sign bit to count a little |
1939 | * bit higher via unsigned math, and thus still get an accurate result. |
1940 | */ |
1941 | return ((unsigned int)folio_ref_count(folio)) >= |
1942 | GUP_PIN_COUNTING_BIAS; |
1943 | } |
1944 | |
1945 | static inline bool page_maybe_dma_pinned(struct page *page) |
1946 | { |
1947 | return folio_maybe_dma_pinned(page_folio(page)); |
1948 | } |
1949 | |
1950 | /* |
1951 | * This should most likely only be called during fork() to see whether we |
1952 | * should break the cow immediately for an anon page on the src mm. |
1953 | * |
1954 | * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. |
1955 | */ |
1956 | static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma, |
1957 | struct page *page) |
1958 | { |
1959 | VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1)); |
1960 | |
1961 | if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)) |
1962 | return false; |
1963 | |
1964 | return page_maybe_dma_pinned(page); |
1965 | } |
1966 | |
1967 | /** |
1968 | * is_zero_page - Query if a page is a zero page |
1969 | * @page: The page to query |
1970 | * |
1971 | * This returns true if @page is one of the permanent zero pages. |
1972 | */ |
1973 | static inline bool is_zero_page(const struct page *page) |
1974 | { |
1975 | return is_zero_pfn(page_to_pfn(page)); |
1976 | } |
1977 | |
1978 | /** |
1979 | * is_zero_folio - Query if a folio is a zero page |
1980 | * @folio: The folio to query |
1981 | * |
1982 | * This returns true if @folio is one of the permanent zero pages. |
1983 | */ |
1984 | static inline bool is_zero_folio(const struct folio *folio) |
1985 | { |
1986 | return is_zero_page(page: &folio->page); |
1987 | } |
1988 | |
1989 | /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */ |
1990 | #ifdef CONFIG_MIGRATION |
1991 | static inline bool folio_is_longterm_pinnable(struct folio *folio) |
1992 | { |
1993 | #ifdef CONFIG_CMA |
1994 | int mt = folio_migratetype(folio); |
1995 | |
1996 | if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) |
1997 | return false; |
1998 | #endif |
1999 | /* The zero page can be "pinned" but gets special handling. */ |
2000 | if (is_zero_folio(folio)) |
2001 | return true; |
2002 | |
2003 | /* Coherent device memory must always allow eviction. */ |
2004 | if (folio_is_device_coherent(folio)) |
2005 | return false; |
2006 | |
2007 | /* Otherwise, non-movable zone folios can be pinned. */ |
2008 | return !folio_is_zone_movable(folio); |
2009 | |
2010 | } |
2011 | #else |
2012 | static inline bool folio_is_longterm_pinnable(struct folio *folio) |
2013 | { |
2014 | return true; |
2015 | } |
2016 | #endif |
2017 | |
2018 | static inline void set_page_zone(struct page *page, enum zone_type zone) |
2019 | { |
2020 | page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); |
2021 | page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; |
2022 | } |
2023 | |
2024 | static inline void set_page_node(struct page *page, unsigned long node) |
2025 | { |
2026 | page->flags &= ~(NODES_MASK << NODES_PGSHIFT); |
2027 | page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; |
2028 | } |
2029 | |
2030 | static inline void set_page_links(struct page *page, enum zone_type zone, |
2031 | unsigned long node, unsigned long pfn) |
2032 | { |
2033 | set_page_zone(page, zone); |
2034 | set_page_node(page, node); |
2035 | #ifdef SECTION_IN_PAGE_FLAGS |
2036 | set_page_section(page, pfn_to_section_nr(pfn)); |
2037 | #endif |
2038 | } |
2039 | |
2040 | /** |
2041 | * folio_nr_pages - The number of pages in the folio. |
2042 | * @folio: The folio. |
2043 | * |
2044 | * Return: A positive power of two. |
2045 | */ |
2046 | static inline long folio_nr_pages(struct folio *folio) |
2047 | { |
2048 | if (!folio_test_large(folio)) |
2049 | return 1; |
2050 | #ifdef CONFIG_64BIT |
2051 | return folio->_folio_nr_pages; |
2052 | #else |
2053 | return 1L << (folio->_flags_1 & 0xff); |
2054 | #endif |
2055 | } |
2056 | |
2057 | /* |
2058 | * compound_nr() returns the number of pages in this potentially compound |
2059 | * page. compound_nr() can be called on a tail page, and is defined to |
2060 | * return 1 in that case. |
2061 | */ |
2062 | static inline unsigned long compound_nr(struct page *page) |
2063 | { |
2064 | struct folio *folio = (struct folio *)page; |
2065 | |
2066 | if (!test_bit(PG_head, &folio->flags)) |
2067 | return 1; |
2068 | #ifdef CONFIG_64BIT |
2069 | return folio->_folio_nr_pages; |
2070 | #else |
2071 | return 1L << (folio->_flags_1 & 0xff); |
2072 | #endif |
2073 | } |
2074 | |
2075 | /** |
2076 | * thp_nr_pages - The number of regular pages in this huge page. |
2077 | * @page: The head page of a huge page. |
2078 | */ |
2079 | static inline int thp_nr_pages(struct page *page) |
2080 | { |
2081 | return folio_nr_pages(folio: (struct folio *)page); |
2082 | } |
2083 | |
2084 | /** |
2085 | * folio_next - Move to the next physical folio. |
2086 | * @folio: The folio we're currently operating on. |
2087 | * |
2088 | * If you have physically contiguous memory which may span more than |
2089 | * one folio (eg a &struct bio_vec), use this function to move from one |
2090 | * folio to the next. Do not use it if the memory is only virtually |
2091 | * contiguous as the folios are almost certainly not adjacent to each |
2092 | * other. This is the folio equivalent to writing ``page++``. |
2093 | * |
2094 | * Context: We assume that the folios are refcounted and/or locked at a |
2095 | * higher level and do not adjust the reference counts. |
2096 | * Return: The next struct folio. |
2097 | */ |
2098 | static inline struct folio *folio_next(struct folio *folio) |
2099 | { |
2100 | return (struct folio *)folio_page(folio, folio_nr_pages(folio)); |
2101 | } |
2102 | |
2103 | /** |
2104 | * folio_shift - The size of the memory described by this folio. |
2105 | * @folio: The folio. |
2106 | * |
2107 | * A folio represents a number of bytes which is a power-of-two in size. |
2108 | * This function tells you which power-of-two the folio is. See also |
2109 | * folio_size() and folio_order(). |
2110 | * |
2111 | * Context: The caller should have a reference on the folio to prevent |
2112 | * it from being split. It is not necessary for the folio to be locked. |
2113 | * Return: The base-2 logarithm of the size of this folio. |
2114 | */ |
2115 | static inline unsigned int folio_shift(struct folio *folio) |
2116 | { |
2117 | return PAGE_SHIFT + folio_order(folio); |
2118 | } |
2119 | |
2120 | /** |
2121 | * folio_size - The number of bytes in a folio. |
2122 | * @folio: The folio. |
2123 | * |
2124 | * Context: The caller should have a reference on the folio to prevent |
2125 | * it from being split. It is not necessary for the folio to be locked. |
2126 | * Return: The number of bytes in this folio. |
2127 | */ |
2128 | static inline size_t folio_size(struct folio *folio) |
2129 | { |
2130 | return PAGE_SIZE << folio_order(folio); |
2131 | } |
2132 | |
2133 | /** |
2134 | * folio_estimated_sharers - Estimate the number of sharers of a folio. |
2135 | * @folio: The folio. |
2136 | * |
2137 | * folio_estimated_sharers() aims to serve as a function to efficiently |
2138 | * estimate the number of processes sharing a folio. This is done by |
2139 | * looking at the precise mapcount of the first subpage in the folio, and |
2140 | * assuming the other subpages are the same. This may not be true for large |
2141 | * folios. If you want exact mapcounts for exact calculations, look at |
2142 | * page_mapcount() or folio_total_mapcount(). |
2143 | * |
2144 | * Return: The estimated number of processes sharing a folio. |
2145 | */ |
2146 | static inline int folio_estimated_sharers(struct folio *folio) |
2147 | { |
2148 | return page_mapcount(folio_page(folio, 0)); |
2149 | } |
2150 | |
2151 | #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE |
2152 | static inline int arch_make_page_accessible(struct page *page) |
2153 | { |
2154 | return 0; |
2155 | } |
2156 | #endif |
2157 | |
2158 | #ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE |
2159 | static inline int arch_make_folio_accessible(struct folio *folio) |
2160 | { |
2161 | int ret; |
2162 | long i, nr = folio_nr_pages(folio); |
2163 | |
2164 | for (i = 0; i < nr; i++) { |
2165 | ret = arch_make_page_accessible(folio_page(folio, i)); |
2166 | if (ret) |
2167 | break; |
2168 | } |
2169 | |
2170 | return ret; |
2171 | } |
2172 | #endif |
2173 | |
2174 | /* |
2175 | * Some inline functions in vmstat.h depend on page_zone() |
2176 | */ |
2177 | #include <linux/vmstat.h> |
2178 | |
2179 | static __always_inline void *lowmem_page_address(const struct page *page) |
2180 | { |
2181 | return page_to_virt(page); |
2182 | } |
2183 | |
2184 | #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) |
2185 | #define HASHED_PAGE_VIRTUAL |
2186 | #endif |
2187 | |
2188 | #if defined(WANT_PAGE_VIRTUAL) |
2189 | static inline void *page_address(const struct page *page) |
2190 | { |
2191 | return page->virtual; |
2192 | } |
2193 | static inline void set_page_address(struct page *page, void *address) |
2194 | { |
2195 | page->virtual = address; |
2196 | } |
2197 | #define page_address_init() do { } while(0) |
2198 | #endif |
2199 | |
2200 | #if defined(HASHED_PAGE_VIRTUAL) |
2201 | void *page_address(const struct page *page); |
2202 | void set_page_address(struct page *page, void *virtual); |
2203 | void page_address_init(void); |
2204 | #endif |
2205 | |
2206 | #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL) |
2207 | #define page_address(page) lowmem_page_address(page) |
2208 | #define set_page_address(page, address) do { } while(0) |
2209 | #define page_address_init() do { } while(0) |
2210 | #endif |
2211 | |
2212 | static inline void *folio_address(const struct folio *folio) |
2213 | { |
2214 | return page_address(&folio->page); |
2215 | } |
2216 | |
2217 | extern pgoff_t __page_file_index(struct page *page); |
2218 | |
2219 | /* |
2220 | * Return the pagecache index of the passed page. Regular pagecache pages |
2221 | * use ->index whereas swapcache pages use swp_offset(->private) |
2222 | */ |
2223 | static inline pgoff_t page_index(struct page *page) |
2224 | { |
2225 | if (unlikely(PageSwapCache(page))) |
2226 | return __page_file_index(page); |
2227 | return page->index; |
2228 | } |
2229 | |
2230 | /* |
2231 | * Return true only if the page has been allocated with |
2232 | * ALLOC_NO_WATERMARKS and the low watermark was not |
2233 | * met implying that the system is under some pressure. |
2234 | */ |
2235 | static inline bool page_is_pfmemalloc(const struct page *page) |
2236 | { |
2237 | /* |
2238 | * lru.next has bit 1 set if the page is allocated from the |
2239 | * pfmemalloc reserves. Callers may simply overwrite it if |
2240 | * they do not need to preserve that information. |
2241 | */ |
2242 | return (uintptr_t)page->lru.next & BIT(1); |
2243 | } |
2244 | |
2245 | /* |
2246 | * Return true only if the folio has been allocated with |
2247 | * ALLOC_NO_WATERMARKS and the low watermark was not |
2248 | * met implying that the system is under some pressure. |
2249 | */ |
2250 | static inline bool folio_is_pfmemalloc(const struct folio *folio) |
2251 | { |
2252 | /* |
2253 | * lru.next has bit 1 set if the page is allocated from the |
2254 | * pfmemalloc reserves. Callers may simply overwrite it if |
2255 | * they do not need to preserve that information. |
2256 | */ |
2257 | return (uintptr_t)folio->lru.next & BIT(1); |
2258 | } |
2259 | |
2260 | /* |
2261 | * Only to be called by the page allocator on a freshly allocated |
2262 | * page. |
2263 | */ |
2264 | static inline void set_page_pfmemalloc(struct page *page) |
2265 | { |
2266 | page->lru.next = (void *)BIT(1); |
2267 | } |
2268 | |
2269 | static inline void clear_page_pfmemalloc(struct page *page) |
2270 | { |
2271 | page->lru.next = NULL; |
2272 | } |
2273 | |
2274 | /* |
2275 | * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. |
2276 | */ |
2277 | extern void pagefault_out_of_memory(void); |
2278 | |
2279 | #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) |
2280 | #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1)) |
2281 | #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1)) |
2282 | |
2283 | /* |
2284 | * Parameter block passed down to zap_pte_range in exceptional cases. |
2285 | */ |
2286 | struct zap_details { |
2287 | struct folio *single_folio; /* Locked folio to be unmapped */ |
2288 | bool even_cows; /* Zap COWed private pages too? */ |
2289 | zap_flags_t zap_flags; /* Extra flags for zapping */ |
2290 | }; |
2291 | |
2292 | /* |
2293 | * Whether to drop the pte markers, for example, the uffd-wp information for |
2294 | * file-backed memory. This should only be specified when we will completely |
2295 | * drop the page in the mm, either by truncation or unmapping of the vma. By |
2296 | * default, the flag is not set. |
2297 | */ |
2298 | #define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0)) |
2299 | /* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */ |
2300 | #define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1)) |
2301 | |
2302 | #ifdef CONFIG_SCHED_MM_CID |
2303 | void sched_mm_cid_before_execve(struct task_struct *t); |
2304 | void sched_mm_cid_after_execve(struct task_struct *t); |
2305 | void sched_mm_cid_fork(struct task_struct *t); |
2306 | void sched_mm_cid_exit_signals(struct task_struct *t); |
2307 | static inline int task_mm_cid(struct task_struct *t) |
2308 | { |
2309 | return t->mm_cid; |
2310 | } |
2311 | #else |
2312 | static inline void sched_mm_cid_before_execve(struct task_struct *t) { } |
2313 | static inline void sched_mm_cid_after_execve(struct task_struct *t) { } |
2314 | static inline void sched_mm_cid_fork(struct task_struct *t) { } |
2315 | static inline void sched_mm_cid_exit_signals(struct task_struct *t) { } |
2316 | static inline int task_mm_cid(struct task_struct *t) |
2317 | { |
2318 | /* |
2319 | * Use the processor id as a fall-back when the mm cid feature is |
2320 | * disabled. This provides functional per-cpu data structure accesses |
2321 | * in user-space, althrough it won't provide the memory usage benefits. |
2322 | */ |
2323 | return raw_smp_processor_id(); |
2324 | } |
2325 | #endif |
2326 | |
2327 | #ifdef CONFIG_MMU |
2328 | extern bool can_do_mlock(void); |
2329 | #else |
2330 | static inline bool can_do_mlock(void) { return false; } |
2331 | #endif |
2332 | extern int user_shm_lock(size_t, struct ucounts *); |
2333 | extern void user_shm_unlock(size_t, struct ucounts *); |
2334 | |
2335 | struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, |
2336 | pte_t pte); |
2337 | struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, |
2338 | pte_t pte); |
2339 | struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, |
2340 | unsigned long addr, pmd_t pmd); |
2341 | struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, |
2342 | pmd_t pmd); |
2343 | |
2344 | void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, |
2345 | unsigned long size); |
2346 | void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, |
2347 | unsigned long size, struct zap_details *details); |
2348 | static inline void zap_vma_pages(struct vm_area_struct *vma) |
2349 | { |
2350 | zap_page_range_single(vma, address: vma->vm_start, |
2351 | size: vma->vm_end - vma->vm_start, NULL); |
2352 | } |
2353 | void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas, |
2354 | struct vm_area_struct *start_vma, unsigned long start, |
2355 | unsigned long end, unsigned long tree_end, bool mm_wr_locked); |
2356 | |
2357 | struct mmu_notifier_range; |
2358 | |
2359 | void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, |
2360 | unsigned long end, unsigned long floor, unsigned long ceiling); |
2361 | int |
2362 | copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); |
2363 | int follow_pte(struct mm_struct *mm, unsigned long address, |
2364 | pte_t **ptepp, spinlock_t **ptlp); |
2365 | int follow_pfn(struct vm_area_struct *vma, unsigned long address, |
2366 | unsigned long *pfn); |
2367 | int follow_phys(struct vm_area_struct *vma, unsigned long address, |
2368 | unsigned int flags, unsigned long *prot, resource_size_t *phys); |
2369 | int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, |
2370 | void *buf, int len, int write); |
2371 | |
2372 | extern void truncate_pagecache(struct inode *inode, loff_t new); |
2373 | extern void truncate_setsize(struct inode *inode, loff_t newsize); |
2374 | void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to); |
2375 | void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); |
2376 | int generic_error_remove_page(struct address_space *mapping, struct page *page); |
2377 | |
2378 | struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, |
2379 | unsigned long address, struct pt_regs *regs); |
2380 | |
2381 | #ifdef CONFIG_MMU |
2382 | extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, |
2383 | unsigned long address, unsigned int flags, |
2384 | struct pt_regs *regs); |
2385 | extern int fixup_user_fault(struct mm_struct *mm, |
2386 | unsigned long address, unsigned int fault_flags, |
2387 | bool *unlocked); |
2388 | void unmap_mapping_pages(struct address_space *mapping, |
2389 | pgoff_t start, pgoff_t nr, bool even_cows); |
2390 | void unmap_mapping_range(struct address_space *mapping, |
2391 | loff_t const holebegin, loff_t const holelen, int even_cows); |
2392 | #else |
2393 | static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, |
2394 | unsigned long address, unsigned int flags, |
2395 | struct pt_regs *regs) |
2396 | { |
2397 | /* should never happen if there's no MMU */ |
2398 | BUG(); |
2399 | return VM_FAULT_SIGBUS; |
2400 | } |
2401 | static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, |
2402 | unsigned int fault_flags, bool *unlocked) |
2403 | { |
2404 | /* should never happen if there's no MMU */ |
2405 | BUG(); |
2406 | return -EFAULT; |
2407 | } |
2408 | static inline void unmap_mapping_pages(struct address_space *mapping, |
2409 | pgoff_t start, pgoff_t nr, bool even_cows) { } |
2410 | static inline void unmap_mapping_range(struct address_space *mapping, |
2411 | loff_t const holebegin, loff_t const holelen, int even_cows) { } |
2412 | #endif |
2413 | |
2414 | static inline void unmap_shared_mapping_range(struct address_space *mapping, |
2415 | loff_t const holebegin, loff_t const holelen) |
2416 | { |
2417 | unmap_mapping_range(mapping, holebegin, holelen, even_cows: 0); |
2418 | } |
2419 | |
2420 | static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, |
2421 | unsigned long addr); |
2422 | |
2423 | extern int access_process_vm(struct task_struct *tsk, unsigned long addr, |
2424 | void *buf, int len, unsigned int gup_flags); |
2425 | extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, |
2426 | void *buf, int len, unsigned int gup_flags); |
2427 | |
2428 | long get_user_pages_remote(struct mm_struct *mm, |
2429 | unsigned long start, unsigned long nr_pages, |
2430 | unsigned int gup_flags, struct page **pages, |
2431 | int *locked); |
2432 | long pin_user_pages_remote(struct mm_struct *mm, |
2433 | unsigned long start, unsigned long nr_pages, |
2434 | unsigned int gup_flags, struct page **pages, |
2435 | int *locked); |
2436 | |
2437 | /* |
2438 | * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT. |
2439 | */ |
2440 | static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, |
2441 | unsigned long addr, |
2442 | int gup_flags, |
2443 | struct vm_area_struct **vmap) |
2444 | { |
2445 | struct page *page; |
2446 | struct vm_area_struct *vma; |
2447 | int got; |
2448 | |
2449 | if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT))) |
2450 | return ERR_PTR(error: -EINVAL); |
2451 | |
2452 | got = get_user_pages_remote(mm, start: addr, nr_pages: 1, gup_flags, pages: &page, NULL); |
2453 | |
2454 | if (got < 0) |
2455 | return ERR_PTR(error: got); |
2456 | |
2457 | vma = vma_lookup(mm, addr); |
2458 | if (WARN_ON_ONCE(!vma)) { |
2459 | put_page(page); |
2460 | return ERR_PTR(error: -EINVAL); |
2461 | } |
2462 | |
2463 | *vmap = vma; |
2464 | return page; |
2465 | } |
2466 | |
2467 | long get_user_pages(unsigned long start, unsigned long nr_pages, |
2468 | unsigned int gup_flags, struct page **pages); |
2469 | long pin_user_pages(unsigned long start, unsigned long nr_pages, |
2470 | unsigned int gup_flags, struct page **pages); |
2471 | long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, |
2472 | struct page **pages, unsigned int gup_flags); |
2473 | long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, |
2474 | struct page **pages, unsigned int gup_flags); |
2475 | |
2476 | int get_user_pages_fast(unsigned long start, int nr_pages, |
2477 | unsigned int gup_flags, struct page **pages); |
2478 | int pin_user_pages_fast(unsigned long start, int nr_pages, |
2479 | unsigned int gup_flags, struct page **pages); |
2480 | void folio_add_pin(struct folio *folio); |
2481 | |
2482 | int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); |
2483 | int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, |
2484 | struct task_struct *task, bool bypass_rlim); |
2485 | |
2486 | struct kvec; |
2487 | struct page *get_dump_page(unsigned long addr); |
2488 | |
2489 | bool folio_mark_dirty(struct folio *folio); |
2490 | bool set_page_dirty(struct page *page); |
2491 | int set_page_dirty_lock(struct page *page); |
2492 | |
2493 | int get_cmdline(struct task_struct *task, char *buffer, int buflen); |
2494 | |
2495 | extern unsigned long move_page_tables(struct vm_area_struct *vma, |
2496 | unsigned long old_addr, struct vm_area_struct *new_vma, |
2497 | unsigned long new_addr, unsigned long len, |
2498 | bool need_rmap_locks, bool for_stack); |
2499 | |
2500 | /* |
2501 | * Flags used by change_protection(). For now we make it a bitmap so |
2502 | * that we can pass in multiple flags just like parameters. However |
2503 | * for now all the callers are only use one of the flags at the same |
2504 | * time. |
2505 | */ |
2506 | /* |
2507 | * Whether we should manually check if we can map individual PTEs writable, |
2508 | * because something (e.g., COW, uffd-wp) blocks that from happening for all |
2509 | * PTEs automatically in a writable mapping. |
2510 | */ |
2511 | #define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0) |
2512 | /* Whether this protection change is for NUMA hints */ |
2513 | #define MM_CP_PROT_NUMA (1UL << 1) |
2514 | /* Whether this change is for write protecting */ |
2515 | #define MM_CP_UFFD_WP (1UL << 2) /* do wp */ |
2516 | #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ |
2517 | #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ |
2518 | MM_CP_UFFD_WP_RESOLVE) |
2519 | |
2520 | bool vma_needs_dirty_tracking(struct vm_area_struct *vma); |
2521 | int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); |
2522 | static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) |
2523 | { |
2524 | /* |
2525 | * We want to check manually if we can change individual PTEs writable |
2526 | * if we can't do that automatically for all PTEs in a mapping. For |
2527 | * private mappings, that's always the case when we have write |
2528 | * permissions as we properly have to handle COW. |
2529 | */ |
2530 | if (vma->vm_flags & VM_SHARED) |
2531 | return vma_wants_writenotify(vma, vm_page_prot: vma->vm_page_prot); |
2532 | return !!(vma->vm_flags & VM_WRITE); |
2533 | |
2534 | } |
2535 | bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, |
2536 | pte_t pte); |
2537 | extern long change_protection(struct mmu_gather *tlb, |
2538 | struct vm_area_struct *vma, unsigned long start, |
2539 | unsigned long end, unsigned long cp_flags); |
2540 | extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, |
2541 | struct vm_area_struct *vma, struct vm_area_struct **pprev, |
2542 | unsigned long start, unsigned long end, unsigned long newflags); |
2543 | |
2544 | /* |
2545 | * doesn't attempt to fault and will return short. |
2546 | */ |
2547 | int get_user_pages_fast_only(unsigned long start, int nr_pages, |
2548 | unsigned int gup_flags, struct page **pages); |
2549 | |
2550 | static inline bool get_user_page_fast_only(unsigned long addr, |
2551 | unsigned int gup_flags, struct page **pagep) |
2552 | { |
2553 | return get_user_pages_fast_only(start: addr, nr_pages: 1, gup_flags, pages: pagep) == 1; |
2554 | } |
2555 | /* |
2556 | * per-process(per-mm_struct) statistics. |
2557 | */ |
2558 | static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) |
2559 | { |
2560 | return percpu_counter_read_positive(fbc: &mm->rss_stat[member]); |
2561 | } |
2562 | |
2563 | void (struct mm_struct *mm, int member); |
2564 | |
2565 | static inline void add_mm_counter(struct mm_struct *mm, int member, long value) |
2566 | { |
2567 | percpu_counter_add(fbc: &mm->rss_stat[member], amount: value); |
2568 | |
2569 | mm_trace_rss_stat(mm, member); |
2570 | } |
2571 | |
2572 | static inline void inc_mm_counter(struct mm_struct *mm, int member) |
2573 | { |
2574 | percpu_counter_inc(fbc: &mm->rss_stat[member]); |
2575 | |
2576 | mm_trace_rss_stat(mm, member); |
2577 | } |
2578 | |
2579 | static inline void dec_mm_counter(struct mm_struct *mm, int member) |
2580 | { |
2581 | percpu_counter_dec(fbc: &mm->rss_stat[member]); |
2582 | |
2583 | mm_trace_rss_stat(mm, member); |
2584 | } |
2585 | |
2586 | /* Optimized variant when page is already known not to be PageAnon */ |
2587 | static inline int mm_counter_file(struct page *page) |
2588 | { |
2589 | if (PageSwapBacked(page)) |
2590 | return MM_SHMEMPAGES; |
2591 | return MM_FILEPAGES; |
2592 | } |
2593 | |
2594 | static inline int mm_counter(struct page *page) |
2595 | { |
2596 | if (PageAnon(page)) |
2597 | return MM_ANONPAGES; |
2598 | return mm_counter_file(page); |
2599 | } |
2600 | |
2601 | static inline unsigned long (struct mm_struct *mm) |
2602 | { |
2603 | return get_mm_counter(mm, member: MM_FILEPAGES) + |
2604 | get_mm_counter(mm, member: MM_ANONPAGES) + |
2605 | get_mm_counter(mm, member: MM_SHMEMPAGES); |
2606 | } |
2607 | |
2608 | static inline unsigned long (struct mm_struct *mm) |
2609 | { |
2610 | return max(mm->hiwater_rss, get_mm_rss(mm)); |
2611 | } |
2612 | |
2613 | static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm) |
2614 | { |
2615 | return max(mm->hiwater_vm, mm->total_vm); |
2616 | } |
2617 | |
2618 | static inline void (struct mm_struct *mm) |
2619 | { |
2620 | unsigned long = get_mm_rss(mm); |
2621 | |
2622 | if ((mm)->hiwater_rss < _rss) |
2623 | (mm)->hiwater_rss = _rss; |
2624 | } |
2625 | |
2626 | static inline void update_hiwater_vm(struct mm_struct *mm) |
2627 | { |
2628 | if (mm->hiwater_vm < mm->total_vm) |
2629 | mm->hiwater_vm = mm->total_vm; |
2630 | } |
2631 | |
2632 | static inline void (struct mm_struct *mm) |
2633 | { |
2634 | mm->hiwater_rss = get_mm_rss(mm); |
2635 | } |
2636 | |
2637 | static inline void (unsigned long *, |
2638 | struct mm_struct *mm) |
2639 | { |
2640 | unsigned long = get_mm_hiwater_rss(mm); |
2641 | |
2642 | if (*maxrss < hiwater_rss) |
2643 | *maxrss = hiwater_rss; |
2644 | } |
2645 | |
2646 | #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL |
2647 | static inline int pte_special(pte_t pte) |
2648 | { |
2649 | return 0; |
2650 | } |
2651 | |
2652 | static inline pte_t pte_mkspecial(pte_t pte) |
2653 | { |
2654 | return pte; |
2655 | } |
2656 | #endif |
2657 | |
2658 | #ifndef CONFIG_ARCH_HAS_PTE_DEVMAP |
2659 | static inline int pte_devmap(pte_t pte) |
2660 | { |
2661 | return 0; |
2662 | } |
2663 | #endif |
2664 | |
2665 | extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, |
2666 | spinlock_t **ptl); |
2667 | static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, |
2668 | spinlock_t **ptl) |
2669 | { |
2670 | pte_t *ptep; |
2671 | __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl)); |
2672 | return ptep; |
2673 | } |
2674 | |
2675 | #ifdef __PAGETABLE_P4D_FOLDED |
2676 | static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, |
2677 | unsigned long address) |
2678 | { |
2679 | return 0; |
2680 | } |
2681 | #else |
2682 | int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address); |
2683 | #endif |
2684 | |
2685 | #if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU) |
2686 | static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, |
2687 | unsigned long address) |
2688 | { |
2689 | return 0; |
2690 | } |
2691 | static inline void mm_inc_nr_puds(struct mm_struct *mm) {} |
2692 | static inline void mm_dec_nr_puds(struct mm_struct *mm) {} |
2693 | |
2694 | #else |
2695 | int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address); |
2696 | |
2697 | static inline void mm_inc_nr_puds(struct mm_struct *mm) |
2698 | { |
2699 | if (mm_pud_folded(mm)) |
2700 | return; |
2701 | atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), v: &mm->pgtables_bytes); |
2702 | } |
2703 | |
2704 | static inline void mm_dec_nr_puds(struct mm_struct *mm) |
2705 | { |
2706 | if (mm_pud_folded(mm)) |
2707 | return; |
2708 | atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), v: &mm->pgtables_bytes); |
2709 | } |
2710 | #endif |
2711 | |
2712 | #if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU) |
2713 | static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, |
2714 | unsigned long address) |
2715 | { |
2716 | return 0; |
2717 | } |
2718 | |
2719 | static inline void mm_inc_nr_pmds(struct mm_struct *mm) {} |
2720 | static inline void mm_dec_nr_pmds(struct mm_struct *mm) {} |
2721 | |
2722 | #else |
2723 | int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); |
2724 | |
2725 | static inline void mm_inc_nr_pmds(struct mm_struct *mm) |
2726 | { |
2727 | if (mm_pmd_folded(mm)) |
2728 | return; |
2729 | atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), v: &mm->pgtables_bytes); |
2730 | } |
2731 | |
2732 | static inline void mm_dec_nr_pmds(struct mm_struct *mm) |
2733 | { |
2734 | if (mm_pmd_folded(mm)) |
2735 | return; |
2736 | atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), v: &mm->pgtables_bytes); |
2737 | } |
2738 | #endif |
2739 | |
2740 | #ifdef CONFIG_MMU |
2741 | static inline void mm_pgtables_bytes_init(struct mm_struct *mm) |
2742 | { |
2743 | atomic_long_set(v: &mm->pgtables_bytes, i: 0); |
2744 | } |
2745 | |
2746 | static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) |
2747 | { |
2748 | return atomic_long_read(v: &mm->pgtables_bytes); |
2749 | } |
2750 | |
2751 | static inline void mm_inc_nr_ptes(struct mm_struct *mm) |
2752 | { |
2753 | atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), v: &mm->pgtables_bytes); |
2754 | } |
2755 | |
2756 | static inline void mm_dec_nr_ptes(struct mm_struct *mm) |
2757 | { |
2758 | atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), v: &mm->pgtables_bytes); |
2759 | } |
2760 | #else |
2761 | |
2762 | static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {} |
2763 | static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) |
2764 | { |
2765 | return 0; |
2766 | } |
2767 | |
2768 | static inline void mm_inc_nr_ptes(struct mm_struct *mm) {} |
2769 | static inline void mm_dec_nr_ptes(struct mm_struct *mm) {} |
2770 | #endif |
2771 | |
2772 | int __pte_alloc(struct mm_struct *mm, pmd_t *pmd); |
2773 | int __pte_alloc_kernel(pmd_t *pmd); |
2774 | |
2775 | #if defined(CONFIG_MMU) |
2776 | |
2777 | static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, |
2778 | unsigned long address) |
2779 | { |
2780 | return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? |
2781 | NULL : p4d_offset(pgd, address); |
2782 | } |
2783 | |
2784 | static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, |
2785 | unsigned long address) |
2786 | { |
2787 | return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? |
2788 | NULL : pud_offset(p4d, address); |
2789 | } |
2790 | |
2791 | static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) |
2792 | { |
2793 | return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? |
2794 | NULL: pmd_offset(pud, address); |
2795 | } |
2796 | #endif /* CONFIG_MMU */ |
2797 | |
2798 | static inline struct ptdesc *virt_to_ptdesc(const void *x) |
2799 | { |
2800 | return page_ptdesc(virt_to_page(x)); |
2801 | } |
2802 | |
2803 | static inline void *ptdesc_to_virt(const struct ptdesc *pt) |
2804 | { |
2805 | return page_to_virt(ptdesc_page(pt)); |
2806 | } |
2807 | |
2808 | static inline void *ptdesc_address(const struct ptdesc *pt) |
2809 | { |
2810 | return folio_address(ptdesc_folio(pt)); |
2811 | } |
2812 | |
2813 | static inline bool pagetable_is_reserved(struct ptdesc *pt) |
2814 | { |
2815 | return folio_test_reserved(ptdesc_folio(pt)); |
2816 | } |
2817 | |
2818 | /** |
2819 | * pagetable_alloc - Allocate pagetables |
2820 | * @gfp: GFP flags |
2821 | * @order: desired pagetable order |
2822 | * |
2823 | * pagetable_alloc allocates memory for page tables as well as a page table |
2824 | * descriptor to describe that memory. |
2825 | * |
2826 | * Return: The ptdesc describing the allocated page tables. |
2827 | */ |
2828 | static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) |
2829 | { |
2830 | struct page *page = alloc_pages(gfp: gfp | __GFP_COMP, order); |
2831 | |
2832 | return page_ptdesc(page); |
2833 | } |
2834 | |
2835 | /** |
2836 | * pagetable_free - Free pagetables |
2837 | * @pt: The page table descriptor |
2838 | * |
2839 | * pagetable_free frees the memory of all page tables described by a page |
2840 | * table descriptor and the memory for the descriptor itself. |
2841 | */ |
2842 | static inline void pagetable_free(struct ptdesc *pt) |
2843 | { |
2844 | struct page *page = ptdesc_page(pt); |
2845 | |
2846 | __free_pages(page, order: compound_order(page)); |
2847 | } |
2848 | |
2849 | #if USE_SPLIT_PTE_PTLOCKS |
2850 | #if ALLOC_SPLIT_PTLOCKS |
2851 | void __init ptlock_cache_init(void); |
2852 | bool ptlock_alloc(struct ptdesc *ptdesc); |
2853 | void ptlock_free(struct ptdesc *ptdesc); |
2854 | |
2855 | static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) |
2856 | { |
2857 | return ptdesc->ptl; |
2858 | } |
2859 | #else /* ALLOC_SPLIT_PTLOCKS */ |
2860 | static inline void ptlock_cache_init(void) |
2861 | { |
2862 | } |
2863 | |
2864 | static inline bool ptlock_alloc(struct ptdesc *ptdesc) |
2865 | { |
2866 | return true; |
2867 | } |
2868 | |
2869 | static inline void ptlock_free(struct ptdesc *ptdesc) |
2870 | { |
2871 | } |
2872 | |
2873 | static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) |
2874 | { |
2875 | return &ptdesc->ptl; |
2876 | } |
2877 | #endif /* ALLOC_SPLIT_PTLOCKS */ |
2878 | |
2879 | static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) |
2880 | { |
2881 | return ptlock_ptr(page_ptdesc(pmd_page(*pmd))); |
2882 | } |
2883 | |
2884 | static inline bool ptlock_init(struct ptdesc *ptdesc) |
2885 | { |
2886 | /* |
2887 | * prep_new_page() initialize page->private (and therefore page->ptl) |
2888 | * with 0. Make sure nobody took it in use in between. |
2889 | * |
2890 | * It can happen if arch try to use slab for page table allocation: |
2891 | * slab code uses page->slab_cache, which share storage with page->ptl. |
2892 | */ |
2893 | VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); |
2894 | if (!ptlock_alloc(ptdesc)) |
2895 | return false; |
2896 | spin_lock_init(ptlock_ptr(ptdesc)); |
2897 | return true; |
2898 | } |
2899 | |
2900 | #else /* !USE_SPLIT_PTE_PTLOCKS */ |
2901 | /* |
2902 | * We use mm->page_table_lock to guard all pagetable pages of the mm. |
2903 | */ |
2904 | static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) |
2905 | { |
2906 | return &mm->page_table_lock; |
2907 | } |
2908 | static inline void ptlock_cache_init(void) {} |
2909 | static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; } |
2910 | static inline void ptlock_free(struct ptdesc *ptdesc) {} |
2911 | #endif /* USE_SPLIT_PTE_PTLOCKS */ |
2912 | |
2913 | static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc) |
2914 | { |
2915 | struct folio *folio = ptdesc_folio(ptdesc); |
2916 | |
2917 | if (!ptlock_init(ptdesc)) |
2918 | return false; |
2919 | __folio_set_pgtable(folio); |
2920 | lruvec_stat_add_folio(folio, idx: NR_PAGETABLE); |
2921 | return true; |
2922 | } |
2923 | |
2924 | static inline void pagetable_pte_dtor(struct ptdesc *ptdesc) |
2925 | { |
2926 | struct folio *folio = ptdesc_folio(ptdesc); |
2927 | |
2928 | ptlock_free(ptdesc); |
2929 | __folio_clear_pgtable(folio); |
2930 | lruvec_stat_sub_folio(folio, idx: NR_PAGETABLE); |
2931 | } |
2932 | |
2933 | pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); |
2934 | static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) |
2935 | { |
2936 | return __pte_offset_map(pmd, addr, NULL); |
2937 | } |
2938 | |
2939 | pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, |
2940 | unsigned long addr, spinlock_t **ptlp); |
2941 | static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, |
2942 | unsigned long addr, spinlock_t **ptlp) |
2943 | { |
2944 | pte_t *pte; |
2945 | |
2946 | __cond_lock(*ptlp, pte = __pte_offset_map_lock(mm, pmd, addr, ptlp)); |
2947 | return pte; |
2948 | } |
2949 | |
2950 | pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd, |
2951 | unsigned long addr, spinlock_t **ptlp); |
2952 | |
2953 | #define pte_unmap_unlock(pte, ptl) do { \ |
2954 | spin_unlock(ptl); \ |
2955 | pte_unmap(pte); \ |
2956 | } while (0) |
2957 | |
2958 | #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) |
2959 | |
2960 | #define pte_alloc_map(mm, pmd, address) \ |
2961 | (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) |
2962 | |
2963 | #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ |
2964 | (pte_alloc(mm, pmd) ? \ |
2965 | NULL : pte_offset_map_lock(mm, pmd, address, ptlp)) |
2966 | |
2967 | #define pte_alloc_kernel(pmd, address) \ |
2968 | ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ |
2969 | NULL: pte_offset_kernel(pmd, address)) |
2970 | |
2971 | #if USE_SPLIT_PMD_PTLOCKS |
2972 | |
2973 | static inline struct page *pmd_pgtable_page(pmd_t *pmd) |
2974 | { |
2975 | unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); |
2976 | return virt_to_page((void *)((unsigned long) pmd & mask)); |
2977 | } |
2978 | |
2979 | static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) |
2980 | { |
2981 | return page_ptdesc(pmd_pgtable_page(pmd)); |
2982 | } |
2983 | |
2984 | static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) |
2985 | { |
2986 | return ptlock_ptr(ptdesc: pmd_ptdesc(pmd)); |
2987 | } |
2988 | |
2989 | static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) |
2990 | { |
2991 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
2992 | ptdesc->pmd_huge_pte = NULL; |
2993 | #endif |
2994 | return ptlock_init(ptdesc); |
2995 | } |
2996 | |
2997 | static inline void pmd_ptlock_free(struct ptdesc *ptdesc) |
2998 | { |
2999 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
3000 | VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc)); |
3001 | #endif |
3002 | ptlock_free(ptdesc); |
3003 | } |
3004 | |
3005 | #define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte) |
3006 | |
3007 | #else |
3008 | |
3009 | static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) |
3010 | { |
3011 | return &mm->page_table_lock; |
3012 | } |
3013 | |
3014 | static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; } |
3015 | static inline void pmd_ptlock_free(struct ptdesc *ptdesc) {} |
3016 | |
3017 | #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) |
3018 | |
3019 | #endif |
3020 | |
3021 | static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) |
3022 | { |
3023 | spinlock_t *ptl = pmd_lockptr(mm, pmd); |
3024 | spin_lock(lock: ptl); |
3025 | return ptl; |
3026 | } |
3027 | |
3028 | static inline bool pagetable_pmd_ctor(struct ptdesc *ptdesc) |
3029 | { |
3030 | struct folio *folio = ptdesc_folio(ptdesc); |
3031 | |
3032 | if (!pmd_ptlock_init(ptdesc)) |
3033 | return false; |
3034 | __folio_set_pgtable(folio); |
3035 | lruvec_stat_add_folio(folio, idx: NR_PAGETABLE); |
3036 | return true; |
3037 | } |
3038 | |
3039 | static inline void pagetable_pmd_dtor(struct ptdesc *ptdesc) |
3040 | { |
3041 | struct folio *folio = ptdesc_folio(ptdesc); |
3042 | |
3043 | pmd_ptlock_free(ptdesc); |
3044 | __folio_clear_pgtable(folio); |
3045 | lruvec_stat_sub_folio(folio, idx: NR_PAGETABLE); |
3046 | } |
3047 | |
3048 | /* |
3049 | * No scalability reason to split PUD locks yet, but follow the same pattern |
3050 | * as the PMD locks to make it easier if we decide to. The VM should not be |
3051 | * considered ready to switch to split PUD locks yet; there may be places |
3052 | * which need to be converted from page_table_lock. |
3053 | */ |
3054 | static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud) |
3055 | { |
3056 | return &mm->page_table_lock; |
3057 | } |
3058 | |
3059 | static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud) |
3060 | { |
3061 | spinlock_t *ptl = pud_lockptr(mm, pud); |
3062 | |
3063 | spin_lock(lock: ptl); |
3064 | return ptl; |
3065 | } |
3066 | |
3067 | static inline void pagetable_pud_ctor(struct ptdesc *ptdesc) |
3068 | { |
3069 | struct folio *folio = ptdesc_folio(ptdesc); |
3070 | |
3071 | __folio_set_pgtable(folio); |
3072 | lruvec_stat_add_folio(folio, idx: NR_PAGETABLE); |
3073 | } |
3074 | |
3075 | static inline void pagetable_pud_dtor(struct ptdesc *ptdesc) |
3076 | { |
3077 | struct folio *folio = ptdesc_folio(ptdesc); |
3078 | |
3079 | __folio_clear_pgtable(folio); |
3080 | lruvec_stat_sub_folio(folio, idx: NR_PAGETABLE); |
3081 | } |
3082 | |
3083 | extern void __init pagecache_init(void); |
3084 | extern void free_initmem(void); |
3085 | |
3086 | /* |
3087 | * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) |
3088 | * into the buddy system. The freed pages will be poisoned with pattern |
3089 | * "poison" if it's within range [0, UCHAR_MAX]. |
3090 | * Return pages freed into the buddy system. |
3091 | */ |
3092 | extern unsigned long free_reserved_area(void *start, void *end, |
3093 | int poison, const char *s); |
3094 | |
3095 | extern void adjust_managed_page_count(struct page *page, long count); |
3096 | |
3097 | extern void reserve_bootmem_region(phys_addr_t start, |
3098 | phys_addr_t end, int nid); |
3099 | |
3100 | /* Free the reserved page into the buddy system, so it gets managed. */ |
3101 | static inline void free_reserved_page(struct page *page) |
3102 | { |
3103 | ClearPageReserved(page); |
3104 | init_page_count(page); |
3105 | __free_page(page); |
3106 | adjust_managed_page_count(page, count: 1); |
3107 | } |
3108 | #define free_highmem_page(page) free_reserved_page(page) |
3109 | |
3110 | static inline void mark_page_reserved(struct page *page) |
3111 | { |
3112 | SetPageReserved(page); |
3113 | adjust_managed_page_count(page, count: -1); |
3114 | } |
3115 | |
3116 | static inline void free_reserved_ptdesc(struct ptdesc *pt) |
3117 | { |
3118 | free_reserved_page(ptdesc_page(pt)); |
3119 | } |
3120 | |
3121 | /* |
3122 | * Default method to free all the __init memory into the buddy system. |
3123 | * The freed pages will be poisoned with pattern "poison" if it's within |
3124 | * range [0, UCHAR_MAX]. |
3125 | * Return pages freed into the buddy system. |
3126 | */ |
3127 | static inline unsigned long free_initmem_default(int poison) |
3128 | { |
3129 | extern char __init_begin[], __init_end[]; |
3130 | |
3131 | return free_reserved_area(start: &__init_begin, end: &__init_end, |
3132 | poison, s: "unused kernel image (initmem)" ); |
3133 | } |
3134 | |
3135 | static inline unsigned long get_num_physpages(void) |
3136 | { |
3137 | int nid; |
3138 | unsigned long phys_pages = 0; |
3139 | |
3140 | for_each_online_node(nid) |
3141 | phys_pages += node_present_pages(nid); |
3142 | |
3143 | return phys_pages; |
3144 | } |
3145 | |
3146 | /* |
3147 | * Using memblock node mappings, an architecture may initialise its |
3148 | * zones, allocate the backing mem_map and account for memory holes in an |
3149 | * architecture independent manner. |
3150 | * |
3151 | * An architecture is expected to register range of page frames backed by |
3152 | * physical memory with memblock_add[_node]() before calling |
3153 | * free_area_init() passing in the PFN each zone ends at. At a basic |
3154 | * usage, an architecture is expected to do something like |
3155 | * |
3156 | * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, |
3157 | * max_highmem_pfn}; |
3158 | * for_each_valid_physical_page_range() |
3159 | * memblock_add_node(base, size, nid, MEMBLOCK_NONE) |
3160 | * free_area_init(max_zone_pfns); |
3161 | */ |
3162 | void free_area_init(unsigned long *max_zone_pfn); |
3163 | unsigned long node_map_pfn_alignment(void); |
3164 | unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn, |
3165 | unsigned long end_pfn); |
3166 | extern unsigned long absent_pages_in_range(unsigned long start_pfn, |
3167 | unsigned long end_pfn); |
3168 | extern void get_pfn_range_for_nid(unsigned int nid, |
3169 | unsigned long *start_pfn, unsigned long *end_pfn); |
3170 | |
3171 | #ifndef CONFIG_NUMA |
3172 | static inline int early_pfn_to_nid(unsigned long pfn) |
3173 | { |
3174 | return 0; |
3175 | } |
3176 | #else |
3177 | /* please see mm/page_alloc.c */ |
3178 | extern int __meminit early_pfn_to_nid(unsigned long pfn); |
3179 | #endif |
3180 | |
3181 | extern void set_dma_reserve(unsigned long new_dma_reserve); |
3182 | extern void mem_init(void); |
3183 | extern void __init mmap_init(void); |
3184 | |
3185 | extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); |
3186 | static inline void show_mem(void) |
3187 | { |
3188 | __show_mem(flags: 0, NULL, max_zone_idx: MAX_NR_ZONES - 1); |
3189 | } |
3190 | extern long si_mem_available(void); |
3191 | extern void si_meminfo(struct sysinfo * val); |
3192 | extern void si_meminfo_node(struct sysinfo *val, int nid); |
3193 | #ifdef __HAVE_ARCH_RESERVED_KERNEL_PAGES |
3194 | extern unsigned long arch_reserved_kernel_pages(void); |
3195 | #endif |
3196 | |
3197 | extern __printf(3, 4) |
3198 | void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); |
3199 | |
3200 | extern void setup_per_cpu_pageset(void); |
3201 | |
3202 | /* nommu.c */ |
3203 | extern atomic_long_t mmap_pages_allocated; |
3204 | extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); |
3205 | |
3206 | /* interval_tree.c */ |
3207 | void vma_interval_tree_insert(struct vm_area_struct *node, |
3208 | struct rb_root_cached *root); |
3209 | void vma_interval_tree_insert_after(struct vm_area_struct *node, |
3210 | struct vm_area_struct *prev, |
3211 | struct rb_root_cached *root); |
3212 | void vma_interval_tree_remove(struct vm_area_struct *node, |
3213 | struct rb_root_cached *root); |
3214 | struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, |
3215 | unsigned long start, unsigned long last); |
3216 | struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, |
3217 | unsigned long start, unsigned long last); |
3218 | |
3219 | #define vma_interval_tree_foreach(vma, root, start, last) \ |
3220 | for (vma = vma_interval_tree_iter_first(root, start, last); \ |
3221 | vma; vma = vma_interval_tree_iter_next(vma, start, last)) |
3222 | |
3223 | void anon_vma_interval_tree_insert(struct anon_vma_chain *node, |
3224 | struct rb_root_cached *root); |
3225 | void anon_vma_interval_tree_remove(struct anon_vma_chain *node, |
3226 | struct rb_root_cached *root); |
3227 | struct anon_vma_chain * |
3228 | anon_vma_interval_tree_iter_first(struct rb_root_cached *root, |
3229 | unsigned long start, unsigned long last); |
3230 | struct anon_vma_chain *anon_vma_interval_tree_iter_next( |
3231 | struct anon_vma_chain *node, unsigned long start, unsigned long last); |
3232 | #ifdef CONFIG_DEBUG_VM_RB |
3233 | void anon_vma_interval_tree_verify(struct anon_vma_chain *node); |
3234 | #endif |
3235 | |
3236 | #define anon_vma_interval_tree_foreach(avc, root, start, last) \ |
3237 | for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ |
3238 | avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) |
3239 | |
3240 | /* mmap.c */ |
3241 | extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin); |
3242 | extern int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma, |
3243 | unsigned long start, unsigned long end, pgoff_t pgoff, |
3244 | struct vm_area_struct *next); |
3245 | extern int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, |
3246 | unsigned long start, unsigned long end, pgoff_t pgoff); |
3247 | extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); |
3248 | extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); |
3249 | extern void unlink_file_vma(struct vm_area_struct *); |
3250 | extern struct vm_area_struct *copy_vma(struct vm_area_struct **, |
3251 | unsigned long addr, unsigned long len, pgoff_t pgoff, |
3252 | bool *need_rmap_locks); |
3253 | extern void exit_mmap(struct mm_struct *); |
3254 | struct vm_area_struct *vma_modify(struct vma_iterator *vmi, |
3255 | struct vm_area_struct *prev, |
3256 | struct vm_area_struct *vma, |
3257 | unsigned long start, unsigned long end, |
3258 | unsigned long vm_flags, |
3259 | struct mempolicy *policy, |
3260 | struct vm_userfaultfd_ctx uffd_ctx, |
3261 | struct anon_vma_name *anon_name); |
3262 | |
3263 | /* We are about to modify the VMA's flags. */ |
3264 | static inline struct vm_area_struct |
3265 | *vma_modify_flags(struct vma_iterator *vmi, |
3266 | struct vm_area_struct *prev, |
3267 | struct vm_area_struct *vma, |
3268 | unsigned long start, unsigned long end, |
3269 | unsigned long new_flags) |
3270 | { |
3271 | return vma_modify(vmi, prev, vma, start, end, vm_flags: new_flags, |
3272 | vma_policy(vma), uffd_ctx: vma->vm_userfaultfd_ctx, |
3273 | anon_name: anon_vma_name(vma)); |
3274 | } |
3275 | |
3276 | /* We are about to modify the VMA's flags and/or anon_name. */ |
3277 | static inline struct vm_area_struct |
3278 | *vma_modify_flags_name(struct vma_iterator *vmi, |
3279 | struct vm_area_struct *prev, |
3280 | struct vm_area_struct *vma, |
3281 | unsigned long start, |
3282 | unsigned long end, |
3283 | unsigned long new_flags, |
3284 | struct anon_vma_name *new_name) |
3285 | { |
3286 | return vma_modify(vmi, prev, vma, start, end, vm_flags: new_flags, |
3287 | vma_policy(vma), uffd_ctx: vma->vm_userfaultfd_ctx, anon_name: new_name); |
3288 | } |
3289 | |
3290 | /* We are about to modify the VMA's memory policy. */ |
3291 | static inline struct vm_area_struct |
3292 | *vma_modify_policy(struct vma_iterator *vmi, |
3293 | struct vm_area_struct *prev, |
3294 | struct vm_area_struct *vma, |
3295 | unsigned long start, unsigned long end, |
3296 | struct mempolicy *new_pol) |
3297 | { |
3298 | return vma_modify(vmi, prev, vma, start, end, vm_flags: vma->vm_flags, |
3299 | policy: new_pol, uffd_ctx: vma->vm_userfaultfd_ctx, anon_name: anon_vma_name(vma)); |
3300 | } |
3301 | |
3302 | /* We are about to modify the VMA's flags and/or uffd context. */ |
3303 | static inline struct vm_area_struct |
3304 | *vma_modify_flags_uffd(struct vma_iterator *vmi, |
3305 | struct vm_area_struct *prev, |
3306 | struct vm_area_struct *vma, |
3307 | unsigned long start, unsigned long end, |
3308 | unsigned long new_flags, |
3309 | struct vm_userfaultfd_ctx new_ctx) |
3310 | { |
3311 | return vma_modify(vmi, prev, vma, start, end, vm_flags: new_flags, |
3312 | vma_policy(vma), uffd_ctx: new_ctx, anon_name: anon_vma_name(vma)); |
3313 | } |
3314 | |
3315 | static inline int check_data_rlimit(unsigned long rlim, |
3316 | unsigned long new, |
3317 | unsigned long start, |
3318 | unsigned long end_data, |
3319 | unsigned long start_data) |
3320 | { |
3321 | if (rlim < RLIM_INFINITY) { |
3322 | if (((new - start) + (end_data - start_data)) > rlim) |
3323 | return -ENOSPC; |
3324 | } |
3325 | |
3326 | return 0; |
3327 | } |
3328 | |
3329 | extern int mm_take_all_locks(struct mm_struct *mm); |
3330 | extern void mm_drop_all_locks(struct mm_struct *mm); |
3331 | |
3332 | extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); |
3333 | extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); |
3334 | extern struct file *get_mm_exe_file(struct mm_struct *mm); |
3335 | extern struct file *get_task_exe_file(struct task_struct *task); |
3336 | |
3337 | extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages); |
3338 | extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages); |
3339 | |
3340 | extern bool vma_is_special_mapping(const struct vm_area_struct *vma, |
3341 | const struct vm_special_mapping *sm); |
3342 | extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, |
3343 | unsigned long addr, unsigned long len, |
3344 | unsigned long flags, |
3345 | const struct vm_special_mapping *spec); |
3346 | /* This is an obsolete alternative to _install_special_mapping. */ |
3347 | extern int install_special_mapping(struct mm_struct *mm, |
3348 | unsigned long addr, unsigned long len, |
3349 | unsigned long flags, struct page **pages); |
3350 | |
3351 | unsigned long randomize_stack_top(unsigned long stack_top); |
3352 | unsigned long randomize_page(unsigned long start, unsigned long range); |
3353 | |
3354 | extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); |
3355 | |
3356 | extern unsigned long mmap_region(struct file *file, unsigned long addr, |
3357 | unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, |
3358 | struct list_head *uf); |
3359 | extern unsigned long do_mmap(struct file *file, unsigned long addr, |
3360 | unsigned long len, unsigned long prot, unsigned long flags, |
3361 | vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, |
3362 | struct list_head *uf); |
3363 | extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, |
3364 | unsigned long start, size_t len, struct list_head *uf, |
3365 | bool unlock); |
3366 | extern int do_munmap(struct mm_struct *, unsigned long, size_t, |
3367 | struct list_head *uf); |
3368 | extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); |
3369 | |
3370 | #ifdef CONFIG_MMU |
3371 | extern int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, |
3372 | unsigned long start, unsigned long end, |
3373 | struct list_head *uf, bool unlock); |
3374 | extern int __mm_populate(unsigned long addr, unsigned long len, |
3375 | int ignore_errors); |
3376 | static inline void mm_populate(unsigned long addr, unsigned long len) |
3377 | { |
3378 | /* Ignore errors */ |
3379 | (void) __mm_populate(addr, len, ignore_errors: 1); |
3380 | } |
3381 | #else |
3382 | static inline void mm_populate(unsigned long addr, unsigned long len) {} |
3383 | #endif |
3384 | |
3385 | /* This takes the mm semaphore itself */ |
3386 | extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long); |
3387 | extern int vm_munmap(unsigned long, size_t); |
3388 | extern unsigned long __must_check vm_mmap(struct file *, unsigned long, |
3389 | unsigned long, unsigned long, |
3390 | unsigned long, unsigned long); |
3391 | |
3392 | struct vm_unmapped_area_info { |
3393 | #define VM_UNMAPPED_AREA_TOPDOWN 1 |
3394 | unsigned long flags; |
3395 | unsigned long length; |
3396 | unsigned long low_limit; |
3397 | unsigned long high_limit; |
3398 | unsigned long align_mask; |
3399 | unsigned long align_offset; |
3400 | }; |
3401 | |
3402 | extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); |
3403 | |
3404 | /* truncate.c */ |
3405 | extern void truncate_inode_pages(struct address_space *, loff_t); |
3406 | extern void truncate_inode_pages_range(struct address_space *, |
3407 | loff_t lstart, loff_t lend); |
3408 | extern void truncate_inode_pages_final(struct address_space *); |
3409 | |
3410 | /* generic vm_area_ops exported for stackable file systems */ |
3411 | extern vm_fault_t filemap_fault(struct vm_fault *vmf); |
3412 | extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, |
3413 | pgoff_t start_pgoff, pgoff_t end_pgoff); |
3414 | extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); |
3415 | |
3416 | extern unsigned long stack_guard_gap; |
3417 | /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ |
3418 | int expand_stack_locked(struct vm_area_struct *vma, unsigned long address); |
3419 | struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr); |
3420 | |
3421 | /* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */ |
3422 | int expand_downwards(struct vm_area_struct *vma, unsigned long address); |
3423 | |
3424 | /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ |
3425 | extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); |
3426 | extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, |
3427 | struct vm_area_struct **pprev); |
3428 | |
3429 | /* |
3430 | * Look up the first VMA which intersects the interval [start_addr, end_addr) |
3431 | * NULL if none. Assume start_addr < end_addr. |
3432 | */ |
3433 | struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, |
3434 | unsigned long start_addr, unsigned long end_addr); |
3435 | |
3436 | /** |
3437 | * vma_lookup() - Find a VMA at a specific address |
3438 | * @mm: The process address space. |
3439 | * @addr: The user address. |
3440 | * |
3441 | * Return: The vm_area_struct at the given address, %NULL otherwise. |
3442 | */ |
3443 | static inline |
3444 | struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) |
3445 | { |
3446 | return mtree_load(mt: &mm->mm_mt, index: addr); |
3447 | } |
3448 | |
3449 | static inline unsigned long stack_guard_start_gap(struct vm_area_struct *vma) |
3450 | { |
3451 | if (vma->vm_flags & VM_GROWSDOWN) |
3452 | return stack_guard_gap; |
3453 | |
3454 | /* See reasoning around the VM_SHADOW_STACK definition */ |
3455 | if (vma->vm_flags & VM_SHADOW_STACK) |
3456 | return PAGE_SIZE; |
3457 | |
3458 | return 0; |
3459 | } |
3460 | |
3461 | static inline unsigned long vm_start_gap(struct vm_area_struct *vma) |
3462 | { |
3463 | unsigned long gap = stack_guard_start_gap(vma); |
3464 | unsigned long vm_start = vma->vm_start; |
3465 | |
3466 | vm_start -= gap; |
3467 | if (vm_start > vma->vm_start) |
3468 | vm_start = 0; |
3469 | return vm_start; |
3470 | } |
3471 | |
3472 | static inline unsigned long vm_end_gap(struct vm_area_struct *vma) |
3473 | { |
3474 | unsigned long vm_end = vma->vm_end; |
3475 | |
3476 | if (vma->vm_flags & VM_GROWSUP) { |
3477 | vm_end += stack_guard_gap; |
3478 | if (vm_end < vma->vm_end) |
3479 | vm_end = -PAGE_SIZE; |
3480 | } |
3481 | return vm_end; |
3482 | } |
3483 | |
3484 | static inline unsigned long vma_pages(struct vm_area_struct *vma) |
3485 | { |
3486 | return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; |
3487 | } |
3488 | |
3489 | /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ |
3490 | static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, |
3491 | unsigned long vm_start, unsigned long vm_end) |
3492 | { |
3493 | struct vm_area_struct *vma = vma_lookup(mm, addr: vm_start); |
3494 | |
3495 | if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) |
3496 | vma = NULL; |
3497 | |
3498 | return vma; |
3499 | } |
3500 | |
3501 | static inline bool range_in_vma(struct vm_area_struct *vma, |
3502 | unsigned long start, unsigned long end) |
3503 | { |
3504 | return (vma && vma->vm_start <= start && end <= vma->vm_end); |
3505 | } |
3506 | |
3507 | #ifdef CONFIG_MMU |
3508 | pgprot_t vm_get_page_prot(unsigned long vm_flags); |
3509 | void vma_set_page_prot(struct vm_area_struct *vma); |
3510 | #else |
3511 | static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) |
3512 | { |
3513 | return __pgprot(0); |
3514 | } |
3515 | static inline void vma_set_page_prot(struct vm_area_struct *vma) |
3516 | { |
3517 | vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); |
3518 | } |
3519 | #endif |
3520 | |
3521 | void vma_set_file(struct vm_area_struct *vma, struct file *file); |
3522 | |
3523 | #ifdef CONFIG_NUMA_BALANCING |
3524 | unsigned long change_prot_numa(struct vm_area_struct *vma, |
3525 | unsigned long start, unsigned long end); |
3526 | #endif |
3527 | |
3528 | struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, |
3529 | unsigned long addr); |
3530 | int remap_pfn_range(struct vm_area_struct *, unsigned long addr, |
3531 | unsigned long pfn, unsigned long size, pgprot_t); |
3532 | int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, |
3533 | unsigned long pfn, unsigned long size, pgprot_t prot); |
3534 | int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); |
3535 | int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, |
3536 | struct page **pages, unsigned long *num); |
3537 | int vm_map_pages(struct vm_area_struct *vma, struct page **pages, |
3538 | unsigned long num); |
3539 | int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, |
3540 | unsigned long num); |
3541 | vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, |
3542 | unsigned long pfn); |
3543 | vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, |
3544 | unsigned long pfn, pgprot_t pgprot); |
3545 | vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, |
3546 | pfn_t pfn); |
3547 | vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, |
3548 | unsigned long addr, pfn_t pfn); |
3549 | int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); |
3550 | |
3551 | static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, |
3552 | unsigned long addr, struct page *page) |
3553 | { |
3554 | int err = vm_insert_page(vma, addr, page); |
3555 | |
3556 | if (err == -ENOMEM) |
3557 | return VM_FAULT_OOM; |
3558 | if (err < 0 && err != -EBUSY) |
3559 | return VM_FAULT_SIGBUS; |
3560 | |
3561 | return VM_FAULT_NOPAGE; |
3562 | } |
3563 | |
3564 | #ifndef io_remap_pfn_range |
3565 | static inline int io_remap_pfn_range(struct vm_area_struct *vma, |
3566 | unsigned long addr, unsigned long pfn, |
3567 | unsigned long size, pgprot_t prot) |
3568 | { |
3569 | return remap_pfn_range(vma, addr, pfn, size, pgprot_decrypted(prot)); |
3570 | } |
3571 | #endif |
3572 | |
3573 | static inline vm_fault_t vmf_error(int err) |
3574 | { |
3575 | if (err == -ENOMEM) |
3576 | return VM_FAULT_OOM; |
3577 | else if (err == -EHWPOISON) |
3578 | return VM_FAULT_HWPOISON; |
3579 | return VM_FAULT_SIGBUS; |
3580 | } |
3581 | |
3582 | /* |
3583 | * Convert errno to return value for ->page_mkwrite() calls. |
3584 | * |
3585 | * This should eventually be merged with vmf_error() above, but will need a |
3586 | * careful audit of all vmf_error() callers. |
3587 | */ |
3588 | static inline vm_fault_t vmf_fs_error(int err) |
3589 | { |
3590 | if (err == 0) |
3591 | return VM_FAULT_LOCKED; |
3592 | if (err == -EFAULT || err == -EAGAIN) |
3593 | return VM_FAULT_NOPAGE; |
3594 | if (err == -ENOMEM) |
3595 | return VM_FAULT_OOM; |
3596 | /* -ENOSPC, -EDQUOT, -EIO ... */ |
3597 | return VM_FAULT_SIGBUS; |
3598 | } |
3599 | |
3600 | struct page *follow_page(struct vm_area_struct *vma, unsigned long address, |
3601 | unsigned int foll_flags); |
3602 | |
3603 | static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) |
3604 | { |
3605 | if (vm_fault & VM_FAULT_OOM) |
3606 | return -ENOMEM; |
3607 | if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) |
3608 | return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT; |
3609 | if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) |
3610 | return -EFAULT; |
3611 | return 0; |
3612 | } |
3613 | |
3614 | /* |
3615 | * Indicates whether GUP can follow a PROT_NONE mapped page, or whether |
3616 | * a (NUMA hinting) fault is required. |
3617 | */ |
3618 | static inline bool gup_can_follow_protnone(struct vm_area_struct *vma, |
3619 | unsigned int flags) |
3620 | { |
3621 | /* |
3622 | * If callers don't want to honor NUMA hinting faults, no need to |
3623 | * determine if we would actually have to trigger a NUMA hinting fault. |
3624 | */ |
3625 | if (!(flags & FOLL_HONOR_NUMA_FAULT)) |
3626 | return true; |
3627 | |
3628 | /* |
3629 | * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs. |
3630 | * |
3631 | * Requiring a fault here even for inaccessible VMAs would mean that |
3632 | * FOLL_FORCE cannot make any progress, because handle_mm_fault() |
3633 | * refuses to process NUMA hinting faults in inaccessible VMAs. |
3634 | */ |
3635 | return !vma_is_accessible(vma); |
3636 | } |
3637 | |
3638 | typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data); |
3639 | extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, |
3640 | unsigned long size, pte_fn_t fn, void *data); |
3641 | extern int apply_to_existing_page_range(struct mm_struct *mm, |
3642 | unsigned long address, unsigned long size, |
3643 | pte_fn_t fn, void *data); |
3644 | |
3645 | #ifdef CONFIG_PAGE_POISONING |
3646 | extern void __kernel_poison_pages(struct page *page, int numpages); |
3647 | extern void __kernel_unpoison_pages(struct page *page, int numpages); |
3648 | extern bool _page_poisoning_enabled_early; |
3649 | DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled); |
3650 | static inline bool page_poisoning_enabled(void) |
3651 | { |
3652 | return _page_poisoning_enabled_early; |
3653 | } |
3654 | /* |
3655 | * For use in fast paths after init_mem_debugging() has run, or when a |
3656 | * false negative result is not harmful when called too early. |
3657 | */ |
3658 | static inline bool page_poisoning_enabled_static(void) |
3659 | { |
3660 | return static_branch_unlikely(&_page_poisoning_enabled); |
3661 | } |
3662 | static inline void kernel_poison_pages(struct page *page, int numpages) |
3663 | { |
3664 | if (page_poisoning_enabled_static()) |
3665 | __kernel_poison_pages(page, numpages); |
3666 | } |
3667 | static inline void kernel_unpoison_pages(struct page *page, int numpages) |
3668 | { |
3669 | if (page_poisoning_enabled_static()) |
3670 | __kernel_unpoison_pages(page, numpages); |
3671 | } |
3672 | #else |
3673 | static inline bool page_poisoning_enabled(void) { return false; } |
3674 | static inline bool page_poisoning_enabled_static(void) { return false; } |
3675 | static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } |
3676 | static inline void kernel_poison_pages(struct page *page, int numpages) { } |
3677 | static inline void kernel_unpoison_pages(struct page *page, int numpages) { } |
3678 | #endif |
3679 | |
3680 | DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); |
3681 | static inline bool want_init_on_alloc(gfp_t flags) |
3682 | { |
3683 | if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, |
3684 | &init_on_alloc)) |
3685 | return true; |
3686 | return flags & __GFP_ZERO; |
3687 | } |
3688 | |
3689 | DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); |
3690 | static inline bool want_init_on_free(void) |
3691 | { |
3692 | return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, |
3693 | &init_on_free); |
3694 | } |
3695 | |
3696 | extern bool _debug_pagealloc_enabled_early; |
3697 | DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); |
3698 | |
3699 | static inline bool debug_pagealloc_enabled(void) |
3700 | { |
3701 | return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && |
3702 | _debug_pagealloc_enabled_early; |
3703 | } |
3704 | |
3705 | /* |
3706 | * For use in fast paths after mem_debugging_and_hardening_init() has run, |
3707 | * or when a false negative result is not harmful when called too early. |
3708 | */ |
3709 | static inline bool debug_pagealloc_enabled_static(void) |
3710 | { |
3711 | if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) |
3712 | return false; |
3713 | |
3714 | return static_branch_unlikely(&_debug_pagealloc_enabled); |
3715 | } |
3716 | |
3717 | /* |
3718 | * To support DEBUG_PAGEALLOC architecture must ensure that |
3719 | * __kernel_map_pages() never fails |
3720 | */ |
3721 | extern void __kernel_map_pages(struct page *page, int numpages, int enable); |
3722 | #ifdef CONFIG_DEBUG_PAGEALLOC |
3723 | static inline void debug_pagealloc_map_pages(struct page *page, int numpages) |
3724 | { |
3725 | if (debug_pagealloc_enabled_static()) |
3726 | __kernel_map_pages(page, numpages, enable: 1); |
3727 | } |
3728 | |
3729 | static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) |
3730 | { |
3731 | if (debug_pagealloc_enabled_static()) |
3732 | __kernel_map_pages(page, numpages, enable: 0); |
3733 | } |
3734 | |
3735 | extern unsigned int _debug_guardpage_minorder; |
3736 | DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled); |
3737 | |
3738 | static inline unsigned int debug_guardpage_minorder(void) |
3739 | { |
3740 | return _debug_guardpage_minorder; |
3741 | } |
3742 | |
3743 | static inline bool debug_guardpage_enabled(void) |
3744 | { |
3745 | return static_branch_unlikely(&_debug_guardpage_enabled); |
3746 | } |
3747 | |
3748 | static inline bool page_is_guard(struct page *page) |
3749 | { |
3750 | if (!debug_guardpage_enabled()) |
3751 | return false; |
3752 | |
3753 | return PageGuard(page); |
3754 | } |
3755 | |
3756 | bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order, |
3757 | int migratetype); |
3758 | static inline bool set_page_guard(struct zone *zone, struct page *page, |
3759 | unsigned int order, int migratetype) |
3760 | { |
3761 | if (!debug_guardpage_enabled()) |
3762 | return false; |
3763 | return __set_page_guard(zone, page, order, migratetype); |
3764 | } |
3765 | |
3766 | void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order, |
3767 | int migratetype); |
3768 | static inline void clear_page_guard(struct zone *zone, struct page *page, |
3769 | unsigned int order, int migratetype) |
3770 | { |
3771 | if (!debug_guardpage_enabled()) |
3772 | return; |
3773 | __clear_page_guard(zone, page, order, migratetype); |
3774 | } |
3775 | |
3776 | #else /* CONFIG_DEBUG_PAGEALLOC */ |
3777 | static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} |
3778 | static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} |
3779 | static inline unsigned int debug_guardpage_minorder(void) { return 0; } |
3780 | static inline bool debug_guardpage_enabled(void) { return false; } |
3781 | static inline bool page_is_guard(struct page *page) { return false; } |
3782 | static inline bool set_page_guard(struct zone *zone, struct page *page, |
3783 | unsigned int order, int migratetype) { return false; } |
3784 | static inline void clear_page_guard(struct zone *zone, struct page *page, |
3785 | unsigned int order, int migratetype) {} |
3786 | #endif /* CONFIG_DEBUG_PAGEALLOC */ |
3787 | |
3788 | #ifdef __HAVE_ARCH_GATE_AREA |
3789 | extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm); |
3790 | extern int in_gate_area_no_mm(unsigned long addr); |
3791 | extern int in_gate_area(struct mm_struct *mm, unsigned long addr); |
3792 | #else |
3793 | static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm) |
3794 | { |
3795 | return NULL; |
3796 | } |
3797 | static inline int in_gate_area_no_mm(unsigned long addr) { return 0; } |
3798 | static inline int in_gate_area(struct mm_struct *mm, unsigned long addr) |
3799 | { |
3800 | return 0; |
3801 | } |
3802 | #endif /* __HAVE_ARCH_GATE_AREA */ |
3803 | |
3804 | extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm); |
3805 | |
3806 | #ifdef CONFIG_SYSCTL |
3807 | extern int sysctl_drop_caches; |
3808 | int drop_caches_sysctl_handler(struct ctl_table *, int, void *, size_t *, |
3809 | loff_t *); |
3810 | #endif |
3811 | |
3812 | void drop_slab(void); |
3813 | |
3814 | #ifndef CONFIG_MMU |
3815 | #define randomize_va_space 0 |
3816 | #else |
3817 | extern int randomize_va_space; |
3818 | #endif |
3819 | |
3820 | const char * arch_vma_name(struct vm_area_struct *vma); |
3821 | #ifdef CONFIG_MMU |
3822 | void print_vma_addr(char *prefix, unsigned long rip); |
3823 | #else |
3824 | static inline void print_vma_addr(char *prefix, unsigned long rip) |
3825 | { |
3826 | } |
3827 | #endif |
3828 | |
3829 | void *sparse_buffer_alloc(unsigned long size); |
3830 | struct page * __populate_section_memmap(unsigned long pfn, |
3831 | unsigned long nr_pages, int nid, struct vmem_altmap *altmap, |
3832 | struct dev_pagemap *pgmap); |
3833 | void pmd_init(void *addr); |
3834 | void pud_init(void *addr); |
3835 | pgd_t *vmemmap_pgd_populate(unsigned long addr, int node); |
3836 | p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node); |
3837 | pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node); |
3838 | pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node); |
3839 | pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, |
3840 | struct vmem_altmap *altmap, struct page *reuse); |
3841 | void *vmemmap_alloc_block(unsigned long size, int node); |
3842 | struct vmem_altmap; |
3843 | void *vmemmap_alloc_block_buf(unsigned long size, int node, |
3844 | struct vmem_altmap *altmap); |
3845 | void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); |
3846 | void vmemmap_set_pmd(pmd_t *pmd, void *p, int node, |
3847 | unsigned long addr, unsigned long next); |
3848 | int vmemmap_check_pmd(pmd_t *pmd, int node, |
3849 | unsigned long addr, unsigned long next); |
3850 | int vmemmap_populate_basepages(unsigned long start, unsigned long end, |
3851 | int node, struct vmem_altmap *altmap); |
3852 | int vmemmap_populate_hugepages(unsigned long start, unsigned long end, |
3853 | int node, struct vmem_altmap *altmap); |
3854 | int vmemmap_populate(unsigned long start, unsigned long end, int node, |
3855 | struct vmem_altmap *altmap); |
3856 | void vmemmap_populate_print_last(void); |
3857 | #ifdef CONFIG_MEMORY_HOTPLUG |
3858 | void vmemmap_free(unsigned long start, unsigned long end, |
3859 | struct vmem_altmap *altmap); |
3860 | #endif |
3861 | |
3862 | #define VMEMMAP_RESERVE_NR 2 |
3863 | #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP |
3864 | static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, |
3865 | struct dev_pagemap *pgmap) |
3866 | { |
3867 | unsigned long nr_pages; |
3868 | unsigned long nr_vmemmap_pages; |
3869 | |
3870 | if (!pgmap || !is_power_of_2(n: sizeof(struct page))) |
3871 | return false; |
3872 | |
3873 | nr_pages = pgmap_vmemmap_nr(pgmap); |
3874 | nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); |
3875 | /* |
3876 | * For vmemmap optimization with DAX we need minimum 2 vmemmap |
3877 | * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst |
3878 | */ |
3879 | return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR); |
3880 | } |
3881 | /* |
3882 | * If we don't have an architecture override, use the generic rule |
3883 | */ |
3884 | #ifndef vmemmap_can_optimize |
3885 | #define vmemmap_can_optimize __vmemmap_can_optimize |
3886 | #endif |
3887 | |
3888 | #else |
3889 | static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, |
3890 | struct dev_pagemap *pgmap) |
3891 | { |
3892 | return false; |
3893 | } |
3894 | #endif |
3895 | |
3896 | void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, |
3897 | unsigned long nr_pages); |
3898 | |
3899 | enum mf_flags { |
3900 | MF_COUNT_INCREASED = 1 << 0, |
3901 | MF_ACTION_REQUIRED = 1 << 1, |
3902 | MF_MUST_KILL = 1 << 2, |
3903 | MF_SOFT_OFFLINE = 1 << 3, |
3904 | MF_UNPOISON = 1 << 4, |
3905 | MF_SW_SIMULATED = 1 << 5, |
3906 | MF_NO_RETRY = 1 << 6, |
3907 | }; |
3908 | int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, |
3909 | unsigned long count, int mf_flags); |
3910 | extern int memory_failure(unsigned long pfn, int flags); |
3911 | extern void memory_failure_queue_kick(int cpu); |
3912 | extern int unpoison_memory(unsigned long pfn); |
3913 | extern void shake_page(struct page *p); |
3914 | extern atomic_long_t num_poisoned_pages __read_mostly; |
3915 | extern int soft_offline_page(unsigned long pfn, int flags); |
3916 | #ifdef CONFIG_MEMORY_FAILURE |
3917 | /* |
3918 | * Sysfs entries for memory failure handling statistics. |
3919 | */ |
3920 | extern const struct attribute_group memory_failure_attr_group; |
3921 | extern void memory_failure_queue(unsigned long pfn, int flags); |
3922 | extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, |
3923 | bool *migratable_cleared); |
3924 | void num_poisoned_pages_inc(unsigned long pfn); |
3925 | void num_poisoned_pages_sub(unsigned long pfn, long i); |
3926 | struct task_struct *task_early_kill(struct task_struct *tsk, int force_early); |
3927 | #else |
3928 | static inline void memory_failure_queue(unsigned long pfn, int flags) |
3929 | { |
3930 | } |
3931 | |
3932 | static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, |
3933 | bool *migratable_cleared) |
3934 | { |
3935 | return 0; |
3936 | } |
3937 | |
3938 | static inline void num_poisoned_pages_inc(unsigned long pfn) |
3939 | { |
3940 | } |
3941 | |
3942 | static inline void num_poisoned_pages_sub(unsigned long pfn, long i) |
3943 | { |
3944 | } |
3945 | #endif |
3946 | |
3947 | #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_KSM) |
3948 | void add_to_kill_ksm(struct task_struct *tsk, struct page *p, |
3949 | struct vm_area_struct *vma, struct list_head *to_kill, |
3950 | unsigned long ksm_addr); |
3951 | #endif |
3952 | |
3953 | #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) |
3954 | extern void memblk_nr_poison_inc(unsigned long pfn); |
3955 | extern void memblk_nr_poison_sub(unsigned long pfn, long i); |
3956 | #else |
3957 | static inline void memblk_nr_poison_inc(unsigned long pfn) |
3958 | { |
3959 | } |
3960 | |
3961 | static inline void memblk_nr_poison_sub(unsigned long pfn, long i) |
3962 | { |
3963 | } |
3964 | #endif |
3965 | |
3966 | #ifndef arch_memory_failure |
3967 | static inline int arch_memory_failure(unsigned long pfn, int flags) |
3968 | { |
3969 | return -ENXIO; |
3970 | } |
3971 | #endif |
3972 | |
3973 | #ifndef arch_is_platform_page |
3974 | static inline bool arch_is_platform_page(u64 paddr) |
3975 | { |
3976 | return false; |
3977 | } |
3978 | #endif |
3979 | |
3980 | /* |
3981 | * Error handlers for various types of pages. |
3982 | */ |
3983 | enum mf_result { |
3984 | MF_IGNORED, /* Error: cannot be handled */ |
3985 | MF_FAILED, /* Error: handling failed */ |
3986 | MF_DELAYED, /* Will be handled later */ |
3987 | MF_RECOVERED, /* Successfully recovered */ |
3988 | }; |
3989 | |
3990 | enum mf_action_page_type { |
3991 | MF_MSG_KERNEL, |
3992 | MF_MSG_KERNEL_HIGH_ORDER, |
3993 | MF_MSG_SLAB, |
3994 | MF_MSG_DIFFERENT_COMPOUND, |
3995 | MF_MSG_HUGE, |
3996 | MF_MSG_FREE_HUGE, |
3997 | MF_MSG_UNMAP_FAILED, |
3998 | MF_MSG_DIRTY_SWAPCACHE, |
3999 | MF_MSG_CLEAN_SWAPCACHE, |
4000 | MF_MSG_DIRTY_MLOCKED_LRU, |
4001 | MF_MSG_CLEAN_MLOCKED_LRU, |
4002 | MF_MSG_DIRTY_UNEVICTABLE_LRU, |
4003 | MF_MSG_CLEAN_UNEVICTABLE_LRU, |
4004 | MF_MSG_DIRTY_LRU, |
4005 | MF_MSG_CLEAN_LRU, |
4006 | MF_MSG_TRUNCATED_LRU, |
4007 | MF_MSG_BUDDY, |
4008 | MF_MSG_DAX, |
4009 | MF_MSG_UNSPLIT_THP, |
4010 | MF_MSG_UNKNOWN, |
4011 | }; |
4012 | |
4013 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) |
4014 | extern void clear_huge_page(struct page *page, |
4015 | unsigned long addr_hint, |
4016 | unsigned int pages_per_huge_page); |
4017 | int copy_user_large_folio(struct folio *dst, struct folio *src, |
4018 | unsigned long addr_hint, |
4019 | struct vm_area_struct *vma); |
4020 | long copy_folio_from_user(struct folio *dst_folio, |
4021 | const void __user *usr_src, |
4022 | bool allow_pagefault); |
4023 | |
4024 | /** |
4025 | * vma_is_special_huge - Are transhuge page-table entries considered special? |
4026 | * @vma: Pointer to the struct vm_area_struct to consider |
4027 | * |
4028 | * Whether transhuge page-table entries are considered "special" following |
4029 | * the definition in vm_normal_page(). |
4030 | * |
4031 | * Return: true if transhuge page-table entries should be considered special, |
4032 | * false otherwise. |
4033 | */ |
4034 | static inline bool vma_is_special_huge(const struct vm_area_struct *vma) |
4035 | { |
4036 | return vma_is_dax(vma) || (vma->vm_file && |
4037 | (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))); |
4038 | } |
4039 | |
4040 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ |
4041 | |
4042 | #if MAX_NUMNODES > 1 |
4043 | void __init setup_nr_node_ids(void); |
4044 | #else |
4045 | static inline void setup_nr_node_ids(void) {} |
4046 | #endif |
4047 | |
4048 | extern int memcmp_pages(struct page *page1, struct page *page2); |
4049 | |
4050 | static inline int pages_identical(struct page *page1, struct page *page2) |
4051 | { |
4052 | return !memcmp_pages(page1, page2); |
4053 | } |
4054 | |
4055 | #ifdef CONFIG_MAPPING_DIRTY_HELPERS |
4056 | unsigned long clean_record_shared_mapping_range(struct address_space *mapping, |
4057 | pgoff_t first_index, pgoff_t nr, |
4058 | pgoff_t bitmap_pgoff, |
4059 | unsigned long *bitmap, |
4060 | pgoff_t *start, |
4061 | pgoff_t *end); |
4062 | |
4063 | unsigned long wp_shared_mapping_range(struct address_space *mapping, |
4064 | pgoff_t first_index, pgoff_t nr); |
4065 | #endif |
4066 | |
4067 | extern int sysctl_nr_trim_pages; |
4068 | |
4069 | #ifdef CONFIG_PRINTK |
4070 | void mem_dump_obj(void *object); |
4071 | #else |
4072 | static inline void mem_dump_obj(void *object) {} |
4073 | #endif |
4074 | |
4075 | /** |
4076 | * seal_check_write - Check for F_SEAL_WRITE or F_SEAL_FUTURE_WRITE flags and |
4077 | * handle them. |
4078 | * @seals: the seals to check |
4079 | * @vma: the vma to operate on |
4080 | * |
4081 | * Check whether F_SEAL_WRITE or F_SEAL_FUTURE_WRITE are set; if so, do proper |
4082 | * check/handling on the vma flags. Return 0 if check pass, or <0 for errors. |
4083 | */ |
4084 | static inline int seal_check_write(int seals, struct vm_area_struct *vma) |
4085 | { |
4086 | if (seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { |
4087 | /* |
4088 | * New PROT_WRITE and MAP_SHARED mmaps are not allowed when |
4089 | * write seals are active. |
4090 | */ |
4091 | if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE)) |
4092 | return -EPERM; |
4093 | |
4094 | /* |
4095 | * Since an F_SEAL_[FUTURE_]WRITE sealed memfd can be mapped as |
4096 | * MAP_SHARED and read-only, take care to not allow mprotect to |
4097 | * revert protections on such mappings. Do this only for shared |
4098 | * mappings. For private mappings, don't need to mask |
4099 | * VM_MAYWRITE as we still want them to be COW-writable. |
4100 | */ |
4101 | if (vma->vm_flags & VM_SHARED) |
4102 | vm_flags_clear(vma, VM_MAYWRITE); |
4103 | } |
4104 | |
4105 | return 0; |
4106 | } |
4107 | |
4108 | #ifdef CONFIG_ANON_VMA_NAME |
4109 | int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, |
4110 | unsigned long len_in, |
4111 | struct anon_vma_name *anon_name); |
4112 | #else |
4113 | static inline int |
4114 | madvise_set_anon_name(struct mm_struct *mm, unsigned long start, |
4115 | unsigned long len_in, struct anon_vma_name *anon_name) { |
4116 | return 0; |
4117 | } |
4118 | #endif |
4119 | |
4120 | #ifdef CONFIG_UNACCEPTED_MEMORY |
4121 | |
4122 | bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end); |
4123 | void accept_memory(phys_addr_t start, phys_addr_t end); |
4124 | |
4125 | #else |
4126 | |
4127 | static inline bool range_contains_unaccepted_memory(phys_addr_t start, |
4128 | phys_addr_t end) |
4129 | { |
4130 | return false; |
4131 | } |
4132 | |
4133 | static inline void accept_memory(phys_addr_t start, phys_addr_t end) |
4134 | { |
4135 | } |
4136 | |
4137 | #endif |
4138 | |
4139 | static inline bool pfn_is_unaccepted_memory(unsigned long pfn) |
4140 | { |
4141 | phys_addr_t paddr = pfn << PAGE_SHIFT; |
4142 | |
4143 | return range_contains_unaccepted_memory(start: paddr, end: paddr + PAGE_SIZE); |
4144 | } |
4145 | |
4146 | #endif /* _LINUX_MM_H */ |
4147 | |