1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef __LINUX_GFP_H |
3 | #define __LINUX_GFP_H |
4 | |
5 | #include <linux/gfp_types.h> |
6 | |
7 | #include <linux/mmzone.h> |
8 | #include <linux/topology.h> |
9 | |
10 | struct vm_area_struct; |
11 | |
12 | /* Convert GFP flags to their corresponding migrate type */ |
13 | #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) |
14 | #define GFP_MOVABLE_SHIFT 3 |
15 | |
16 | static inline int gfp_migratetype(const gfp_t gfp_flags) |
17 | { |
18 | VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); |
19 | BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); |
20 | BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); |
21 | |
22 | if (unlikely(page_group_by_mobility_disabled)) |
23 | return MIGRATE_UNMOVABLE; |
24 | |
25 | /* Group based on mobility */ |
26 | return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; |
27 | } |
28 | #undef GFP_MOVABLE_MASK |
29 | #undef GFP_MOVABLE_SHIFT |
30 | |
31 | static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) |
32 | { |
33 | return !!(gfp_flags & __GFP_DIRECT_RECLAIM); |
34 | } |
35 | |
36 | /** |
37 | * gfpflags_normal_context - is gfp_flags a normal sleepable context? |
38 | * @gfp_flags: gfp_flags to test |
39 | * |
40 | * Test whether @gfp_flags indicates that the allocation is from the |
41 | * %current context and allowed to sleep. |
42 | * |
43 | * An allocation being allowed to block doesn't mean it owns the %current |
44 | * context. When direct reclaim path tries to allocate memory, the |
45 | * allocation context is nested inside whatever %current was doing at the |
46 | * time of the original allocation. The nested allocation may be allowed |
47 | * to block but modifying anything %current owns can corrupt the outer |
48 | * context's expectations. |
49 | * |
50 | * %true result from this function indicates that the allocation context |
51 | * can sleep and use anything that's associated with %current. |
52 | */ |
53 | static inline bool gfpflags_normal_context(const gfp_t gfp_flags) |
54 | { |
55 | return (gfp_flags & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC)) == |
56 | __GFP_DIRECT_RECLAIM; |
57 | } |
58 | |
59 | #ifdef CONFIG_HIGHMEM |
60 | #define OPT_ZONE_HIGHMEM ZONE_HIGHMEM |
61 | #else |
62 | #define OPT_ZONE_HIGHMEM ZONE_NORMAL |
63 | #endif |
64 | |
65 | #ifdef CONFIG_ZONE_DMA |
66 | #define OPT_ZONE_DMA ZONE_DMA |
67 | #else |
68 | #define OPT_ZONE_DMA ZONE_NORMAL |
69 | #endif |
70 | |
71 | #ifdef CONFIG_ZONE_DMA32 |
72 | #define OPT_ZONE_DMA32 ZONE_DMA32 |
73 | #else |
74 | #define OPT_ZONE_DMA32 ZONE_NORMAL |
75 | #endif |
76 | |
77 | /* |
78 | * GFP_ZONE_TABLE is a word size bitstring that is used for looking up the |
79 | * zone to use given the lowest 4 bits of gfp_t. Entries are GFP_ZONES_SHIFT |
80 | * bits long and there are 16 of them to cover all possible combinations of |
81 | * __GFP_DMA, __GFP_DMA32, __GFP_MOVABLE and __GFP_HIGHMEM. |
82 | * |
83 | * The zone fallback order is MOVABLE=>HIGHMEM=>NORMAL=>DMA32=>DMA. |
84 | * But GFP_MOVABLE is not only a zone specifier but also an allocation |
85 | * policy. Therefore __GFP_MOVABLE plus another zone selector is valid. |
86 | * Only 1 bit of the lowest 3 bits (DMA,DMA32,HIGHMEM) can be set to "1". |
87 | * |
88 | * bit result |
89 | * ================= |
90 | * 0x0 => NORMAL |
91 | * 0x1 => DMA or NORMAL |
92 | * 0x2 => HIGHMEM or NORMAL |
93 | * 0x3 => BAD (DMA+HIGHMEM) |
94 | * 0x4 => DMA32 or NORMAL |
95 | * 0x5 => BAD (DMA+DMA32) |
96 | * 0x6 => BAD (HIGHMEM+DMA32) |
97 | * 0x7 => BAD (HIGHMEM+DMA32+DMA) |
98 | * 0x8 => NORMAL (MOVABLE+0) |
99 | * 0x9 => DMA or NORMAL (MOVABLE+DMA) |
100 | * 0xa => MOVABLE (Movable is valid only if HIGHMEM is set too) |
101 | * 0xb => BAD (MOVABLE+HIGHMEM+DMA) |
102 | * 0xc => DMA32 or NORMAL (MOVABLE+DMA32) |
103 | * 0xd => BAD (MOVABLE+DMA32+DMA) |
104 | * 0xe => BAD (MOVABLE+DMA32+HIGHMEM) |
105 | * 0xf => BAD (MOVABLE+DMA32+HIGHMEM+DMA) |
106 | * |
107 | * GFP_ZONES_SHIFT must be <= 2 on 32 bit platforms. |
108 | */ |
109 | |
110 | #if defined(CONFIG_ZONE_DEVICE) && (MAX_NR_ZONES-1) <= 4 |
111 | /* ZONE_DEVICE is not a valid GFP zone specifier */ |
112 | #define GFP_ZONES_SHIFT 2 |
113 | #else |
114 | #define GFP_ZONES_SHIFT ZONES_SHIFT |
115 | #endif |
116 | |
117 | #if 16 * GFP_ZONES_SHIFT > BITS_PER_LONG |
118 | #error GFP_ZONES_SHIFT too large to create GFP_ZONE_TABLE integer |
119 | #endif |
120 | |
121 | #define GFP_ZONE_TABLE ( \ |
122 | (ZONE_NORMAL << 0 * GFP_ZONES_SHIFT) \ |
123 | | (OPT_ZONE_DMA << ___GFP_DMA * GFP_ZONES_SHIFT) \ |
124 | | (OPT_ZONE_HIGHMEM << ___GFP_HIGHMEM * GFP_ZONES_SHIFT) \ |
125 | | (OPT_ZONE_DMA32 << ___GFP_DMA32 * GFP_ZONES_SHIFT) \ |
126 | | (ZONE_NORMAL << ___GFP_MOVABLE * GFP_ZONES_SHIFT) \ |
127 | | (OPT_ZONE_DMA << (___GFP_MOVABLE | ___GFP_DMA) * GFP_ZONES_SHIFT) \ |
128 | | (ZONE_MOVABLE << (___GFP_MOVABLE | ___GFP_HIGHMEM) * GFP_ZONES_SHIFT)\ |
129 | | (OPT_ZONE_DMA32 << (___GFP_MOVABLE | ___GFP_DMA32) * GFP_ZONES_SHIFT)\ |
130 | ) |
131 | |
132 | /* |
133 | * GFP_ZONE_BAD is a bitmap for all combinations of __GFP_DMA, __GFP_DMA32 |
134 | * __GFP_HIGHMEM and __GFP_MOVABLE that are not permitted. One flag per |
135 | * entry starting with bit 0. Bit is set if the combination is not |
136 | * allowed. |
137 | */ |
138 | #define GFP_ZONE_BAD ( \ |
139 | 1 << (___GFP_DMA | ___GFP_HIGHMEM) \ |
140 | | 1 << (___GFP_DMA | ___GFP_DMA32) \ |
141 | | 1 << (___GFP_DMA32 | ___GFP_HIGHMEM) \ |
142 | | 1 << (___GFP_DMA | ___GFP_DMA32 | ___GFP_HIGHMEM) \ |
143 | | 1 << (___GFP_MOVABLE | ___GFP_HIGHMEM | ___GFP_DMA) \ |
144 | | 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_DMA) \ |
145 | | 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_HIGHMEM) \ |
146 | | 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_DMA | ___GFP_HIGHMEM) \ |
147 | ) |
148 | |
149 | static inline enum zone_type gfp_zone(gfp_t flags) |
150 | { |
151 | enum zone_type z; |
152 | int bit = (__force int) (flags & GFP_ZONEMASK); |
153 | |
154 | z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) & |
155 | ((1 << GFP_ZONES_SHIFT) - 1); |
156 | VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1); |
157 | return z; |
158 | } |
159 | |
160 | /* |
161 | * There is only one page-allocator function, and two main namespaces to |
162 | * it. The alloc_page*() variants return 'struct page *' and as such |
163 | * can allocate highmem pages, the *get*page*() variants return |
164 | * virtual kernel addresses to the allocated page(s). |
165 | */ |
166 | |
167 | static inline int gfp_zonelist(gfp_t flags) |
168 | { |
169 | #ifdef CONFIG_NUMA |
170 | if (unlikely(flags & __GFP_THISNODE)) |
171 | return ZONELIST_NOFALLBACK; |
172 | #endif |
173 | return ZONELIST_FALLBACK; |
174 | } |
175 | |
176 | /* |
177 | * We get the zone list from the current node and the gfp_mask. |
178 | * This zone list contains a maximum of MAX_NUMNODES*MAX_NR_ZONES zones. |
179 | * There are two zonelists per node, one for all zones with memory and |
180 | * one containing just zones from the node the zonelist belongs to. |
181 | * |
182 | * For the case of non-NUMA systems the NODE_DATA() gets optimized to |
183 | * &contig_page_data at compile-time. |
184 | */ |
185 | static inline struct zonelist *node_zonelist(int nid, gfp_t flags) |
186 | { |
187 | return NODE_DATA(nid)->node_zonelists + gfp_zonelist(flags); |
188 | } |
189 | |
190 | #ifndef HAVE_ARCH_FREE_PAGE |
191 | static inline void arch_free_page(struct page *page, int order) { } |
192 | #endif |
193 | #ifndef HAVE_ARCH_ALLOC_PAGE |
194 | static inline void arch_alloc_page(struct page *page, int order) { } |
195 | #endif |
196 | |
197 | struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, |
198 | nodemask_t *nodemask); |
199 | struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid, |
200 | nodemask_t *nodemask); |
201 | |
202 | unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, |
203 | nodemask_t *nodemask, int nr_pages, |
204 | struct list_head *page_list, |
205 | struct page **page_array); |
206 | |
207 | unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp, |
208 | unsigned long nr_pages, |
209 | struct page **page_array); |
210 | |
211 | /* Bulk allocate order-0 pages */ |
212 | static inline unsigned long |
213 | alloc_pages_bulk_list(gfp_t gfp, unsigned long nr_pages, struct list_head *list) |
214 | { |
215 | return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, list, NULL); |
216 | } |
217 | |
218 | static inline unsigned long |
219 | alloc_pages_bulk_array(gfp_t gfp, unsigned long nr_pages, struct page **page_array) |
220 | { |
221 | return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, NULL, page_array); |
222 | } |
223 | |
224 | static inline unsigned long |
225 | alloc_pages_bulk_array_node(gfp_t gfp, int nid, unsigned long nr_pages, struct page **page_array) |
226 | { |
227 | if (nid == NUMA_NO_NODE) |
228 | nid = numa_mem_id(); |
229 | |
230 | return __alloc_pages_bulk(gfp, nid, NULL, nr_pages, NULL, page_array); |
231 | } |
232 | |
233 | /* |
234 | * Allocate pages, preferring the node given as nid. The node must be valid and |
235 | * online. For more general interface, see alloc_pages_node(). |
236 | */ |
237 | static inline struct page * |
238 | __alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order) |
239 | { |
240 | VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); |
241 | VM_WARN_ON((gfp_mask & __GFP_THISNODE) && !node_online(nid)); |
242 | |
243 | return __alloc_pages(gfp_mask, order, nid, NULL); |
244 | } |
245 | |
246 | static inline |
247 | struct folio *__folio_alloc_node(gfp_t gfp, unsigned int order, int nid) |
248 | { |
249 | VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); |
250 | VM_WARN_ON((gfp & __GFP_THISNODE) && !node_online(nid)); |
251 | |
252 | return __folio_alloc(gfp, order, nid, NULL); |
253 | } |
254 | |
255 | /* |
256 | * Allocate pages, preferring the node given as nid. When nid == NUMA_NO_NODE, |
257 | * prefer the current CPU's closest node. Otherwise node must be valid and |
258 | * online. |
259 | */ |
260 | static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask, |
261 | unsigned int order) |
262 | { |
263 | if (nid == NUMA_NO_NODE) |
264 | nid = numa_mem_id(); |
265 | |
266 | return __alloc_pages_node(nid, gfp_mask, order); |
267 | } |
268 | |
269 | #ifdef CONFIG_NUMA |
270 | struct page *alloc_pages(gfp_t gfp, unsigned int order); |
271 | struct folio *folio_alloc(gfp_t gfp, unsigned order); |
272 | struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma, |
273 | unsigned long addr, bool hugepage); |
274 | #else |
275 | static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order) |
276 | { |
277 | return alloc_pages_node(numa_node_id(), gfp_mask, order); |
278 | } |
279 | static inline struct folio *folio_alloc(gfp_t gfp, unsigned int order) |
280 | { |
281 | return __folio_alloc_node(gfp, order, numa_node_id()); |
282 | } |
283 | #define vma_alloc_folio(gfp, order, vma, addr, hugepage) \ |
284 | folio_alloc(gfp, order) |
285 | #endif |
286 | #define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0) |
287 | static inline struct page *alloc_page_vma(gfp_t gfp, |
288 | struct vm_area_struct *vma, unsigned long addr) |
289 | { |
290 | struct folio *folio = vma_alloc_folio(gfp, 0, vma, addr, false); |
291 | |
292 | return &folio->page; |
293 | } |
294 | |
295 | extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order); |
296 | extern unsigned long get_zeroed_page(gfp_t gfp_mask); |
297 | |
298 | void *alloc_pages_exact(size_t size, gfp_t gfp_mask) __alloc_size(1); |
299 | void free_pages_exact(void *virt, size_t size); |
300 | __meminit void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2); |
301 | |
302 | #define __get_free_page(gfp_mask) \ |
303 | __get_free_pages((gfp_mask), 0) |
304 | |
305 | #define __get_dma_pages(gfp_mask, order) \ |
306 | __get_free_pages((gfp_mask) | GFP_DMA, (order)) |
307 | |
308 | extern void __free_pages(struct page *page, unsigned int order); |
309 | extern void free_pages(unsigned long addr, unsigned int order); |
310 | |
311 | struct page_frag_cache; |
312 | extern void __page_frag_cache_drain(struct page *page, unsigned int count); |
313 | extern void *page_frag_alloc_align(struct page_frag_cache *nc, |
314 | unsigned int fragsz, gfp_t gfp_mask, |
315 | unsigned int align_mask); |
316 | |
317 | static inline void *page_frag_alloc(struct page_frag_cache *nc, |
318 | unsigned int fragsz, gfp_t gfp_mask) |
319 | { |
320 | return page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u); |
321 | } |
322 | |
323 | extern void page_frag_free(void *addr); |
324 | |
325 | #define __free_page(page) __free_pages((page), 0) |
326 | #define free_page(addr) free_pages((addr), 0) |
327 | |
328 | void page_alloc_init(void); |
329 | void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); |
330 | void drain_all_pages(struct zone *zone); |
331 | void drain_local_pages(struct zone *zone); |
332 | |
333 | void page_alloc_init_late(void); |
334 | |
335 | /* |
336 | * gfp_allowed_mask is set to GFP_BOOT_MASK during early boot to restrict what |
337 | * GFP flags are used before interrupts are enabled. Once interrupts are |
338 | * enabled, it is set to __GFP_BITS_MASK while the system is running. During |
339 | * hibernation, it is used by PM to avoid I/O during memory allocation while |
340 | * devices are suspended. |
341 | */ |
342 | extern gfp_t gfp_allowed_mask; |
343 | |
344 | /* Returns true if the gfp_mask allows use of ALLOC_NO_WATERMARK */ |
345 | bool gfp_pfmemalloc_allowed(gfp_t gfp_mask); |
346 | |
347 | extern void pm_restrict_gfp_mask(void); |
348 | extern void pm_restore_gfp_mask(void); |
349 | |
350 | extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma); |
351 | |
352 | #ifdef CONFIG_PM_SLEEP |
353 | extern bool pm_suspended_storage(void); |
354 | #else |
355 | static inline bool pm_suspended_storage(void) |
356 | { |
357 | return false; |
358 | } |
359 | #endif /* CONFIG_PM_SLEEP */ |
360 | |
361 | #ifdef CONFIG_CONTIG_ALLOC |
362 | /* The below functions must be run on a range from a single zone. */ |
363 | extern int alloc_contig_range(unsigned long start, unsigned long end, |
364 | unsigned migratetype, gfp_t gfp_mask); |
365 | extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, |
366 | int nid, nodemask_t *nodemask); |
367 | #endif |
368 | void free_contig_range(unsigned long pfn, unsigned long nr_pages); |
369 | |
370 | #ifdef CONFIG_CMA |
371 | /* CMA stuff */ |
372 | extern void init_cma_reserved_pageblock(struct page *page); |
373 | #endif |
374 | |
375 | #endif /* __LINUX_GFP_H */ |
376 | |