1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * address space "slices" (meta-segments) support |
4 | * |
5 | * Copyright (C) 2007 Benjamin Herrenschmidt, IBM Corporation. |
6 | * |
7 | * Based on hugetlb implementation |
8 | * |
9 | * Copyright (C) 2003 David Gibson, IBM Corporation. |
10 | */ |
11 | |
12 | #undef DEBUG |
13 | |
14 | #include <linux/kernel.h> |
15 | #include <linux/mm.h> |
16 | #include <linux/pagemap.h> |
17 | #include <linux/err.h> |
18 | #include <linux/spinlock.h> |
19 | #include <linux/export.h> |
20 | #include <linux/hugetlb.h> |
21 | #include <linux/sched/mm.h> |
22 | #include <linux/security.h> |
23 | #include <asm/mman.h> |
24 | #include <asm/mmu.h> |
25 | #include <asm/copro.h> |
26 | #include <asm/hugetlb.h> |
27 | #include <asm/mmu_context.h> |
28 | |
29 | static DEFINE_SPINLOCK(slice_convert_lock); |
30 | |
31 | #ifdef DEBUG |
32 | int _slice_debug = 1; |
33 | |
34 | static void slice_print_mask(const char *label, const struct slice_mask *mask) |
35 | { |
36 | if (!_slice_debug) |
37 | return; |
38 | pr_devel("%s low_slice: %*pbl\n" , label, |
39 | (int)SLICE_NUM_LOW, &mask->low_slices); |
40 | pr_devel("%s high_slice: %*pbl\n" , label, |
41 | (int)SLICE_NUM_HIGH, mask->high_slices); |
42 | } |
43 | |
44 | #define slice_dbg(fmt...) do { if (_slice_debug) pr_devel(fmt); } while (0) |
45 | |
46 | #else |
47 | |
48 | static void slice_print_mask(const char *label, const struct slice_mask *mask) {} |
49 | #define slice_dbg(fmt...) |
50 | |
51 | #endif |
52 | |
53 | static inline notrace bool slice_addr_is_low(unsigned long addr) |
54 | { |
55 | u64 tmp = (u64)addr; |
56 | |
57 | return tmp < SLICE_LOW_TOP; |
58 | } |
59 | |
60 | static void slice_range_to_mask(unsigned long start, unsigned long len, |
61 | struct slice_mask *ret) |
62 | { |
63 | unsigned long end = start + len - 1; |
64 | |
65 | ret->low_slices = 0; |
66 | if (SLICE_NUM_HIGH) |
67 | bitmap_zero(dst: ret->high_slices, nbits: SLICE_NUM_HIGH); |
68 | |
69 | if (slice_addr_is_low(addr: start)) { |
70 | unsigned long mend = min(end, |
71 | (unsigned long)(SLICE_LOW_TOP - 1)); |
72 | |
73 | ret->low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1)) |
74 | - (1u << GET_LOW_SLICE_INDEX(start)); |
75 | } |
76 | |
77 | if (SLICE_NUM_HIGH && !slice_addr_is_low(addr: end)) { |
78 | unsigned long start_index = GET_HIGH_SLICE_INDEX(start); |
79 | unsigned long align_end = ALIGN(end, (1UL << SLICE_HIGH_SHIFT)); |
80 | unsigned long count = GET_HIGH_SLICE_INDEX(align_end) - start_index; |
81 | |
82 | bitmap_set(map: ret->high_slices, start: start_index, nbits: count); |
83 | } |
84 | } |
85 | |
86 | static int slice_area_is_free(struct mm_struct *mm, unsigned long addr, |
87 | unsigned long len) |
88 | { |
89 | struct vm_area_struct *vma; |
90 | |
91 | if ((mm_ctx_slb_addr_limit(&mm->context) - len) < addr) |
92 | return 0; |
93 | vma = find_vma(mm, addr); |
94 | return (!vma || (addr + len) <= vm_start_gap(vma)); |
95 | } |
96 | |
97 | static int slice_low_has_vma(struct mm_struct *mm, unsigned long slice) |
98 | { |
99 | return !slice_area_is_free(mm, addr: slice << SLICE_LOW_SHIFT, |
100 | len: 1ul << SLICE_LOW_SHIFT); |
101 | } |
102 | |
103 | static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice) |
104 | { |
105 | unsigned long start = slice << SLICE_HIGH_SHIFT; |
106 | unsigned long end = start + (1ul << SLICE_HIGH_SHIFT); |
107 | |
108 | /* Hack, so that each addresses is controlled by exactly one |
109 | * of the high or low area bitmaps, the first high area starts |
110 | * at 4GB, not 0 */ |
111 | if (start == 0) |
112 | start = (unsigned long)SLICE_LOW_TOP; |
113 | |
114 | return !slice_area_is_free(mm, addr: start, len: end - start); |
115 | } |
116 | |
117 | static void slice_mask_for_free(struct mm_struct *mm, struct slice_mask *ret, |
118 | unsigned long high_limit) |
119 | { |
120 | unsigned long i; |
121 | |
122 | ret->low_slices = 0; |
123 | if (SLICE_NUM_HIGH) |
124 | bitmap_zero(dst: ret->high_slices, nbits: SLICE_NUM_HIGH); |
125 | |
126 | for (i = 0; i < SLICE_NUM_LOW; i++) |
127 | if (!slice_low_has_vma(mm, slice: i)) |
128 | ret->low_slices |= 1u << i; |
129 | |
130 | if (slice_addr_is_low(addr: high_limit - 1)) |
131 | return; |
132 | |
133 | for (i = 0; i < GET_HIGH_SLICE_INDEX(high_limit); i++) |
134 | if (!slice_high_has_vma(mm, slice: i)) |
135 | __set_bit(i, ret->high_slices); |
136 | } |
137 | |
138 | static bool slice_check_range_fits(struct mm_struct *mm, |
139 | const struct slice_mask *available, |
140 | unsigned long start, unsigned long len) |
141 | { |
142 | unsigned long end = start + len - 1; |
143 | u64 low_slices = 0; |
144 | |
145 | if (slice_addr_is_low(addr: start)) { |
146 | unsigned long mend = min(end, |
147 | (unsigned long)(SLICE_LOW_TOP - 1)); |
148 | |
149 | low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1)) |
150 | - (1u << GET_LOW_SLICE_INDEX(start)); |
151 | } |
152 | if ((low_slices & available->low_slices) != low_slices) |
153 | return false; |
154 | |
155 | if (SLICE_NUM_HIGH && !slice_addr_is_low(addr: end)) { |
156 | unsigned long start_index = GET_HIGH_SLICE_INDEX(start); |
157 | unsigned long align_end = ALIGN(end, (1UL << SLICE_HIGH_SHIFT)); |
158 | unsigned long count = GET_HIGH_SLICE_INDEX(align_end) - start_index; |
159 | unsigned long i; |
160 | |
161 | for (i = start_index; i < start_index + count; i++) { |
162 | if (!test_bit(i, available->high_slices)) |
163 | return false; |
164 | } |
165 | } |
166 | |
167 | return true; |
168 | } |
169 | |
170 | static void slice_flush_segments(void *parm) |
171 | { |
172 | #ifdef CONFIG_PPC64 |
173 | struct mm_struct *mm = parm; |
174 | unsigned long flags; |
175 | |
176 | if (mm != current->active_mm) |
177 | return; |
178 | |
179 | copy_mm_to_paca(current->active_mm); |
180 | |
181 | local_irq_save(flags); |
182 | slb_flush_and_restore_bolted(); |
183 | local_irq_restore(flags); |
184 | #endif |
185 | } |
186 | |
187 | static void slice_convert(struct mm_struct *mm, |
188 | const struct slice_mask *mask, int psize) |
189 | { |
190 | int index, mask_index; |
191 | /* Write the new slice psize bits */ |
192 | unsigned char *hpsizes, *lpsizes; |
193 | struct slice_mask *psize_mask, *old_mask; |
194 | unsigned long i, flags; |
195 | int old_psize; |
196 | |
197 | slice_dbg("slice_convert(mm=%p, psize=%d)\n" , mm, psize); |
198 | slice_print_mask(label: " mask" , mask); |
199 | |
200 | psize_mask = slice_mask_for_size(&mm->context, psize); |
201 | |
202 | /* We need to use a spinlock here to protect against |
203 | * concurrent 64k -> 4k demotion ... |
204 | */ |
205 | spin_lock_irqsave(&slice_convert_lock, flags); |
206 | |
207 | lpsizes = mm_ctx_low_slices(&mm->context); |
208 | for (i = 0; i < SLICE_NUM_LOW; i++) { |
209 | if (!(mask->low_slices & (1u << i))) |
210 | continue; |
211 | |
212 | mask_index = i & 0x1; |
213 | index = i >> 1; |
214 | |
215 | /* Update the slice_mask */ |
216 | old_psize = (lpsizes[index] >> (mask_index * 4)) & 0xf; |
217 | old_mask = slice_mask_for_size(&mm->context, old_psize); |
218 | old_mask->low_slices &= ~(1u << i); |
219 | psize_mask->low_slices |= 1u << i; |
220 | |
221 | /* Update the sizes array */ |
222 | lpsizes[index] = (lpsizes[index] & ~(0xf << (mask_index * 4))) | |
223 | (((unsigned long)psize) << (mask_index * 4)); |
224 | } |
225 | |
226 | hpsizes = mm_ctx_high_slices(&mm->context); |
227 | for (i = 0; i < GET_HIGH_SLICE_INDEX(mm_ctx_slb_addr_limit(&mm->context)); i++) { |
228 | if (!test_bit(i, mask->high_slices)) |
229 | continue; |
230 | |
231 | mask_index = i & 0x1; |
232 | index = i >> 1; |
233 | |
234 | /* Update the slice_mask */ |
235 | old_psize = (hpsizes[index] >> (mask_index * 4)) & 0xf; |
236 | old_mask = slice_mask_for_size(&mm->context, old_psize); |
237 | __clear_bit(i, old_mask->high_slices); |
238 | __set_bit(i, psize_mask->high_slices); |
239 | |
240 | /* Update the sizes array */ |
241 | hpsizes[index] = (hpsizes[index] & ~(0xf << (mask_index * 4))) | |
242 | (((unsigned long)psize) << (mask_index * 4)); |
243 | } |
244 | |
245 | slice_dbg(" lsps=%lx, hsps=%lx\n" , |
246 | (unsigned long)mm_ctx_low_slices(&mm->context), |
247 | (unsigned long)mm_ctx_high_slices(&mm->context)); |
248 | |
249 | spin_unlock_irqrestore(lock: &slice_convert_lock, flags); |
250 | |
251 | copro_flush_all_slbs(mm); |
252 | } |
253 | |
254 | /* |
255 | * Compute which slice addr is part of; |
256 | * set *boundary_addr to the start or end boundary of that slice |
257 | * (depending on 'end' parameter); |
258 | * return boolean indicating if the slice is marked as available in the |
259 | * 'available' slice_mark. |
260 | */ |
261 | static bool slice_scan_available(unsigned long addr, |
262 | const struct slice_mask *available, |
263 | int end, unsigned long *boundary_addr) |
264 | { |
265 | unsigned long slice; |
266 | if (slice_addr_is_low(addr)) { |
267 | slice = GET_LOW_SLICE_INDEX(addr); |
268 | *boundary_addr = (slice + end) << SLICE_LOW_SHIFT; |
269 | return !!(available->low_slices & (1u << slice)); |
270 | } else { |
271 | slice = GET_HIGH_SLICE_INDEX(addr); |
272 | *boundary_addr = (slice + end) ? |
273 | ((slice + end) << SLICE_HIGH_SHIFT) : SLICE_LOW_TOP; |
274 | return !!test_bit(slice, available->high_slices); |
275 | } |
276 | } |
277 | |
278 | static unsigned long slice_find_area_bottomup(struct mm_struct *mm, |
279 | unsigned long addr, unsigned long len, |
280 | const struct slice_mask *available, |
281 | int psize, unsigned long high_limit) |
282 | { |
283 | int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); |
284 | unsigned long found, next_end; |
285 | struct vm_unmapped_area_info info; |
286 | |
287 | info.flags = 0; |
288 | info.length = len; |
289 | info.align_mask = PAGE_MASK & ((1ul << pshift) - 1); |
290 | info.align_offset = 0; |
291 | /* |
292 | * Check till the allow max value for this mmap request |
293 | */ |
294 | while (addr < high_limit) { |
295 | info.low_limit = addr; |
296 | if (!slice_scan_available(addr, available, end: 1, boundary_addr: &addr)) |
297 | continue; |
298 | |
299 | next_slice: |
300 | /* |
301 | * At this point [info.low_limit; addr) covers |
302 | * available slices only and ends at a slice boundary. |
303 | * Check if we need to reduce the range, or if we can |
304 | * extend it to cover the next available slice. |
305 | */ |
306 | if (addr >= high_limit) |
307 | addr = high_limit; |
308 | else if (slice_scan_available(addr, available, end: 1, boundary_addr: &next_end)) { |
309 | addr = next_end; |
310 | goto next_slice; |
311 | } |
312 | info.high_limit = addr; |
313 | |
314 | found = vm_unmapped_area(info: &info); |
315 | if (!(found & ~PAGE_MASK)) |
316 | return found; |
317 | } |
318 | |
319 | return -ENOMEM; |
320 | } |
321 | |
322 | static unsigned long slice_find_area_topdown(struct mm_struct *mm, |
323 | unsigned long addr, unsigned long len, |
324 | const struct slice_mask *available, |
325 | int psize, unsigned long high_limit) |
326 | { |
327 | int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); |
328 | unsigned long found, prev; |
329 | struct vm_unmapped_area_info info; |
330 | unsigned long min_addr = max(PAGE_SIZE, mmap_min_addr); |
331 | |
332 | info.flags = VM_UNMAPPED_AREA_TOPDOWN; |
333 | info.length = len; |
334 | info.align_mask = PAGE_MASK & ((1ul << pshift) - 1); |
335 | info.align_offset = 0; |
336 | /* |
337 | * If we are trying to allocate above DEFAULT_MAP_WINDOW |
338 | * Add the different to the mmap_base. |
339 | * Only for that request for which high_limit is above |
340 | * DEFAULT_MAP_WINDOW we should apply this. |
341 | */ |
342 | if (high_limit > DEFAULT_MAP_WINDOW) |
343 | addr += mm_ctx_slb_addr_limit(&mm->context) - DEFAULT_MAP_WINDOW; |
344 | |
345 | while (addr > min_addr) { |
346 | info.high_limit = addr; |
347 | if (!slice_scan_available(addr: addr - 1, available, end: 0, boundary_addr: &addr)) |
348 | continue; |
349 | |
350 | prev_slice: |
351 | /* |
352 | * At this point [addr; info.high_limit) covers |
353 | * available slices only and starts at a slice boundary. |
354 | * Check if we need to reduce the range, or if we can |
355 | * extend it to cover the previous available slice. |
356 | */ |
357 | if (addr < min_addr) |
358 | addr = min_addr; |
359 | else if (slice_scan_available(addr: addr - 1, available, end: 0, boundary_addr: &prev)) { |
360 | addr = prev; |
361 | goto prev_slice; |
362 | } |
363 | info.low_limit = addr; |
364 | |
365 | found = vm_unmapped_area(info: &info); |
366 | if (!(found & ~PAGE_MASK)) |
367 | return found; |
368 | } |
369 | |
370 | /* |
371 | * A failed mmap() very likely causes application failure, |
372 | * so fall back to the bottom-up function here. This scenario |
373 | * can happen with large stack limits and large mmap() |
374 | * allocations. |
375 | */ |
376 | return slice_find_area_bottomup(mm, TASK_UNMAPPED_BASE, len, available, psize, high_limit); |
377 | } |
378 | |
379 | |
380 | static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len, |
381 | const struct slice_mask *mask, int psize, |
382 | int topdown, unsigned long high_limit) |
383 | { |
384 | if (topdown) |
385 | return slice_find_area_topdown(mm, addr: mm->mmap_base, len, available: mask, psize, high_limit); |
386 | else |
387 | return slice_find_area_bottomup(mm, addr: mm->mmap_base, len, available: mask, psize, high_limit); |
388 | } |
389 | |
390 | static inline void slice_copy_mask(struct slice_mask *dst, |
391 | const struct slice_mask *src) |
392 | { |
393 | dst->low_slices = src->low_slices; |
394 | if (!SLICE_NUM_HIGH) |
395 | return; |
396 | bitmap_copy(dst: dst->high_slices, src: src->high_slices, nbits: SLICE_NUM_HIGH); |
397 | } |
398 | |
399 | static inline void slice_or_mask(struct slice_mask *dst, |
400 | const struct slice_mask *src1, |
401 | const struct slice_mask *src2) |
402 | { |
403 | dst->low_slices = src1->low_slices | src2->low_slices; |
404 | if (!SLICE_NUM_HIGH) |
405 | return; |
406 | bitmap_or(dst: dst->high_slices, src1: src1->high_slices, src2: src2->high_slices, nbits: SLICE_NUM_HIGH); |
407 | } |
408 | |
409 | static inline void slice_andnot_mask(struct slice_mask *dst, |
410 | const struct slice_mask *src1, |
411 | const struct slice_mask *src2) |
412 | { |
413 | dst->low_slices = src1->low_slices & ~src2->low_slices; |
414 | if (!SLICE_NUM_HIGH) |
415 | return; |
416 | bitmap_andnot(dst: dst->high_slices, src1: src1->high_slices, src2: src2->high_slices, nbits: SLICE_NUM_HIGH); |
417 | } |
418 | |
419 | #ifdef CONFIG_PPC_64K_PAGES |
420 | #define MMU_PAGE_BASE MMU_PAGE_64K |
421 | #else |
422 | #define MMU_PAGE_BASE MMU_PAGE_4K |
423 | #endif |
424 | |
425 | unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, |
426 | unsigned long flags, unsigned int psize, |
427 | int topdown) |
428 | { |
429 | struct slice_mask good_mask; |
430 | struct slice_mask potential_mask; |
431 | const struct slice_mask *maskp; |
432 | const struct slice_mask *compat_maskp = NULL; |
433 | int fixed = (flags & MAP_FIXED); |
434 | int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); |
435 | unsigned long page_size = 1UL << pshift; |
436 | struct mm_struct *mm = current->mm; |
437 | unsigned long newaddr; |
438 | unsigned long high_limit; |
439 | |
440 | high_limit = DEFAULT_MAP_WINDOW; |
441 | if (addr >= high_limit || (fixed && (addr + len > high_limit))) |
442 | high_limit = TASK_SIZE; |
443 | |
444 | if (len > high_limit) |
445 | return -ENOMEM; |
446 | if (len & (page_size - 1)) |
447 | return -EINVAL; |
448 | if (fixed) { |
449 | if (addr & (page_size - 1)) |
450 | return -EINVAL; |
451 | if (addr > high_limit - len) |
452 | return -ENOMEM; |
453 | } |
454 | |
455 | if (high_limit > mm_ctx_slb_addr_limit(&mm->context)) { |
456 | /* |
457 | * Increasing the slb_addr_limit does not require |
458 | * slice mask cache to be recalculated because it should |
459 | * be already initialised beyond the old address limit. |
460 | */ |
461 | mm_ctx_set_slb_addr_limit(&mm->context, high_limit); |
462 | |
463 | on_each_cpu(func: slice_flush_segments, info: mm, wait: 1); |
464 | } |
465 | |
466 | /* Sanity checks */ |
467 | BUG_ON(mm->task_size == 0); |
468 | BUG_ON(mm_ctx_slb_addr_limit(&mm->context) == 0); |
469 | VM_BUG_ON(radix_enabled()); |
470 | |
471 | slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n" , mm, psize); |
472 | slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d\n" , |
473 | addr, len, flags, topdown); |
474 | |
475 | /* If hint, make sure it matches our alignment restrictions */ |
476 | if (!fixed && addr) { |
477 | addr = ALIGN(addr, page_size); |
478 | slice_dbg(" aligned addr=%lx\n" , addr); |
479 | /* Ignore hint if it's too large or overlaps a VMA */ |
480 | if (addr > high_limit - len || addr < mmap_min_addr || |
481 | !slice_area_is_free(mm, addr, len)) |
482 | addr = 0; |
483 | } |
484 | |
485 | /* First make up a "good" mask of slices that have the right size |
486 | * already |
487 | */ |
488 | maskp = slice_mask_for_size(&mm->context, psize); |
489 | |
490 | /* |
491 | * Here "good" means slices that are already the right page size, |
492 | * "compat" means slices that have a compatible page size (i.e. |
493 | * 4k in a 64k pagesize kernel), and "free" means slices without |
494 | * any VMAs. |
495 | * |
496 | * If MAP_FIXED: |
497 | * check if fits in good | compat => OK |
498 | * check if fits in good | compat | free => convert free |
499 | * else bad |
500 | * If have hint: |
501 | * check if hint fits in good => OK |
502 | * check if hint fits in good | free => convert free |
503 | * Otherwise: |
504 | * search in good, found => OK |
505 | * search in good | free, found => convert free |
506 | * search in good | compat | free, found => convert free. |
507 | */ |
508 | |
509 | /* |
510 | * If we support combo pages, we can allow 64k pages in 4k slices |
511 | * The mask copies could be avoided in most cases here if we had |
512 | * a pointer to good mask for the next code to use. |
513 | */ |
514 | if (IS_ENABLED(CONFIG_PPC_64K_PAGES) && psize == MMU_PAGE_64K) { |
515 | compat_maskp = slice_mask_for_size(&mm->context, MMU_PAGE_4K); |
516 | if (fixed) |
517 | slice_or_mask(dst: &good_mask, src1: maskp, src2: compat_maskp); |
518 | else |
519 | slice_copy_mask(dst: &good_mask, src: maskp); |
520 | } else { |
521 | slice_copy_mask(dst: &good_mask, src: maskp); |
522 | } |
523 | |
524 | slice_print_mask(label: " good_mask" , mask: &good_mask); |
525 | if (compat_maskp) |
526 | slice_print_mask(label: " compat_mask" , mask: compat_maskp); |
527 | |
528 | /* First check hint if it's valid or if we have MAP_FIXED */ |
529 | if (addr != 0 || fixed) { |
530 | /* Check if we fit in the good mask. If we do, we just return, |
531 | * nothing else to do |
532 | */ |
533 | if (slice_check_range_fits(mm, available: &good_mask, start: addr, len)) { |
534 | slice_dbg(" fits good !\n" ); |
535 | newaddr = addr; |
536 | goto return_addr; |
537 | } |
538 | } else { |
539 | /* Now let's see if we can find something in the existing |
540 | * slices for that size |
541 | */ |
542 | newaddr = slice_find_area(mm, len, mask: &good_mask, |
543 | psize, topdown, high_limit); |
544 | if (newaddr != -ENOMEM) { |
545 | /* Found within the good mask, we don't have to setup, |
546 | * we thus return directly |
547 | */ |
548 | slice_dbg(" found area at 0x%lx\n" , newaddr); |
549 | goto return_addr; |
550 | } |
551 | } |
552 | /* |
553 | * We don't fit in the good mask, check what other slices are |
554 | * empty and thus can be converted |
555 | */ |
556 | slice_mask_for_free(mm, ret: &potential_mask, high_limit); |
557 | slice_or_mask(dst: &potential_mask, src1: &potential_mask, src2: &good_mask); |
558 | slice_print_mask(label: " potential" , mask: &potential_mask); |
559 | |
560 | if (addr != 0 || fixed) { |
561 | if (slice_check_range_fits(mm, available: &potential_mask, start: addr, len)) { |
562 | slice_dbg(" fits potential !\n" ); |
563 | newaddr = addr; |
564 | goto convert; |
565 | } |
566 | } |
567 | |
568 | /* If we have MAP_FIXED and failed the above steps, then error out */ |
569 | if (fixed) |
570 | return -EBUSY; |
571 | |
572 | slice_dbg(" search...\n" ); |
573 | |
574 | /* If we had a hint that didn't work out, see if we can fit |
575 | * anywhere in the good area. |
576 | */ |
577 | if (addr) { |
578 | newaddr = slice_find_area(mm, len, mask: &good_mask, |
579 | psize, topdown, high_limit); |
580 | if (newaddr != -ENOMEM) { |
581 | slice_dbg(" found area at 0x%lx\n" , newaddr); |
582 | goto return_addr; |
583 | } |
584 | } |
585 | |
586 | /* Now let's see if we can find something in the existing slices |
587 | * for that size plus free slices |
588 | */ |
589 | newaddr = slice_find_area(mm, len, mask: &potential_mask, |
590 | psize, topdown, high_limit); |
591 | |
592 | if (IS_ENABLED(CONFIG_PPC_64K_PAGES) && newaddr == -ENOMEM && |
593 | psize == MMU_PAGE_64K) { |
594 | /* retry the search with 4k-page slices included */ |
595 | slice_or_mask(dst: &potential_mask, src1: &potential_mask, src2: compat_maskp); |
596 | newaddr = slice_find_area(mm, len, mask: &potential_mask, |
597 | psize, topdown, high_limit); |
598 | } |
599 | |
600 | if (newaddr == -ENOMEM) |
601 | return -ENOMEM; |
602 | |
603 | slice_range_to_mask(start: newaddr, len, ret: &potential_mask); |
604 | slice_dbg(" found potential area at 0x%lx\n" , newaddr); |
605 | slice_print_mask(label: " mask" , mask: &potential_mask); |
606 | |
607 | convert: |
608 | /* |
609 | * Try to allocate the context before we do slice convert |
610 | * so that we handle the context allocation failure gracefully. |
611 | */ |
612 | if (need_extra_context(mm, newaddr)) { |
613 | if (alloc_extended_context(mm, newaddr) < 0) |
614 | return -ENOMEM; |
615 | } |
616 | |
617 | slice_andnot_mask(dst: &potential_mask, src1: &potential_mask, src2: &good_mask); |
618 | if (compat_maskp && !fixed) |
619 | slice_andnot_mask(dst: &potential_mask, src1: &potential_mask, src2: compat_maskp); |
620 | if (potential_mask.low_slices || |
621 | (SLICE_NUM_HIGH && |
622 | !bitmap_empty(potential_mask.high_slices, SLICE_NUM_HIGH))) { |
623 | slice_convert(mm, mask: &potential_mask, psize); |
624 | if (psize > MMU_PAGE_BASE) |
625 | on_each_cpu(func: slice_flush_segments, info: mm, wait: 1); |
626 | } |
627 | return newaddr; |
628 | |
629 | return_addr: |
630 | if (need_extra_context(mm, newaddr)) { |
631 | if (alloc_extended_context(mm, newaddr) < 0) |
632 | return -ENOMEM; |
633 | } |
634 | return newaddr; |
635 | } |
636 | EXPORT_SYMBOL_GPL(slice_get_unmapped_area); |
637 | |
638 | unsigned long arch_get_unmapped_area(struct file *filp, |
639 | unsigned long addr, |
640 | unsigned long len, |
641 | unsigned long pgoff, |
642 | unsigned long flags) |
643 | { |
644 | if (radix_enabled()) |
645 | return generic_get_unmapped_area(filp, addr, len, pgoff, flags); |
646 | |
647 | return slice_get_unmapped_area(addr, len, flags, |
648 | mm_ctx_user_psize(¤t->mm->context), 0); |
649 | } |
650 | |
651 | unsigned long arch_get_unmapped_area_topdown(struct file *filp, |
652 | const unsigned long addr0, |
653 | const unsigned long len, |
654 | const unsigned long pgoff, |
655 | const unsigned long flags) |
656 | { |
657 | if (radix_enabled()) |
658 | return generic_get_unmapped_area_topdown(filp, addr: addr0, len, pgoff, flags); |
659 | |
660 | return slice_get_unmapped_area(addr0, len, flags, |
661 | mm_ctx_user_psize(¤t->mm->context), 1); |
662 | } |
663 | |
664 | unsigned int notrace get_slice_psize(struct mm_struct *mm, unsigned long addr) |
665 | { |
666 | unsigned char *psizes; |
667 | int index, mask_index; |
668 | |
669 | VM_BUG_ON(radix_enabled()); |
670 | |
671 | if (slice_addr_is_low(addr)) { |
672 | psizes = mm_ctx_low_slices(&mm->context); |
673 | index = GET_LOW_SLICE_INDEX(addr); |
674 | } else { |
675 | psizes = mm_ctx_high_slices(&mm->context); |
676 | index = GET_HIGH_SLICE_INDEX(addr); |
677 | } |
678 | mask_index = index & 0x1; |
679 | return (psizes[index >> 1] >> (mask_index * 4)) & 0xf; |
680 | } |
681 | EXPORT_SYMBOL_GPL(get_slice_psize); |
682 | |
683 | void slice_init_new_context_exec(struct mm_struct *mm) |
684 | { |
685 | unsigned char *hpsizes, *lpsizes; |
686 | struct slice_mask *mask; |
687 | unsigned int psize = mmu_virtual_psize; |
688 | |
689 | slice_dbg("slice_init_new_context_exec(mm=%p)\n" , mm); |
690 | |
691 | /* |
692 | * In the case of exec, use the default limit. In the |
693 | * case of fork it is just inherited from the mm being |
694 | * duplicated. |
695 | */ |
696 | mm_ctx_set_slb_addr_limit(&mm->context, SLB_ADDR_LIMIT_DEFAULT); |
697 | mm_ctx_set_user_psize(&mm->context, psize); |
698 | |
699 | /* |
700 | * Set all slice psizes to the default. |
701 | */ |
702 | lpsizes = mm_ctx_low_slices(&mm->context); |
703 | memset(lpsizes, (psize << 4) | psize, SLICE_NUM_LOW >> 1); |
704 | |
705 | hpsizes = mm_ctx_high_slices(&mm->context); |
706 | memset(hpsizes, (psize << 4) | psize, SLICE_NUM_HIGH >> 1); |
707 | |
708 | /* |
709 | * Slice mask cache starts zeroed, fill the default size cache. |
710 | */ |
711 | mask = slice_mask_for_size(&mm->context, psize); |
712 | mask->low_slices = ~0UL; |
713 | if (SLICE_NUM_HIGH) |
714 | bitmap_fill(mask->high_slices, SLICE_NUM_HIGH); |
715 | } |
716 | |
717 | void slice_setup_new_exec(void) |
718 | { |
719 | struct mm_struct *mm = current->mm; |
720 | |
721 | slice_dbg("slice_setup_new_exec(mm=%p)\n" , mm); |
722 | |
723 | if (!is_32bit_task()) |
724 | return; |
725 | |
726 | mm_ctx_set_slb_addr_limit(&mm->context, DEFAULT_MAP_WINDOW); |
727 | } |
728 | |
729 | void slice_set_range_psize(struct mm_struct *mm, unsigned long start, |
730 | unsigned long len, unsigned int psize) |
731 | { |
732 | struct slice_mask mask; |
733 | |
734 | VM_BUG_ON(radix_enabled()); |
735 | |
736 | slice_range_to_mask(start, len, ret: &mask); |
737 | slice_convert(mm, mask: &mask, psize); |
738 | } |
739 | |
740 | #ifdef CONFIG_HUGETLB_PAGE |
741 | /* |
742 | * is_hugepage_only_range() is used by generic code to verify whether |
743 | * a normal mmap mapping (non hugetlbfs) is valid on a given area. |
744 | * |
745 | * until the generic code provides a more generic hook and/or starts |
746 | * calling arch get_unmapped_area for MAP_FIXED (which our implementation |
747 | * here knows how to deal with), we hijack it to keep standard mappings |
748 | * away from us. |
749 | * |
750 | * because of that generic code limitation, MAP_FIXED mapping cannot |
751 | * "convert" back a slice with no VMAs to the standard page size, only |
752 | * get_unmapped_area() can. It would be possible to fix it here but I |
753 | * prefer working on fixing the generic code instead. |
754 | * |
755 | * WARNING: This will not work if hugetlbfs isn't enabled since the |
756 | * generic code will redefine that function as 0 in that. This is ok |
757 | * for now as we only use slices with hugetlbfs enabled. This should |
758 | * be fixed as the generic code gets fixed. |
759 | */ |
760 | int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, |
761 | unsigned long len) |
762 | { |
763 | const struct slice_mask *maskp; |
764 | unsigned int psize = mm_ctx_user_psize(&mm->context); |
765 | |
766 | VM_BUG_ON(radix_enabled()); |
767 | |
768 | maskp = slice_mask_for_size(&mm->context, psize); |
769 | |
770 | /* We need to account for 4k slices too */ |
771 | if (IS_ENABLED(CONFIG_PPC_64K_PAGES) && psize == MMU_PAGE_64K) { |
772 | const struct slice_mask *compat_maskp; |
773 | struct slice_mask available; |
774 | |
775 | compat_maskp = slice_mask_for_size(&mm->context, MMU_PAGE_4K); |
776 | slice_or_mask(dst: &available, src1: maskp, src2: compat_maskp); |
777 | return !slice_check_range_fits(mm, available: &available, start: addr, len); |
778 | } |
779 | |
780 | return !slice_check_range_fits(mm, available: maskp, start: addr, len); |
781 | } |
782 | |
783 | unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) |
784 | { |
785 | /* With radix we don't use slice, so derive it from vma*/ |
786 | if (radix_enabled()) |
787 | return vma_kernel_pagesize(vma); |
788 | |
789 | return 1UL << mmu_psize_to_shift(get_slice_psize(vma->vm_mm, vma->vm_start)); |
790 | } |
791 | |
792 | static int file_to_psize(struct file *file) |
793 | { |
794 | struct hstate *hstate = hstate_file(f: file); |
795 | return shift_to_mmu_psize(huge_page_shift(h: hstate)); |
796 | } |
797 | |
798 | unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, |
799 | unsigned long len, unsigned long pgoff, |
800 | unsigned long flags) |
801 | { |
802 | if (radix_enabled()) |
803 | return generic_hugetlb_get_unmapped_area(file, addr, len, pgoff, flags); |
804 | |
805 | return slice_get_unmapped_area(addr, len, flags, file_to_psize(file), 1); |
806 | } |
807 | #endif |
808 | |