1// SPDX-License-Identifier: GPL-2.0
2#define boot_fmt(fmt) "vmem: " fmt
3#include <linux/cpufeature.h>
4#include <linux/sched/task.h>
5#include <linux/pgtable.h>
6#include <linux/kasan.h>
7#include <asm/page-states.h>
8#include <asm/pgalloc.h>
9#include <asm/facility.h>
10#include <asm/sections.h>
11#include <asm/ctlreg.h>
12#include <asm/physmem_info.h>
13#include <asm/maccess.h>
14#include <asm/machine.h>
15#include <asm/abs_lowcore.h>
16#include "decompressor.h"
17#include "boot.h"
18
19struct ctlreg __bootdata_preserved(s390_invalid_asce);
20
21#ifdef CONFIG_PROC_FS
22atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]);
23#endif
24
25#define init_mm (*(struct mm_struct *)vmlinux.init_mm_off)
26#define swapper_pg_dir vmlinux.swapper_pg_dir_off
27#define invalid_pg_dir vmlinux.invalid_pg_dir_off
28
29enum populate_mode {
30 POPULATE_NONE,
31 POPULATE_DIRECT,
32 POPULATE_LOWCORE,
33 POPULATE_ABS_LOWCORE,
34 POPULATE_IDENTITY,
35 POPULATE_KERNEL,
36#ifdef CONFIG_KASAN
37 /* KASAN modes should be last and grouped together, see is_kasan_populate_mode() */
38 POPULATE_KASAN_MAP_SHADOW,
39 POPULATE_KASAN_ZERO_SHADOW,
40 POPULATE_KASAN_SHALLOW
41#endif
42};
43
44#define POPULATE_MODE_NAME(t) case POPULATE_ ## t: return #t
45static inline const char *get_populate_mode_name(enum populate_mode t)
46{
47 switch (t) {
48 POPULATE_MODE_NAME(NONE);
49 POPULATE_MODE_NAME(DIRECT);
50 POPULATE_MODE_NAME(LOWCORE);
51 POPULATE_MODE_NAME(ABS_LOWCORE);
52 POPULATE_MODE_NAME(IDENTITY);
53 POPULATE_MODE_NAME(KERNEL);
54#ifdef CONFIG_KASAN
55 POPULATE_MODE_NAME(KASAN_MAP_SHADOW);
56 POPULATE_MODE_NAME(KASAN_ZERO_SHADOW);
57 POPULATE_MODE_NAME(KASAN_SHALLOW);
58#endif
59 default:
60 return "UNKNOWN";
61 }
62}
63
64static bool is_kasan_populate_mode(enum populate_mode mode)
65{
66#ifdef CONFIG_KASAN
67 return mode >= POPULATE_KASAN_MAP_SHADOW;
68#else
69 return false;
70#endif
71}
72
73static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
74
75#ifdef CONFIG_KASAN
76
77#define kasan_early_shadow_page vmlinux.kasan_early_shadow_page_off
78#define kasan_early_shadow_pte ((pte_t *)vmlinux.kasan_early_shadow_pte_off)
79#define kasan_early_shadow_pmd ((pmd_t *)vmlinux.kasan_early_shadow_pmd_off)
80#define kasan_early_shadow_pud ((pud_t *)vmlinux.kasan_early_shadow_pud_off)
81#define kasan_early_shadow_p4d ((p4d_t *)vmlinux.kasan_early_shadow_p4d_off)
82#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
83
84static pte_t pte_z;
85
86static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode)
87{
88 unsigned long sha_start = PAGE_ALIGN_DOWN(__sha(start));
89 unsigned long sha_end = PAGE_ALIGN(__sha(end));
90
91 boot_debug("%-17s 0x%016lx-0x%016lx >> 0x%016lx-0x%016lx\n", get_populate_mode_name(mode),
92 start, end, sha_start, sha_end);
93 pgtable_populate(addr: sha_start, end: sha_end, mode);
94}
95
96static void kasan_populate_shadow(unsigned long kernel_start, unsigned long kernel_end)
97{
98 pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
99 pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
100 p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
101 unsigned long memgap_start = 0;
102 unsigned long start, end;
103 int i;
104
105 pte_z = __pte(__pa(kasan_early_shadow_page) | pgprot_val(PAGE_KERNEL_RO));
106 crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d: p4d_z));
107 crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud: pud_z));
108 crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd: pmd_z));
109 memset64(s: (u64 *)kasan_early_shadow_pte, v: pte_val(pte: pte_z), PTRS_PER_PTE);
110 __arch_set_page_dat(kasan_early_shadow_p4d, 1UL << CRST_ALLOC_ORDER);
111 __arch_set_page_dat(kasan_early_shadow_pud, 1UL << CRST_ALLOC_ORDER);
112 __arch_set_page_dat(kasan_early_shadow_pmd, 1UL << CRST_ALLOC_ORDER);
113 __arch_set_page_dat(kasan_early_shadow_pte, 1);
114
115 for_each_physmem_usable_range(i, &start, &end) {
116 kasan_populate((unsigned long)__identity_va(start),
117 (unsigned long)__identity_va(end),
118 POPULATE_KASAN_MAP_SHADOW);
119 if (memgap_start && physmem_info.info_source == MEM_DETECT_DIAG260) {
120 kasan_populate((unsigned long)__identity_va(memgap_start),
121 (unsigned long)__identity_va(start),
122 POPULATE_KASAN_ZERO_SHADOW);
123 }
124 memgap_start = end;
125 }
126 kasan_populate(start: kernel_start + TEXT_OFFSET, end: kernel_end, mode: POPULATE_KASAN_MAP_SHADOW);
127 kasan_populate(0, (unsigned long)__identity_va(0), POPULATE_KASAN_ZERO_SHADOW);
128 kasan_populate(start: AMODE31_START, end: AMODE31_END, mode: POPULATE_KASAN_ZERO_SHADOW);
129 /* shallowly populate kasan shadow for vmalloc and modules */
130 kasan_populate(VMALLOC_START, MODULES_END, mode: POPULATE_KASAN_SHALLOW);
131 /* populate kasan shadow for untracked memory */
132 kasan_populate((unsigned long)__identity_va(ident_map_size), VMALLOC_START,
133 POPULATE_KASAN_ZERO_SHADOW);
134 kasan_populate(start: kernel_end, end: _REGION1_SIZE, mode: POPULATE_KASAN_ZERO_SHADOW);
135}
136
137static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,
138 unsigned long end, enum populate_mode mode)
139{
140 if (mode == POPULATE_KASAN_ZERO_SHADOW &&
141 IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
142 pgd_populate(mm: &init_mm, pgd, kasan_early_shadow_p4d);
143 return true;
144 }
145 return false;
146}
147
148static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,
149 unsigned long end, enum populate_mode mode)
150{
151 if (mode == POPULATE_KASAN_ZERO_SHADOW &&
152 IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
153 p4d_populate(mm: &init_mm, p4d, kasan_early_shadow_pud);
154 return true;
155 }
156 return false;
157}
158
159static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,
160 unsigned long end, enum populate_mode mode)
161{
162 if (mode == POPULATE_KASAN_ZERO_SHADOW &&
163 IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
164 pud_populate(mm: &init_mm, pud, kasan_early_shadow_pmd);
165 return true;
166 }
167 return false;
168}
169
170static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,
171 unsigned long end, enum populate_mode mode)
172{
173 if (mode == POPULATE_KASAN_ZERO_SHADOW &&
174 IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
175 pmd_populate(mm: &init_mm, pmd, kasan_early_shadow_pte);
176 return true;
177 }
178 return false;
179}
180
181static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
182{
183 if (mode == POPULATE_KASAN_ZERO_SHADOW) {
184 set_pte(ptep: pte, pte: pte_z);
185 return true;
186 }
187 return false;
188}
189#else
190
191static inline void kasan_populate_shadow(unsigned long kernel_start, unsigned long kernel_end)
192{
193}
194
195static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,
196 unsigned long end, enum populate_mode mode)
197{
198 return false;
199}
200
201static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,
202 unsigned long end, enum populate_mode mode)
203{
204 return false;
205}
206
207static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,
208 unsigned long end, enum populate_mode mode)
209{
210 return false;
211}
212
213static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,
214 unsigned long end, enum populate_mode mode)
215{
216 return false;
217}
218
219static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
220{
221 return false;
222}
223
224#endif
225
226/*
227 * Mimic virt_to_kpte() in lack of init_mm symbol. Skip pmd NULL check though.
228 */
229static inline pte_t *__virt_to_kpte(unsigned long va)
230{
231 return pte_offset_kernel(pmd_offset(pud_offset(p4d: p4d_offset(pgd_offset_k(va), address: va), address: va), address: va), address: va);
232}
233
234static void *boot_crst_alloc(unsigned long val)
235{
236 unsigned long size = PAGE_SIZE << CRST_ALLOC_ORDER;
237 unsigned long *table;
238
239 table = (unsigned long *)physmem_alloc_or_die(RR_VMEM, size, size);
240 crst_table_init(table, val);
241 __arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER);
242 return table;
243}
244
245static pte_t *boot_pte_alloc(void)
246{
247 pte_t *pte;
248
249 pte = (void *)physmem_alloc_or_die(RR_VMEM, PAGE_SIZE, PAGE_SIZE);
250 __arch_set_page_dat(pte, 1);
251 memset64(s: (u64 *)pte, v: _PAGE_INVALID, PTRS_PER_PTE);
252 return pte;
253}
254
255static unsigned long resolve_pa_may_alloc(unsigned long addr, unsigned long size,
256 enum populate_mode mode)
257{
258 switch (mode) {
259 case POPULATE_NONE:
260 return INVALID_PHYS_ADDR;
261 case POPULATE_DIRECT:
262 return addr;
263 case POPULATE_LOWCORE:
264 return __lowcore_pa(addr);
265 case POPULATE_ABS_LOWCORE:
266 return __abs_lowcore_pa(addr);
267 case POPULATE_KERNEL:
268 return __kernel_pa(addr);
269 case POPULATE_IDENTITY:
270 return __identity_pa(addr);
271#ifdef CONFIG_KASAN
272 case POPULATE_KASAN_MAP_SHADOW:
273 /* Allow to fail large page allocations, this will fall back to 1mb/4k pages */
274 addr = physmem_alloc(RR_VMEM, size, size, size == PAGE_SIZE);
275 if (addr) {
276 memset((void *)addr, 0, size);
277 return addr;
278 }
279 return INVALID_PHYS_ADDR;
280#endif
281 default:
282 return INVALID_PHYS_ADDR;
283 }
284}
285
286static bool large_page_mapping_allowed(enum populate_mode mode)
287{
288 switch (mode) {
289 case POPULATE_DIRECT:
290 case POPULATE_IDENTITY:
291 case POPULATE_KERNEL:
292#ifdef CONFIG_KASAN
293 case POPULATE_KASAN_MAP_SHADOW:
294#endif
295 return true;
296 default:
297 return false;
298 }
299}
300
301static unsigned long try_get_large_pud_pa(pud_t *pu_dir, unsigned long addr, unsigned long end,
302 enum populate_mode mode)
303{
304 unsigned long pa, size = end - addr;
305
306 if (!cpu_has_edat2() || !large_page_mapping_allowed(mode) ||
307 !IS_ALIGNED(addr, PUD_SIZE) || (size < PUD_SIZE))
308 return INVALID_PHYS_ADDR;
309
310 pa = resolve_pa_may_alloc(addr, size, mode);
311 if (!IS_ALIGNED(pa, PUD_SIZE))
312 return INVALID_PHYS_ADDR;
313
314 return pa;
315}
316
317static unsigned long try_get_large_pmd_pa(pmd_t *pm_dir, unsigned long addr, unsigned long end,
318 enum populate_mode mode)
319{
320 unsigned long pa, size = end - addr;
321
322 if (!cpu_has_edat1() || !large_page_mapping_allowed(mode) ||
323 !IS_ALIGNED(addr, PMD_SIZE) || (size < PMD_SIZE))
324 return INVALID_PHYS_ADDR;
325
326 pa = resolve_pa_may_alloc(addr, size, mode);
327 if (!IS_ALIGNED(pa, PMD_SIZE))
328 return INVALID_PHYS_ADDR;
329
330 return pa;
331}
332
333static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end,
334 enum populate_mode mode)
335{
336 unsigned long pages = 0;
337 pte_t *pte, entry;
338
339 pte = pte_offset_kernel(pmd, address: addr);
340 for (; addr < end; addr += PAGE_SIZE, pte++) {
341 if (pte_none(pte: *pte)) {
342 if (kasan_pte_populate_zero_shadow(pte, mode))
343 continue;
344 entry = __pte(val: resolve_pa_may_alloc(addr, PAGE_SIZE, mode));
345 entry = set_pte_bit(entry, PAGE_KERNEL);
346 set_pte(ptep: pte, pte: entry);
347 pages++;
348 }
349 }
350 if (mode == POPULATE_IDENTITY)
351 update_page_count(PG_DIRECT_MAP_4K, pages);
352}
353
354static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end,
355 enum populate_mode mode)
356{
357 unsigned long pa, next, pages = 0;
358 pmd_t *pmd, entry;
359 pte_t *pte;
360
361 pmd = pmd_offset(pud, address: addr);
362 for (; addr < end; addr = next, pmd++) {
363 next = pmd_addr_end(addr, end);
364 if (pmd_none(pmd: *pmd)) {
365 if (kasan_pmd_populate_zero_shadow(pmd, addr, end: next, mode))
366 continue;
367 pa = try_get_large_pmd_pa(pm_dir: pmd, addr, end: next, mode);
368 if (pa != INVALID_PHYS_ADDR) {
369 entry = __pmd(val: pa);
370 entry = set_pmd_bit(entry, SEGMENT_KERNEL);
371 set_pmd(pmdp: pmd, pmd: entry);
372 pages++;
373 continue;
374 }
375 pte = boot_pte_alloc();
376 pmd_populate(mm: &init_mm, pmd, pte);
377 } else if (pmd_leaf(pte: *pmd)) {
378 continue;
379 }
380 pgtable_pte_populate(pmd, addr, end: next, mode);
381 }
382 if (mode == POPULATE_IDENTITY)
383 update_page_count(PG_DIRECT_MAP_1M, pages);
384}
385
386static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end,
387 enum populate_mode mode)
388{
389 unsigned long pa, next, pages = 0;
390 pud_t *pud, entry;
391 pmd_t *pmd;
392
393 pud = pud_offset(p4d, address: addr);
394 for (; addr < end; addr = next, pud++) {
395 next = pud_addr_end(addr, end);
396 if (pud_none(pud: *pud)) {
397 if (kasan_pud_populate_zero_shadow(pud, addr, end: next, mode))
398 continue;
399 pa = try_get_large_pud_pa(pu_dir: pud, addr, end: next, mode);
400 if (pa != INVALID_PHYS_ADDR) {
401 entry = __pud(val: pa);
402 entry = set_pud_bit(entry, REGION3_KERNEL);
403 set_pud(pudp: pud, pud: entry);
404 pages++;
405 continue;
406 }
407 pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY);
408 pud_populate(mm: &init_mm, pud, pmd);
409 } else if (pud_leaf(pud: *pud)) {
410 continue;
411 }
412 pgtable_pmd_populate(pud, addr, end: next, mode);
413 }
414 if (mode == POPULATE_IDENTITY)
415 update_page_count(PG_DIRECT_MAP_2G, pages);
416}
417
418static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end,
419 enum populate_mode mode)
420{
421 unsigned long next;
422 p4d_t *p4d;
423 pud_t *pud;
424
425 p4d = p4d_offset(pgd, address: addr);
426 for (; addr < end; addr = next, p4d++) {
427 next = p4d_addr_end(addr, end);
428 if (p4d_none(p4d: *p4d)) {
429 if (kasan_p4d_populate_zero_shadow(p4d, addr, end: next, mode))
430 continue;
431 pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY);
432 p4d_populate(mm: &init_mm, p4d, pud);
433 }
434 pgtable_pud_populate(p4d, addr, end: next, mode);
435 }
436}
437
438static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode)
439{
440 unsigned long next;
441 pgd_t *pgd;
442 p4d_t *p4d;
443
444 if (!is_kasan_populate_mode(mode)) {
445 boot_debug("%-17s 0x%016lx-0x%016lx -> 0x%016lx-0x%016lx\n",
446 get_populate_mode_name(mode), addr, end,
447 resolve_pa_may_alloc(addr, 0, mode),
448 resolve_pa_may_alloc(end - 1, 0, mode) + 1);
449 }
450
451 pgd = pgd_offset(&init_mm, addr);
452 for (; addr < end; addr = next, pgd++) {
453 next = pgd_addr_end(addr, end);
454 if (pgd_none(pgd: *pgd)) {
455 if (kasan_pgd_populate_zero_shadow(pgd, addr, end: next, mode))
456 continue;
457 p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY);
458 pgd_populate(mm: &init_mm, pgd, p4d);
459 }
460#ifdef CONFIG_KASAN
461 if (mode == POPULATE_KASAN_SHALLOW)
462 continue;
463#endif
464 pgtable_p4d_populate(pgd, addr, end: next, mode);
465 }
466}
467
468void setup_vmem(unsigned long kernel_start, unsigned long kernel_end, unsigned long asce_limit)
469{
470 unsigned long lowcore_address = 0;
471 unsigned long start, end;
472 unsigned long asce_type;
473 unsigned long asce_bits;
474 pgd_t *init_mm_pgd;
475 int i;
476
477 /*
478 * Mark whole memory as no-dat. This must be done before any
479 * page tables are allocated, or kernel image builtin pages
480 * are marked as dat tables.
481 */
482 for_each_physmem_online_range(i, &start, &end)
483 __arch_set_page_nodat((void *)start, (end - start) >> PAGE_SHIFT);
484
485 /*
486 * init_mm->pgd contains virtual address of swapper_pg_dir.
487 * It is unusable at this stage since DAT is yet off. Swap
488 * it for physical address of swapper_pg_dir and restore
489 * the virtual address after all page tables are created.
490 */
491 init_mm_pgd = init_mm.pgd;
492 init_mm.pgd = (pgd_t *)swapper_pg_dir;
493
494 if (asce_limit == _REGION1_SIZE) {
495 asce_type = _REGION2_ENTRY_EMPTY;
496 asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;
497 } else {
498 asce_type = _REGION3_ENTRY_EMPTY;
499 asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;
500 }
501 s390_invalid_asce.val = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;
502
503 crst_table_init((unsigned long *)swapper_pg_dir, asce_type);
504 crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY);
505 __arch_set_page_dat((void *)swapper_pg_dir, 1UL << CRST_ALLOC_ORDER);
506 __arch_set_page_dat((void *)invalid_pg_dir, 1UL << CRST_ALLOC_ORDER);
507
508 if (machine_has_relocated_lowcore())
509 lowcore_address = LOWCORE_ALT_ADDRESS;
510
511 /*
512 * To allow prefixing the lowcore must be mapped with 4KB pages.
513 * To prevent creation of a large page at address 0 first map
514 * the lowcore and create the identity mapping only afterwards.
515 */
516 pgtable_populate(lowcore_address,
517 lowcore_address + sizeof(struct lowcore),
518 POPULATE_LOWCORE);
519 for_each_physmem_usable_range(i, &start, &end) {
520 /* Do not map lowcore with identity mapping */
521 if (!start)
522 start = sizeof(struct lowcore);
523 pgtable_populate((unsigned long)__identity_va(start),
524 (unsigned long)__identity_va(end),
525 POPULATE_IDENTITY);
526 }
527
528 /*
529 * [kernel_start..kernel_start + TEXT_OFFSET] region is never
530 * accessed as per the linker script:
531 *
532 * . = TEXT_OFFSET;
533 *
534 * Therefore, skip mapping TEXT_OFFSET bytes to prevent access to
535 * [__kaslr_offset_phys..__kaslr_offset_phys + TEXT_OFFSET] region.
536 */
537 pgtable_populate(kernel_start + TEXT_OFFSET, kernel_end, POPULATE_KERNEL);
538 pgtable_populate(AMODE31_START, AMODE31_END, POPULATE_DIRECT);
539 pgtable_populate(__abs_lowcore, __abs_lowcore + sizeof(struct lowcore),
540 POPULATE_ABS_LOWCORE);
541 pgtable_populate(__memcpy_real_area, __memcpy_real_area + PAGE_SIZE,
542 POPULATE_NONE);
543 memcpy_real_ptep = __identity_va(__virt_to_kpte(__memcpy_real_area));
544
545 kasan_populate_shadow(kernel_start, kernel_end);
546
547 get_lowcore()->kernel_asce.val = swapper_pg_dir | asce_bits;
548 get_lowcore()->user_asce = s390_invalid_asce;
549
550 local_ctl_load(1, &get_lowcore()->kernel_asce);
551 local_ctl_load(7, &get_lowcore()->user_asce);
552 local_ctl_load(13, &get_lowcore()->kernel_asce);
553
554 init_mm.context.asce = get_lowcore()->kernel_asce.val;
555 init_mm.pgd = init_mm_pgd;
556}
557

source code of linux/arch/s390/boot/vmem.c