1 | // SPDX-License-Identifier: GPL-2.0 |
2 | // Copyright (C) 2019 Andes Technology Corporation |
3 | |
4 | #include <linux/pfn.h> |
5 | #include <linux/init_task.h> |
6 | #include <linux/kasan.h> |
7 | #include <linux/kernel.h> |
8 | #include <linux/memblock.h> |
9 | #include <linux/pgtable.h> |
10 | #include <asm/tlbflush.h> |
11 | #include <asm/fixmap.h> |
12 | #include <asm/pgalloc.h> |
13 | |
14 | /* |
15 | * Kasan shadow region must lie at a fixed address across sv39, sv48 and sv57 |
16 | * which is right before the kernel. |
17 | * |
18 | * For sv39, the region is aligned on PGDIR_SIZE so we only need to populate |
19 | * the page global directory with kasan_early_shadow_pmd. |
20 | * |
21 | * For sv48 and sv57, the region start is aligned on PGDIR_SIZE whereas the end |
22 | * region is not and then we have to go down to the PUD level. |
23 | */ |
24 | |
25 | static pgd_t tmp_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
26 | static p4d_t tmp_p4d[PTRS_PER_P4D] __page_aligned_bss; |
27 | static pud_t tmp_pud[PTRS_PER_PUD] __page_aligned_bss; |
28 | |
29 | static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end) |
30 | { |
31 | phys_addr_t phys_addr; |
32 | pte_t *ptep, *p; |
33 | |
34 | if (pmd_none(pmd: pmdp_get(pmdp: pmd))) { |
35 | p = memblock_alloc(PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); |
36 | set_pmd(pmdp: pmd, pmd: pfn_pmd(PFN_DOWN(__pa(p)), pgprot: PAGE_TABLE)); |
37 | } |
38 | |
39 | ptep = pte_offset_kernel(pmd, address: vaddr); |
40 | |
41 | do { |
42 | if (pte_none(pte: ptep_get(ptep))) { |
43 | phys_addr = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); |
44 | set_pte(ptep, pte: pfn_pte(PFN_DOWN(phys_addr), PAGE_KERNEL)); |
45 | memset(__va(phys_addr), KASAN_SHADOW_INIT, PAGE_SIZE); |
46 | } |
47 | } while (ptep++, vaddr += PAGE_SIZE, vaddr != end); |
48 | } |
49 | |
50 | static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned long end) |
51 | { |
52 | phys_addr_t phys_addr; |
53 | pmd_t *pmdp, *p; |
54 | unsigned long next; |
55 | |
56 | if (pud_none(pud: pudp_get(pudp: pud))) { |
57 | p = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); |
58 | set_pud(pudp: pud, pud: pfn_pud(PFN_DOWN(__pa(p)), pgprot: PAGE_TABLE)); |
59 | } |
60 | |
61 | pmdp = pmd_offset(pud, address: vaddr); |
62 | |
63 | do { |
64 | next = pmd_addr_end(vaddr, end); |
65 | |
66 | if (pmd_none(pmd: pmdp_get(pmdp)) && IS_ALIGNED(vaddr, PMD_SIZE) && |
67 | (next - vaddr) >= PMD_SIZE) { |
68 | phys_addr = memblock_phys_alloc(PMD_SIZE, PMD_SIZE); |
69 | if (phys_addr) { |
70 | set_pmd(pmdp, pmd: pfn_pmd(PFN_DOWN(phys_addr), PAGE_KERNEL)); |
71 | memset(__va(phys_addr), KASAN_SHADOW_INIT, PMD_SIZE); |
72 | continue; |
73 | } |
74 | } |
75 | |
76 | kasan_populate_pte(pmd: pmdp, vaddr, end: next); |
77 | } while (pmdp++, vaddr = next, vaddr != end); |
78 | } |
79 | |
80 | static void __init kasan_populate_pud(p4d_t *p4d, |
81 | unsigned long vaddr, unsigned long end) |
82 | { |
83 | phys_addr_t phys_addr; |
84 | pud_t *pudp, *p; |
85 | unsigned long next; |
86 | |
87 | if (p4d_none(p4d: p4dp_get(p4dp: p4d))) { |
88 | p = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE); |
89 | set_p4d(p4dp: p4d, p4d: pfn_p4d(PFN_DOWN(__pa(p)), PAGE_TABLE)); |
90 | } |
91 | |
92 | pudp = pud_offset(p4d, address: vaddr); |
93 | |
94 | do { |
95 | next = pud_addr_end(vaddr, end); |
96 | |
97 | if (pud_none(pud: pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) && |
98 | (next - vaddr) >= PUD_SIZE) { |
99 | phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE); |
100 | if (phys_addr) { |
101 | set_pud(pudp, pud: pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL)); |
102 | memset(__va(phys_addr), KASAN_SHADOW_INIT, PUD_SIZE); |
103 | continue; |
104 | } |
105 | } |
106 | |
107 | kasan_populate_pmd(pud: pudp, vaddr, end: next); |
108 | } while (pudp++, vaddr = next, vaddr != end); |
109 | } |
110 | |
111 | static void __init kasan_populate_p4d(pgd_t *pgd, |
112 | unsigned long vaddr, unsigned long end) |
113 | { |
114 | phys_addr_t phys_addr; |
115 | p4d_t *p4dp, *p; |
116 | unsigned long next; |
117 | |
118 | if (pgd_none(pgd: pgdp_get(pgdp: pgd))) { |
119 | p = memblock_alloc(PTRS_PER_P4D * sizeof(p4d_t), PAGE_SIZE); |
120 | set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(p)), PAGE_TABLE)); |
121 | } |
122 | |
123 | p4dp = p4d_offset(pgd, address: vaddr); |
124 | |
125 | do { |
126 | next = p4d_addr_end(vaddr, end); |
127 | |
128 | if (p4d_none(p4d: p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) && |
129 | (next - vaddr) >= P4D_SIZE) { |
130 | phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE); |
131 | if (phys_addr) { |
132 | set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL)); |
133 | memset(__va(phys_addr), KASAN_SHADOW_INIT, P4D_SIZE); |
134 | continue; |
135 | } |
136 | } |
137 | |
138 | kasan_populate_pud(p4d: p4dp, vaddr, end: next); |
139 | } while (p4dp++, vaddr = next, vaddr != end); |
140 | } |
141 | |
142 | static void __init kasan_populate_pgd(pgd_t *pgdp, |
143 | unsigned long vaddr, unsigned long end) |
144 | { |
145 | phys_addr_t phys_addr; |
146 | unsigned long next; |
147 | |
148 | do { |
149 | next = pgd_addr_end(vaddr, end); |
150 | |
151 | if (pgd_none(pgd: pgdp_get(pgdp)) && IS_ALIGNED(vaddr, PGDIR_SIZE) && |
152 | (next - vaddr) >= PGDIR_SIZE) { |
153 | phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE); |
154 | if (phys_addr) { |
155 | set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_KERNEL)); |
156 | memset(__va(phys_addr), KASAN_SHADOW_INIT, PGDIR_SIZE); |
157 | continue; |
158 | } |
159 | } |
160 | |
161 | kasan_populate_p4d(pgd: pgdp, vaddr, end: next); |
162 | } while (pgdp++, vaddr = next, vaddr != end); |
163 | } |
164 | |
165 | static void __init kasan_early_clear_pud(p4d_t *p4dp, |
166 | unsigned long vaddr, unsigned long end) |
167 | { |
168 | pud_t *pudp, *base_pud; |
169 | unsigned long next; |
170 | |
171 | if (!pgtable_l4_enabled) { |
172 | pudp = (pud_t *)p4dp; |
173 | } else { |
174 | base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); |
175 | pudp = base_pud + pud_index(address: vaddr); |
176 | } |
177 | |
178 | do { |
179 | next = pud_addr_end(vaddr, end); |
180 | |
181 | if (IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) { |
182 | pud_clear(pudp); |
183 | continue; |
184 | } |
185 | |
186 | BUG(); |
187 | } while (pudp++, vaddr = next, vaddr != end); |
188 | } |
189 | |
190 | static void __init kasan_early_clear_p4d(pgd_t *pgdp, |
191 | unsigned long vaddr, unsigned long end) |
192 | { |
193 | p4d_t *p4dp, *base_p4d; |
194 | unsigned long next; |
195 | |
196 | if (!pgtable_l5_enabled) { |
197 | p4dp = (p4d_t *)pgdp; |
198 | } else { |
199 | base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); |
200 | p4dp = base_p4d + p4d_index(address: vaddr); |
201 | } |
202 | |
203 | do { |
204 | next = p4d_addr_end(vaddr, end); |
205 | |
206 | if (pgtable_l4_enabled && IS_ALIGNED(vaddr, P4D_SIZE) && |
207 | (next - vaddr) >= P4D_SIZE) { |
208 | p4d_clear(p4dp); |
209 | continue; |
210 | } |
211 | |
212 | kasan_early_clear_pud(p4dp, vaddr, end: next); |
213 | } while (p4dp++, vaddr = next, vaddr != end); |
214 | } |
215 | |
216 | static void __init kasan_early_clear_pgd(pgd_t *pgdp, |
217 | unsigned long vaddr, unsigned long end) |
218 | { |
219 | unsigned long next; |
220 | |
221 | do { |
222 | next = pgd_addr_end(vaddr, end); |
223 | |
224 | if (pgtable_l5_enabled && IS_ALIGNED(vaddr, PGDIR_SIZE) && |
225 | (next - vaddr) >= PGDIR_SIZE) { |
226 | pgd_clear(pgdp); |
227 | continue; |
228 | } |
229 | |
230 | kasan_early_clear_p4d(pgdp, vaddr, end: next); |
231 | } while (pgdp++, vaddr = next, vaddr != end); |
232 | } |
233 | |
234 | static void __init kasan_early_populate_pud(p4d_t *p4dp, |
235 | unsigned long vaddr, |
236 | unsigned long end) |
237 | { |
238 | pud_t *pudp, *base_pud; |
239 | phys_addr_t phys_addr; |
240 | unsigned long next; |
241 | |
242 | if (!pgtable_l4_enabled) { |
243 | pudp = (pud_t *)p4dp; |
244 | } else { |
245 | base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); |
246 | pudp = base_pud + pud_index(address: vaddr); |
247 | } |
248 | |
249 | do { |
250 | next = pud_addr_end(vaddr, end); |
251 | |
252 | if (pud_none(pud: pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) && |
253 | (next - vaddr) >= PUD_SIZE) { |
254 | phys_addr = __pa((uintptr_t)kasan_early_shadow_pmd); |
255 | set_pud(pudp, pud: pfn_pud(PFN_DOWN(phys_addr), pgprot: PAGE_TABLE)); |
256 | continue; |
257 | } |
258 | |
259 | BUG(); |
260 | } while (pudp++, vaddr = next, vaddr != end); |
261 | } |
262 | |
263 | static void __init kasan_early_populate_p4d(pgd_t *pgdp, |
264 | unsigned long vaddr, |
265 | unsigned long end) |
266 | { |
267 | p4d_t *p4dp, *base_p4d; |
268 | phys_addr_t phys_addr; |
269 | unsigned long next; |
270 | |
271 | /* |
272 | * We can't use pgd_page_vaddr here as it would return a linear |
273 | * mapping address but it is not mapped yet, but when populating |
274 | * early_pg_dir, we need the physical address and when populating |
275 | * swapper_pg_dir, we need the kernel virtual address so use |
276 | * pt_ops facility. |
277 | * Note that this test is then completely equivalent to |
278 | * p4dp = p4d_offset(pgdp, vaddr) |
279 | */ |
280 | if (!pgtable_l5_enabled) { |
281 | p4dp = (p4d_t *)pgdp; |
282 | } else { |
283 | base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); |
284 | p4dp = base_p4d + p4d_index(address: vaddr); |
285 | } |
286 | |
287 | do { |
288 | next = p4d_addr_end(vaddr, end); |
289 | |
290 | if (p4d_none(p4d: p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) && |
291 | (next - vaddr) >= P4D_SIZE) { |
292 | phys_addr = __pa((uintptr_t)kasan_early_shadow_pud); |
293 | set_p4d(p4dp, p4d: pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE)); |
294 | continue; |
295 | } |
296 | |
297 | kasan_early_populate_pud(p4dp, vaddr, end: next); |
298 | } while (p4dp++, vaddr = next, vaddr != end); |
299 | } |
300 | |
301 | static void __init kasan_early_populate_pgd(pgd_t *pgdp, |
302 | unsigned long vaddr, |
303 | unsigned long end) |
304 | { |
305 | phys_addr_t phys_addr; |
306 | unsigned long next; |
307 | |
308 | do { |
309 | next = pgd_addr_end(vaddr, end); |
310 | |
311 | if (pgd_none(pgd: pgdp_get(pgdp)) && IS_ALIGNED(vaddr, PGDIR_SIZE) && |
312 | (next - vaddr) >= PGDIR_SIZE) { |
313 | phys_addr = __pa((uintptr_t)kasan_early_shadow_p4d); |
314 | set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE)); |
315 | continue; |
316 | } |
317 | |
318 | kasan_early_populate_p4d(pgdp, vaddr, end: next); |
319 | } while (pgdp++, vaddr = next, vaddr != end); |
320 | } |
321 | |
322 | asmlinkage void __init kasan_early_init(void) |
323 | { |
324 | uintptr_t i; |
325 | |
326 | BUILD_BUG_ON(KASAN_SHADOW_OFFSET != |
327 | KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT))); |
328 | |
329 | for (i = 0; i < PTRS_PER_PTE; ++i) |
330 | set_pte(ptep: kasan_early_shadow_pte + i, |
331 | pte: pfn_pte(page_nr: virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL)); |
332 | |
333 | for (i = 0; i < PTRS_PER_PMD; ++i) |
334 | set_pmd(kasan_early_shadow_pmd + i, |
335 | pfn_pmd(PFN_DOWN |
336 | (__pa((uintptr_t)kasan_early_shadow_pte)), |
337 | PAGE_TABLE)); |
338 | |
339 | if (pgtable_l4_enabled) { |
340 | for (i = 0; i < PTRS_PER_PUD; ++i) |
341 | set_pud(kasan_early_shadow_pud + i, |
342 | pfn_pud(PFN_DOWN |
343 | (__pa(((uintptr_t)kasan_early_shadow_pmd))), |
344 | PAGE_TABLE)); |
345 | } |
346 | |
347 | if (pgtable_l5_enabled) { |
348 | for (i = 0; i < PTRS_PER_P4D; ++i) |
349 | set_p4d(kasan_early_shadow_p4d + i, |
350 | pfn_p4d(PFN_DOWN |
351 | (__pa(((uintptr_t)kasan_early_shadow_pud))), |
352 | PAGE_TABLE)); |
353 | } |
354 | |
355 | kasan_early_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START), |
356 | KASAN_SHADOW_START, KASAN_SHADOW_END); |
357 | |
358 | local_flush_tlb_all(); |
359 | } |
360 | |
361 | void __init kasan_swapper_init(void) |
362 | { |
363 | kasan_early_populate_pgd(pgd_offset_k(KASAN_SHADOW_START), |
364 | KASAN_SHADOW_START, KASAN_SHADOW_END); |
365 | |
366 | local_flush_tlb_all(); |
367 | } |
368 | |
369 | static void __init kasan_populate(void *start, void *end) |
370 | { |
371 | unsigned long vaddr = (unsigned long)start & PAGE_MASK; |
372 | unsigned long vend = PAGE_ALIGN((unsigned long)end); |
373 | |
374 | kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, end: vend); |
375 | } |
376 | |
377 | static void __init kasan_shallow_populate_pud(p4d_t *p4d, |
378 | unsigned long vaddr, unsigned long end) |
379 | { |
380 | unsigned long next; |
381 | void *p; |
382 | pud_t *pud_k = pud_offset(p4d, address: vaddr); |
383 | |
384 | do { |
385 | next = pud_addr_end(vaddr, end); |
386 | |
387 | if (pud_none(pud: pudp_get(pudp: pud_k))) { |
388 | p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); |
389 | set_pud(pud_k, pfn_pud(PFN_DOWN(__pa(p)), PAGE_TABLE)); |
390 | continue; |
391 | } |
392 | |
393 | BUG(); |
394 | } while (pud_k++, vaddr = next, vaddr != end); |
395 | } |
396 | |
397 | static void __init kasan_shallow_populate_p4d(pgd_t *pgd, |
398 | unsigned long vaddr, unsigned long end) |
399 | { |
400 | unsigned long next; |
401 | void *p; |
402 | p4d_t *p4d_k = p4d_offset(pgd, address: vaddr); |
403 | |
404 | do { |
405 | next = p4d_addr_end(vaddr, end); |
406 | |
407 | if (p4d_none(p4d: p4dp_get(p4dp: p4d_k))) { |
408 | p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); |
409 | set_p4d(p4d_k, pfn_p4d(PFN_DOWN(__pa(p)), PAGE_TABLE)); |
410 | continue; |
411 | } |
412 | |
413 | kasan_shallow_populate_pud(p4d: p4d_k, vaddr, end); |
414 | } while (p4d_k++, vaddr = next, vaddr != end); |
415 | } |
416 | |
417 | static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long end) |
418 | { |
419 | unsigned long next; |
420 | void *p; |
421 | pgd_t *pgd_k = pgd_offset_k(vaddr); |
422 | |
423 | do { |
424 | next = pgd_addr_end(vaddr, end); |
425 | |
426 | if (pgd_none(pgd: pgdp_get(pgdp: pgd_k))) { |
427 | p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); |
428 | set_pgd(pgd_k, pfn_pgd(PFN_DOWN(__pa(p)), PAGE_TABLE)); |
429 | continue; |
430 | } |
431 | |
432 | kasan_shallow_populate_p4d(pgd: pgd_k, vaddr, end: next); |
433 | } while (pgd_k++, vaddr = next, vaddr != end); |
434 | } |
435 | |
436 | static void __init kasan_shallow_populate(void *start, void *end) |
437 | { |
438 | unsigned long vaddr = (unsigned long)start & PAGE_MASK; |
439 | unsigned long vend = PAGE_ALIGN((unsigned long)end); |
440 | |
441 | kasan_shallow_populate_pgd(vaddr, end: vend); |
442 | } |
443 | |
444 | #ifdef CONFIG_KASAN_VMALLOC |
445 | void __init kasan_populate_early_vm_area_shadow(void *start, unsigned long size) |
446 | { |
447 | kasan_populate(kasan_mem_to_shadow(start), |
448 | kasan_mem_to_shadow(start + size)); |
449 | } |
450 | #endif |
451 | |
452 | static void __init create_tmp_mapping(void) |
453 | { |
454 | void *ptr; |
455 | p4d_t *base_p4d; |
456 | |
457 | /* |
458 | * We need to clean the early mapping: this is hard to achieve "in-place", |
459 | * so install a temporary mapping like arm64 and x86 do. |
460 | */ |
461 | memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(pgd_t) * PTRS_PER_PGD); |
462 | |
463 | /* Copy the last p4d since it is shared with the kernel mapping. */ |
464 | if (pgtable_l5_enabled) { |
465 | ptr = (p4d_t *)pgd_page_vaddr(pgdp_get(pgd_offset_k(KASAN_SHADOW_END))); |
466 | memcpy(tmp_p4d, ptr, sizeof(p4d_t) * PTRS_PER_P4D); |
467 | set_pgd(&tmp_pg_dir[pgd_index(KASAN_SHADOW_END)], |
468 | pfn_pgd(PFN_DOWN(__pa(tmp_p4d)), PAGE_TABLE)); |
469 | base_p4d = tmp_p4d; |
470 | } else { |
471 | base_p4d = (p4d_t *)tmp_pg_dir; |
472 | } |
473 | |
474 | /* Copy the last pud since it is shared with the kernel mapping. */ |
475 | if (pgtable_l4_enabled) { |
476 | ptr = (pud_t *)p4d_page_vaddr(p4dp_get(base_p4d + p4d_index(KASAN_SHADOW_END))); |
477 | memcpy(tmp_pud, ptr, sizeof(pud_t) * PTRS_PER_PUD); |
478 | set_p4d(&base_p4d[p4d_index(KASAN_SHADOW_END)], |
479 | pfn_p4d(PFN_DOWN(__pa(tmp_pud)), PAGE_TABLE)); |
480 | } |
481 | } |
482 | |
483 | void __init kasan_init(void) |
484 | { |
485 | phys_addr_t p_start, p_end; |
486 | u64 i; |
487 | |
488 | create_tmp_mapping(); |
489 | csr_write(CSR_SATP, PFN_DOWN(__pa(tmp_pg_dir)) | satp_mode); |
490 | |
491 | kasan_early_clear_pgd(pgd_offset_k(KASAN_SHADOW_START), |
492 | KASAN_SHADOW_START, KASAN_SHADOW_END); |
493 | |
494 | kasan_populate_early_shadow((void *)kasan_mem_to_shadow((void *)FIXADDR_START), |
495 | (void *)kasan_mem_to_shadow((void *)VMALLOC_START)); |
496 | |
497 | if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { |
498 | kasan_shallow_populate( |
499 | start: (void *)kasan_mem_to_shadow((void *)VMALLOC_START), |
500 | end: (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); |
501 | /* Shallow populate modules and BPF which are vmalloc-allocated */ |
502 | kasan_shallow_populate( |
503 | start: (void *)kasan_mem_to_shadow((void *)MODULES_VADDR), |
504 | end: (void *)kasan_mem_to_shadow((void *)MODULES_END)); |
505 | } else { |
506 | kasan_populate_early_shadow((void *)kasan_mem_to_shadow((void *)VMALLOC_START), |
507 | (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); |
508 | } |
509 | |
510 | /* Populate the linear mapping */ |
511 | for_each_mem_range(i, &p_start, &p_end) { |
512 | void *start = (void *)__va(p_start); |
513 | void *end = (void *)__va(p_end); |
514 | |
515 | if (start >= end) |
516 | break; |
517 | |
518 | kasan_populate(start: kasan_mem_to_shadow(start), end: kasan_mem_to_shadow(end)); |
519 | } |
520 | |
521 | /* Populate kernel */ |
522 | kasan_populate(start: kasan_mem_to_shadow((const void *)MODULES_END), |
523 | end: kasan_mem_to_shadow((const void *)MODULES_VADDR + SZ_2G)); |
524 | |
525 | for (i = 0; i < PTRS_PER_PTE; i++) |
526 | set_pte(&kasan_early_shadow_pte[i], |
527 | mk_pte(virt_to_page(kasan_early_shadow_page), |
528 | __pgprot(_PAGE_PRESENT | _PAGE_READ | |
529 | _PAGE_ACCESSED))); |
530 | |
531 | memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE); |
532 | init_task.kasan_depth = 0; |
533 | |
534 | csr_write(CSR_SATP, PFN_DOWN(__pa(swapper_pg_dir)) | satp_mode); |
535 | local_flush_tlb_all(); |
536 | } |
537 | |