1// SPDX-License-Identifier: GPL-2.0
2
3#define DISABLE_BRANCH_PROFILING
4
5#include <linux/kasan.h>
6#include <linux/printk.h>
7#include <linux/memblock.h>
8#include <linux/sched/task.h>
9#include <asm/pgalloc.h>
10#include <asm/text-patching.h>
11#include <mm/mmu_decl.h>
12
13static pgprot_t __init kasan_prot_ro(void)
14{
15 if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE))
16 return PAGE_READONLY;
17
18 return PAGE_KERNEL_RO;
19}
20
21static void __init kasan_populate_pte(pte_t *ptep, pgprot_t prot)
22{
23 unsigned long va = (unsigned long)kasan_early_shadow_page;
24 phys_addr_t pa = __pa(kasan_early_shadow_page);
25 int i;
26
27 for (i = 0; i < PTRS_PER_PTE; i++, ptep++)
28 __set_pte_at(&init_mm, va, ptep, pfn_pte(PHYS_PFN(pa), pgprot: prot), 1);
29}
30
31int __init kasan_init_shadow_page_tables(unsigned long k_start, unsigned long k_end)
32{
33 pmd_t *pmd;
34 unsigned long k_cur, k_next;
35
36 pmd = pmd_off_k(va: k_start);
37
38 for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) {
39 pte_t *new;
40
41 k_next = pgd_addr_end(k_cur, k_end);
42 if ((void *)pmd_page_vaddr(pmd: *pmd) != kasan_early_shadow_pte)
43 continue;
44
45 new = memblock_alloc(size: PTE_FRAG_SIZE, align: PTE_FRAG_SIZE);
46
47 if (!new)
48 return -ENOMEM;
49 kasan_populate_pte(ptep: new, PAGE_KERNEL);
50 pmd_populate_kernel(mm: &init_mm, pmd, pte: new);
51 }
52 return 0;
53}
54
55int __init __weak kasan_init_region(void *start, size_t size)
56{
57 unsigned long k_start = (unsigned long)kasan_mem_to_shadow(addr: start);
58 unsigned long k_end = (unsigned long)kasan_mem_to_shadow(addr: start + size);
59 unsigned long k_cur;
60 int ret;
61 void *block;
62
63 ret = kasan_init_shadow_page_tables(k_start, k_end);
64 if (ret)
65 return ret;
66
67 k_start = k_start & PAGE_MASK;
68 block = memblock_alloc(size: k_end - k_start, PAGE_SIZE);
69 if (!block)
70 return -ENOMEM;
71
72 for (k_cur = k_start & PAGE_MASK; k_cur < k_end; k_cur += PAGE_SIZE) {
73 pmd_t *pmd = pmd_off_k(va: k_cur);
74 void *va = block + k_cur - k_start;
75 pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL);
76
77 __set_pte_at(&init_mm, k_cur, pte_offset_kernel(pmd, address: k_cur), pte, 0);
78 }
79 flush_tlb_kernel_range(start: k_start, end: k_end);
80 return 0;
81}
82
83void __init
84kasan_update_early_region(unsigned long k_start, unsigned long k_end, pte_t pte)
85{
86 unsigned long k_cur;
87
88 for (k_cur = k_start; k_cur != k_end; k_cur += PAGE_SIZE) {
89 pmd_t *pmd = pmd_off_k(va: k_cur);
90 pte_t *ptep = pte_offset_kernel(pmd, address: k_cur);
91
92 if (pte_page(*ptep) != virt_to_page(lm_alias(kasan_early_shadow_page)))
93 continue;
94
95 __set_pte_at(&init_mm, k_cur, ptep, pte, 0);
96 }
97
98 flush_tlb_kernel_range(start: k_start, end: k_end);
99}
100
101static void __init kasan_remap_early_shadow_ro(void)
102{
103 pgprot_t prot = kasan_prot_ro();
104 phys_addr_t pa = __pa(kasan_early_shadow_page);
105
106 kasan_populate_pte(ptep: kasan_early_shadow_pte, prot);
107
108 kasan_update_early_region(KASAN_SHADOW_START, KASAN_SHADOW_END,
109 pte: pfn_pte(PHYS_PFN(pa), pgprot: prot));
110}
111
112static void __init kasan_unmap_early_shadow_vmalloc(void)
113{
114 unsigned long k_start = (unsigned long)kasan_mem_to_shadow(addr: (void *)VMALLOC_START);
115 unsigned long k_end = (unsigned long)kasan_mem_to_shadow(addr: (void *)VMALLOC_END);
116
117 kasan_update_early_region(k_start, k_end, pte: __pte(val: 0));
118
119#ifdef MODULES_VADDR
120 k_start = (unsigned long)kasan_mem_to_shadow(addr: (void *)MODULES_VADDR);
121 k_end = (unsigned long)kasan_mem_to_shadow(addr: (void *)MODULES_END);
122 kasan_update_early_region(k_start, k_end, pte: __pte(val: 0));
123#endif
124}
125
126void __init kasan_mmu_init(void)
127{
128 int ret;
129
130 if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) {
131 ret = kasan_init_shadow_page_tables(KASAN_SHADOW_START, KASAN_SHADOW_END);
132
133 if (ret)
134 panic(fmt: "kasan: kasan_init_shadow_page_tables() failed");
135 }
136}
137
138void __init kasan_init(void)
139{
140 phys_addr_t base, end;
141 u64 i;
142 int ret;
143
144 for_each_mem_range(i, &base, &end) {
145 phys_addr_t top = min(end, total_lowmem);
146
147 if (base >= top)
148 continue;
149
150 ret = kasan_init_region(__va(base), size: top - base);
151 if (ret)
152 panic(fmt: "kasan: kasan_init_region() failed");
153 }
154
155 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
156 ret = kasan_init_shadow_page_tables(KASAN_SHADOW_START, KASAN_SHADOW_END);
157
158 if (ret)
159 panic(fmt: "kasan: kasan_init_shadow_page_tables() failed");
160 }
161
162 kasan_remap_early_shadow_ro();
163
164 clear_page(page: kasan_early_shadow_page);
165
166 /* At this point kasan is fully initialized. Enable error messages */
167 init_task.kasan_depth = 0;
168 kasan_init_generic();
169}
170
171void __init kasan_late_init(void)
172{
173 if (IS_ENABLED(CONFIG_KASAN_VMALLOC))
174 kasan_unmap_early_shadow_vmalloc();
175}
176
177void __init kasan_early_init(void)
178{
179 unsigned long addr = KASAN_SHADOW_START;
180 unsigned long end = KASAN_SHADOW_END;
181 unsigned long next;
182 pmd_t *pmd = pmd_off_k(va: addr);
183
184 BUILD_BUG_ON(KASAN_SHADOW_START & ~PGDIR_MASK);
185
186 kasan_populate_pte(ptep: kasan_early_shadow_pte, PAGE_KERNEL);
187
188 do {
189 next = pgd_addr_end(addr, end);
190 pmd_populate_kernel(mm: &init_mm, pmd, pte: kasan_early_shadow_pte);
191 } while (pmd++, addr = next, addr != end);
192}
193

source code of linux/arch/powerpc/mm/kasan/init_32.c