1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2019 SiFive |
4 | */ |
5 | |
6 | #include <linux/pagewalk.h> |
7 | #include <linux/pgtable.h> |
8 | #include <linux/vmalloc.h> |
9 | #include <asm/tlbflush.h> |
10 | #include <asm/bitops.h> |
11 | #include <asm/set_memory.h> |
12 | |
13 | struct pageattr_masks { |
14 | pgprot_t set_mask; |
15 | pgprot_t clear_mask; |
16 | }; |
17 | |
18 | static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) |
19 | { |
20 | struct pageattr_masks *masks = walk->private; |
21 | unsigned long new_val = val; |
22 | |
23 | new_val &= ~(pgprot_val(masks->clear_mask)); |
24 | new_val |= (pgprot_val(masks->set_mask)); |
25 | |
26 | return new_val; |
27 | } |
28 | |
29 | static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr, |
30 | unsigned long next, struct mm_walk *walk) |
31 | { |
32 | p4d_t val = p4dp_get(p4dp: p4d); |
33 | |
34 | if (p4d_leaf(p4d: val)) { |
35 | val = __p4d(val: set_pageattr_masks(val: p4d_val(p4d: val), walk)); |
36 | set_p4d(p4dp: p4d, p4d: val); |
37 | } |
38 | |
39 | return 0; |
40 | } |
41 | |
42 | static int pageattr_pud_entry(pud_t *pud, unsigned long addr, |
43 | unsigned long next, struct mm_walk *walk) |
44 | { |
45 | pud_t val = pudp_get(pudp: pud); |
46 | |
47 | if (pud_leaf(pud: val)) { |
48 | val = __pud(val: set_pageattr_masks(val: pud_val(pud: val), walk)); |
49 | set_pud(pudp: pud, pud: val); |
50 | } |
51 | |
52 | return 0; |
53 | } |
54 | |
55 | static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, |
56 | unsigned long next, struct mm_walk *walk) |
57 | { |
58 | pmd_t val = pmdp_get(pmdp: pmd); |
59 | |
60 | if (pmd_leaf(pte: val)) { |
61 | val = __pmd(val: set_pageattr_masks(val: pmd_val(pmd: val), walk)); |
62 | set_pmd(pmdp: pmd, pmd: val); |
63 | } |
64 | |
65 | return 0; |
66 | } |
67 | |
68 | static int pageattr_pte_entry(pte_t *pte, unsigned long addr, |
69 | unsigned long next, struct mm_walk *walk) |
70 | { |
71 | pte_t val = ptep_get(ptep: pte); |
72 | |
73 | val = __pte(val: set_pageattr_masks(val: pte_val(pte: val), walk)); |
74 | set_pte(ptep: pte, pte: val); |
75 | |
76 | return 0; |
77 | } |
78 | |
79 | static int pageattr_pte_hole(unsigned long addr, unsigned long next, |
80 | int depth, struct mm_walk *walk) |
81 | { |
82 | /* Nothing to do here */ |
83 | return 0; |
84 | } |
85 | |
86 | static const struct mm_walk_ops pageattr_ops = { |
87 | .p4d_entry = pageattr_p4d_entry, |
88 | .pud_entry = pageattr_pud_entry, |
89 | .pmd_entry = pageattr_pmd_entry, |
90 | .pte_entry = pageattr_pte_entry, |
91 | .pte_hole = pageattr_pte_hole, |
92 | .walk_lock = PGWALK_RDLOCK, |
93 | }; |
94 | |
95 | #ifdef CONFIG_64BIT |
96 | static int __split_linear_mapping_pmd(pud_t *pudp, |
97 | unsigned long vaddr, unsigned long end) |
98 | { |
99 | pmd_t *pmdp; |
100 | unsigned long next; |
101 | |
102 | pmdp = pmd_offset(pud: pudp, address: vaddr); |
103 | |
104 | do { |
105 | next = pmd_addr_end(vaddr, end); |
106 | |
107 | if (next - vaddr >= PMD_SIZE && |
108 | vaddr <= (vaddr & PMD_MASK) && end >= next) |
109 | continue; |
110 | |
111 | if (pmd_leaf(pte: pmdp_get(pmdp))) { |
112 | struct page *pte_page; |
113 | unsigned long pfn = _pmd_pfn(pmdp_get(pmdp)); |
114 | pgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) & ~_PAGE_PFN_MASK); |
115 | pte_t *ptep_new; |
116 | int i; |
117 | |
118 | pte_page = alloc_page(GFP_KERNEL); |
119 | if (!pte_page) |
120 | return -ENOMEM; |
121 | |
122 | ptep_new = (pte_t *)page_address(pte_page); |
123 | for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep_new) |
124 | set_pte(ptep: ptep_new, pte: pfn_pte(page_nr: pfn + i, pgprot: prot)); |
125 | |
126 | smp_wmb(); |
127 | |
128 | set_pmd(pmdp, pmd: pfn_pmd(page_to_pfn(pte_page), pgprot: PAGE_TABLE)); |
129 | } |
130 | } while (pmdp++, vaddr = next, vaddr != end); |
131 | |
132 | return 0; |
133 | } |
134 | |
135 | static int __split_linear_mapping_pud(p4d_t *p4dp, |
136 | unsigned long vaddr, unsigned long end) |
137 | { |
138 | pud_t *pudp; |
139 | unsigned long next; |
140 | int ret; |
141 | |
142 | pudp = pud_offset(p4d: p4dp, address: vaddr); |
143 | |
144 | do { |
145 | next = pud_addr_end(vaddr, end); |
146 | |
147 | if (next - vaddr >= PUD_SIZE && |
148 | vaddr <= (vaddr & PUD_MASK) && end >= next) |
149 | continue; |
150 | |
151 | if (pud_leaf(pud: pudp_get(pudp))) { |
152 | struct page *pmd_page; |
153 | unsigned long pfn = _pud_pfn(pudp_get(pudp)); |
154 | pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK); |
155 | pmd_t *pmdp_new; |
156 | int i; |
157 | |
158 | pmd_page = alloc_page(GFP_KERNEL); |
159 | if (!pmd_page) |
160 | return -ENOMEM; |
161 | |
162 | pmdp_new = (pmd_t *)page_address(pmd_page); |
163 | for (i = 0; i < PTRS_PER_PMD; ++i, ++pmdp_new) |
164 | set_pmd(pmdp: pmdp_new, |
165 | pmd: pfn_pmd(page_nr: pfn + ((i * PMD_SIZE) >> PAGE_SHIFT), pgprot: prot)); |
166 | |
167 | smp_wmb(); |
168 | |
169 | set_pud(pudp, pud: pfn_pud(page_to_pfn(pmd_page), pgprot: PAGE_TABLE)); |
170 | } |
171 | |
172 | ret = __split_linear_mapping_pmd(pudp, vaddr, end: next); |
173 | if (ret) |
174 | return ret; |
175 | } while (pudp++, vaddr = next, vaddr != end); |
176 | |
177 | return 0; |
178 | } |
179 | |
180 | static int __split_linear_mapping_p4d(pgd_t *pgdp, |
181 | unsigned long vaddr, unsigned long end) |
182 | { |
183 | p4d_t *p4dp; |
184 | unsigned long next; |
185 | int ret; |
186 | |
187 | p4dp = p4d_offset(pgd: pgdp, address: vaddr); |
188 | |
189 | do { |
190 | next = p4d_addr_end(vaddr, end); |
191 | |
192 | /* |
193 | * If [vaddr; end] contains [vaddr & P4D_MASK; next], we don't |
194 | * need to split, we'll change the protections on the whole P4D. |
195 | */ |
196 | if (next - vaddr >= P4D_SIZE && |
197 | vaddr <= (vaddr & P4D_MASK) && end >= next) |
198 | continue; |
199 | |
200 | if (p4d_leaf(p4d: p4dp_get(p4dp))) { |
201 | struct page *pud_page; |
202 | unsigned long pfn = _p4d_pfn(p4dp_get(p4dp)); |
203 | pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK); |
204 | pud_t *pudp_new; |
205 | int i; |
206 | |
207 | pud_page = alloc_page(GFP_KERNEL); |
208 | if (!pud_page) |
209 | return -ENOMEM; |
210 | |
211 | /* |
212 | * Fill the pud level with leaf puds that have the same |
213 | * protections as the leaf p4d. |
214 | */ |
215 | pudp_new = (pud_t *)page_address(pud_page); |
216 | for (i = 0; i < PTRS_PER_PUD; ++i, ++pudp_new) |
217 | set_pud(pudp: pudp_new, |
218 | pud: pfn_pud(page_nr: pfn + ((i * PUD_SIZE) >> PAGE_SHIFT), pgprot: prot)); |
219 | |
220 | /* |
221 | * Make sure the pud filling is not reordered with the |
222 | * p4d store which could result in seeing a partially |
223 | * filled pud level. |
224 | */ |
225 | smp_wmb(); |
226 | |
227 | set_p4d(p4dp, p4d: pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE)); |
228 | } |
229 | |
230 | ret = __split_linear_mapping_pud(p4dp, vaddr, end: next); |
231 | if (ret) |
232 | return ret; |
233 | } while (p4dp++, vaddr = next, vaddr != end); |
234 | |
235 | return 0; |
236 | } |
237 | |
238 | static int __split_linear_mapping_pgd(pgd_t *pgdp, |
239 | unsigned long vaddr, |
240 | unsigned long end) |
241 | { |
242 | unsigned long next; |
243 | int ret; |
244 | |
245 | do { |
246 | next = pgd_addr_end(vaddr, end); |
247 | /* We never use PGD mappings for the linear mapping */ |
248 | ret = __split_linear_mapping_p4d(pgdp, vaddr, end: next); |
249 | if (ret) |
250 | return ret; |
251 | } while (pgdp++, vaddr = next, vaddr != end); |
252 | |
253 | return 0; |
254 | } |
255 | |
256 | static int split_linear_mapping(unsigned long start, unsigned long end) |
257 | { |
258 | return __split_linear_mapping_pgd(pgd_offset_k(start), vaddr: start, end); |
259 | } |
260 | #endif /* CONFIG_64BIT */ |
261 | |
262 | static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, |
263 | pgprot_t clear_mask) |
264 | { |
265 | int ret; |
266 | unsigned long start = addr; |
267 | unsigned long end = start + PAGE_SIZE * numpages; |
268 | unsigned long __maybe_unused lm_start; |
269 | unsigned long __maybe_unused lm_end; |
270 | struct pageattr_masks masks = { |
271 | .set_mask = set_mask, |
272 | .clear_mask = clear_mask |
273 | }; |
274 | |
275 | if (!numpages) |
276 | return 0; |
277 | |
278 | mmap_write_lock(mm: &init_mm); |
279 | |
280 | #ifdef CONFIG_64BIT |
281 | /* |
282 | * We are about to change the permissions of a kernel mapping, we must |
283 | * apply the same changes to its linear mapping alias, which may imply |
284 | * splitting a huge mapping. |
285 | */ |
286 | |
287 | if (is_vmalloc_or_module_addr(x: (void *)start)) { |
288 | struct vm_struct *area = NULL; |
289 | int i, page_start; |
290 | |
291 | area = find_vm_area(addr: (void *)start); |
292 | page_start = (start - (unsigned long)area->addr) >> PAGE_SHIFT; |
293 | |
294 | for (i = page_start; i < page_start + numpages; ++i) { |
295 | lm_start = (unsigned long)page_address(area->pages[i]); |
296 | lm_end = lm_start + PAGE_SIZE; |
297 | |
298 | ret = split_linear_mapping(start: lm_start, end: lm_end); |
299 | if (ret) |
300 | goto unlock; |
301 | |
302 | ret = walk_page_range_novma(mm: &init_mm, start: lm_start, end: lm_end, |
303 | ops: &pageattr_ops, NULL, private: &masks); |
304 | if (ret) |
305 | goto unlock; |
306 | } |
307 | } else if (is_kernel_mapping(start) || is_linear_mapping(start)) { |
308 | if (is_kernel_mapping(start)) { |
309 | lm_start = (unsigned long)lm_alias(start); |
310 | lm_end = (unsigned long)lm_alias(end); |
311 | } else { |
312 | lm_start = start; |
313 | lm_end = end; |
314 | } |
315 | |
316 | ret = split_linear_mapping(start: lm_start, end: lm_end); |
317 | if (ret) |
318 | goto unlock; |
319 | |
320 | ret = walk_page_range_novma(mm: &init_mm, start: lm_start, end: lm_end, |
321 | ops: &pageattr_ops, NULL, private: &masks); |
322 | if (ret) |
323 | goto unlock; |
324 | } |
325 | |
326 | ret = walk_page_range_novma(mm: &init_mm, start, end, ops: &pageattr_ops, NULL, |
327 | private: &masks); |
328 | |
329 | unlock: |
330 | mmap_write_unlock(mm: &init_mm); |
331 | |
332 | /* |
333 | * We can't use flush_tlb_kernel_range() here as we may have split a |
334 | * hugepage that is larger than that, so let's flush everything. |
335 | */ |
336 | flush_tlb_all(); |
337 | #else |
338 | ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, |
339 | &masks); |
340 | |
341 | mmap_write_unlock(&init_mm); |
342 | |
343 | flush_tlb_kernel_range(start, end); |
344 | #endif |
345 | |
346 | return ret; |
347 | } |
348 | |
349 | int set_memory_rw_nx(unsigned long addr, int numpages) |
350 | { |
351 | return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE), |
352 | __pgprot(_PAGE_EXEC)); |
353 | } |
354 | |
355 | int set_memory_ro(unsigned long addr, int numpages) |
356 | { |
357 | return __set_memory(addr, numpages, __pgprot(_PAGE_READ), |
358 | __pgprot(_PAGE_WRITE)); |
359 | } |
360 | |
361 | int set_memory_rw(unsigned long addr, int numpages) |
362 | { |
363 | return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE), |
364 | __pgprot(0)); |
365 | } |
366 | |
367 | int set_memory_x(unsigned long addr, int numpages) |
368 | { |
369 | return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0)); |
370 | } |
371 | |
372 | int set_memory_nx(unsigned long addr, int numpages) |
373 | { |
374 | return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); |
375 | } |
376 | |
377 | int set_direct_map_invalid_noflush(struct page *page) |
378 | { |
379 | return __set_memory(addr: (unsigned long)page_address(page), numpages: 1, |
380 | __pgprot(0), __pgprot(_PAGE_PRESENT)); |
381 | } |
382 | |
383 | int set_direct_map_default_noflush(struct page *page) |
384 | { |
385 | return __set_memory((unsigned long)page_address(page), 1, |
386 | PAGE_KERNEL, __pgprot(_PAGE_EXEC)); |
387 | } |
388 | |
389 | #ifdef CONFIG_DEBUG_PAGEALLOC |
390 | void __kernel_map_pages(struct page *page, int numpages, int enable) |
391 | { |
392 | if (!debug_pagealloc_enabled()) |
393 | return; |
394 | |
395 | if (enable) |
396 | __set_memory(addr: (unsigned long)page_address(page), numpages, |
397 | __pgprot(_PAGE_PRESENT), __pgprot(0)); |
398 | else |
399 | __set_memory(addr: (unsigned long)page_address(page), numpages, |
400 | __pgprot(0), __pgprot(_PAGE_PRESENT)); |
401 | } |
402 | #endif |
403 | |
404 | bool kernel_page_present(struct page *page) |
405 | { |
406 | unsigned long addr = (unsigned long)page_address(page); |
407 | pgd_t *pgd; |
408 | pud_t *pud; |
409 | p4d_t *p4d; |
410 | pmd_t *pmd; |
411 | pte_t *pte; |
412 | |
413 | pgd = pgd_offset_k(addr); |
414 | if (!pgd_present(pgd: pgdp_get(pgdp: pgd))) |
415 | return false; |
416 | if (pgd_leaf(pgd: pgdp_get(pgdp: pgd))) |
417 | return true; |
418 | |
419 | p4d = p4d_offset(pgd, address: addr); |
420 | if (!p4d_present(p4d: p4dp_get(p4dp: p4d))) |
421 | return false; |
422 | if (p4d_leaf(p4d: p4dp_get(p4dp: p4d))) |
423 | return true; |
424 | |
425 | pud = pud_offset(p4d, address: addr); |
426 | if (!pud_present(pud: pudp_get(pudp: pud))) |
427 | return false; |
428 | if (pud_leaf(pud: pudp_get(pudp: pud))) |
429 | return true; |
430 | |
431 | pmd = pmd_offset(pud, address: addr); |
432 | if (!pmd_present(pmd: pmdp_get(pmdp: pmd))) |
433 | return false; |
434 | if (pmd_leaf(pte: pmdp_get(pmdp: pmd))) |
435 | return true; |
436 | |
437 | pte = pte_offset_kernel(pmd, address: addr); |
438 | return pte_present(a: ptep_get(ptep: pte)); |
439 | } |
440 | |