| 1 | // SPDX-License-Identifier: GPL-2.0 |
|---|
| 2 | |
|---|
| 3 | #define DISABLE_BRANCH_PROFILING |
|---|
| 4 | |
|---|
| 5 | #include <linux/kasan.h> |
|---|
| 6 | #include <linux/memblock.h> |
|---|
| 7 | #include <linux/hugetlb.h> |
|---|
| 8 | |
|---|
| 9 | #include <asm/pgalloc.h> |
|---|
| 10 | |
|---|
| 11 | static int __init |
|---|
| 12 | kasan_init_shadow_8M(unsigned long k_start, unsigned long k_end, void *block) |
|---|
| 13 | { |
|---|
| 14 | pmd_t *pmd = pmd_off_k(va: k_start); |
|---|
| 15 | unsigned long k_cur, k_next; |
|---|
| 16 | |
|---|
| 17 | for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++, block += SZ_4M) { |
|---|
| 18 | pte_t *ptep; |
|---|
| 19 | int i; |
|---|
| 20 | |
|---|
| 21 | k_next = pgd_addr_end(k_cur, k_end); |
|---|
| 22 | if ((void *)pmd_page_vaddr(pmd: *pmd) != kasan_early_shadow_pte) |
|---|
| 23 | continue; |
|---|
| 24 | |
|---|
| 25 | ptep = memblock_alloc(size: PTE_FRAG_SIZE, align: PTE_FRAG_SIZE); |
|---|
| 26 | if (!ptep) |
|---|
| 27 | return -ENOMEM; |
|---|
| 28 | |
|---|
| 29 | for (i = 0; i < PTRS_PER_PTE; i++) { |
|---|
| 30 | pte_t pte = pte_mkhuge(pte: pfn_pte(PHYS_PFN(__pa(block + i * PAGE_SIZE)), PAGE_KERNEL)); |
|---|
| 31 | |
|---|
| 32 | __set_pte_at(&init_mm, k_cur, ptep + i, pte, 1); |
|---|
| 33 | } |
|---|
| 34 | pmd_populate_kernel(mm: &init_mm, pmd, pte: ptep); |
|---|
| 35 | *pmd = __pmd(val: pmd_val(pmd: *pmd) | _PMD_PAGE_8M); |
|---|
| 36 | } |
|---|
| 37 | return 0; |
|---|
| 38 | } |
|---|
| 39 | |
|---|
| 40 | int __init kasan_init_region(void *start, size_t size) |
|---|
| 41 | { |
|---|
| 42 | unsigned long k_start = (unsigned long)kasan_mem_to_shadow(addr: start); |
|---|
| 43 | unsigned long k_end = (unsigned long)kasan_mem_to_shadow(addr: start + size); |
|---|
| 44 | unsigned long k_cur; |
|---|
| 45 | int ret; |
|---|
| 46 | void *block; |
|---|
| 47 | |
|---|
| 48 | block = memblock_alloc(size: k_end - k_start, SZ_8M); |
|---|
| 49 | if (!block) |
|---|
| 50 | return -ENOMEM; |
|---|
| 51 | |
|---|
| 52 | if (IS_ALIGNED(k_start, SZ_8M)) { |
|---|
| 53 | kasan_init_shadow_8M(k_start, ALIGN_DOWN(k_end, SZ_8M), block); |
|---|
| 54 | k_cur = ALIGN_DOWN(k_end, SZ_8M); |
|---|
| 55 | if (k_cur == k_end) |
|---|
| 56 | goto finish; |
|---|
| 57 | } else { |
|---|
| 58 | k_cur = k_start; |
|---|
| 59 | } |
|---|
| 60 | |
|---|
| 61 | ret = kasan_init_shadow_page_tables(k_start, k_end); |
|---|
| 62 | if (ret) |
|---|
| 63 | return ret; |
|---|
| 64 | |
|---|
| 65 | for (; k_cur < k_end; k_cur += PAGE_SIZE) { |
|---|
| 66 | pmd_t *pmd = pmd_off_k(va: k_cur); |
|---|
| 67 | void *va = block + k_cur - k_start; |
|---|
| 68 | pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL); |
|---|
| 69 | |
|---|
| 70 | if (k_cur < ALIGN_DOWN(k_end, SZ_512K)) |
|---|
| 71 | pte = pte_mkhuge(pte); |
|---|
| 72 | |
|---|
| 73 | __set_pte_at(&init_mm, k_cur, pte_offset_kernel(pmd, address: k_cur), pte, 0); |
|---|
| 74 | } |
|---|
| 75 | finish: |
|---|
| 76 | flush_tlb_kernel_range(start: k_start, end: k_end); |
|---|
| 77 | return 0; |
|---|
| 78 | } |
|---|
| 79 | |
|---|
