1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/kernel.h> |
3 | #include <linux/spinlock.h> |
4 | #include <linux/kprobes.h> |
5 | #include <linux/mm.h> |
6 | #include <linux/stop_machine.h> |
7 | |
8 | #include <asm/cacheflush.h> |
9 | #include <asm/fixmap.h> |
10 | #include <asm/smp_plat.h> |
11 | #include <asm/opcodes.h> |
12 | #include <asm/patch.h> |
13 | |
14 | struct patch { |
15 | void *addr; |
16 | unsigned int insn; |
17 | }; |
18 | |
19 | #ifdef CONFIG_MMU |
20 | static DEFINE_RAW_SPINLOCK(patch_lock); |
21 | |
22 | static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags) |
23 | { |
24 | unsigned int uintaddr = (uintptr_t) addr; |
25 | bool module = !core_kernel_text(addr: uintaddr); |
26 | struct page *page; |
27 | |
28 | if (module && IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) |
29 | page = vmalloc_to_page(addr); |
30 | else if (!module && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) |
31 | page = virt_to_page(addr); |
32 | else |
33 | return addr; |
34 | |
35 | if (flags) |
36 | raw_spin_lock_irqsave(&patch_lock, *flags); |
37 | |
38 | set_fixmap(fixmap, page_to_phys(page)); |
39 | |
40 | return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK)); |
41 | } |
42 | |
43 | static void __kprobes patch_unmap(int fixmap, unsigned long *flags) |
44 | { |
45 | clear_fixmap(fixmap); |
46 | |
47 | if (flags) |
48 | raw_spin_unlock_irqrestore(&patch_lock, *flags); |
49 | } |
50 | #else |
51 | static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags) |
52 | { |
53 | return addr; |
54 | } |
55 | static void __kprobes patch_unmap(int fixmap, unsigned long *flags) { } |
56 | #endif |
57 | |
58 | void __kprobes __patch_text_real(void *addr, unsigned int insn, bool remap) |
59 | { |
60 | bool thumb2 = IS_ENABLED(CONFIG_THUMB2_KERNEL); |
61 | unsigned int uintaddr = (uintptr_t) addr; |
62 | bool twopage = false; |
63 | unsigned long flags; |
64 | void *waddr = addr; |
65 | int size; |
66 | |
67 | if (remap) |
68 | waddr = patch_map(addr, fixmap: FIX_TEXT_POKE0, flags: &flags); |
69 | |
70 | if (thumb2 && __opcode_is_thumb16(insn)) { |
71 | *(u16 *)waddr = __opcode_to_mem_thumb16(insn); |
72 | size = sizeof(u16); |
73 | } else if (thumb2 && (uintaddr & 2)) { |
74 | u16 first = __opcode_thumb32_first(insn); |
75 | u16 second = __opcode_thumb32_second(insn); |
76 | u16 *addrh0 = waddr; |
77 | u16 *addrh1 = waddr + 2; |
78 | |
79 | twopage = (uintaddr & ~PAGE_MASK) == PAGE_SIZE - 2; |
80 | if (twopage && remap) |
81 | addrh1 = patch_map(addr: addr + 2, fixmap: FIX_TEXT_POKE1, NULL); |
82 | |
83 | *addrh0 = __opcode_to_mem_thumb16(first); |
84 | *addrh1 = __opcode_to_mem_thumb16(second); |
85 | |
86 | if (twopage && addrh1 != addr + 2) { |
87 | flush_kernel_vmap_range(addrh1, 2); |
88 | patch_unmap(fixmap: FIX_TEXT_POKE1, NULL); |
89 | } |
90 | |
91 | size = sizeof(u32); |
92 | } else { |
93 | if (thumb2) |
94 | insn = __opcode_to_mem_thumb32(insn); |
95 | else |
96 | insn = __opcode_to_mem_arm(insn); |
97 | |
98 | *(u32 *)waddr = insn; |
99 | size = sizeof(u32); |
100 | } |
101 | |
102 | if (waddr != addr) { |
103 | flush_kernel_vmap_range(waddr, twopage ? size / 2 : size); |
104 | patch_unmap(fixmap: FIX_TEXT_POKE0, flags: &flags); |
105 | } |
106 | |
107 | flush_icache_range(start: (uintptr_t)(addr), |
108 | end: (uintptr_t)(addr) + size); |
109 | } |
110 | |
111 | static int __kprobes patch_text_stop_machine(void *data) |
112 | { |
113 | struct patch *patch = data; |
114 | |
115 | __patch_text(patch->addr, patch->insn); |
116 | |
117 | return 0; |
118 | } |
119 | |
120 | void __kprobes patch_text(void *addr, unsigned int insn) |
121 | { |
122 | struct patch patch = { |
123 | .addr = addr, |
124 | .insn = insn, |
125 | }; |
126 | |
127 | stop_machine_cpuslocked(fn: patch_text_stop_machine, data: &patch, NULL); |
128 | } |
129 | |