1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kernel.h>
3#include <linux/mm.h>
4#include <linux/smp.h>
5#include <linux/spinlock.h>
6#include <linux/stop_machine.h>
7#include <linux/uaccess.h>
8
9#include <asm/cacheflush.h>
10#include <asm/fixmap.h>
11#include <asm/insn.h>
12#include <asm/kprobes.h>
13#include <asm/patching.h>
14#include <asm/sections.h>
15
16static DEFINE_RAW_SPINLOCK(patch_lock);
17
18static bool is_exit_text(unsigned long addr)
19{
20 /* discarded with init text/data */
21 return system_state < SYSTEM_RUNNING &&
22 addr >= (unsigned long)__exittext_begin &&
23 addr < (unsigned long)__exittext_end;
24}
25
26static bool is_image_text(unsigned long addr)
27{
28 return core_kernel_text(addr) || is_exit_text(addr);
29}
30
31static void __kprobes *patch_map(void *addr, int fixmap)
32{
33 unsigned long uintaddr = (uintptr_t) addr;
34 bool image = is_image_text(addr: uintaddr);
35 struct page *page;
36
37 if (image)
38 page = phys_to_page(__pa_symbol(addr));
39 else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
40 page = vmalloc_to_page(addr);
41 else
42 return addr;
43
44 BUG_ON(!page);
45 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
46 (uintaddr & ~PAGE_MASK));
47}
48
49static void __kprobes patch_unmap(int fixmap)
50{
51 clear_fixmap(fixmap);
52}
53/*
54 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
55 * little-endian.
56 */
57int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
58{
59 int ret;
60 __le32 val;
61
62 ret = copy_from_kernel_nofault(dst: &val, src: addr, size: AARCH64_INSN_SIZE);
63 if (!ret)
64 *insnp = le32_to_cpu(val);
65
66 return ret;
67}
68
69static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
70{
71 void *waddr = addr;
72 unsigned long flags = 0;
73 int ret;
74
75 raw_spin_lock_irqsave(&patch_lock, flags);
76 waddr = patch_map(addr, fixmap: FIX_TEXT_POKE0);
77
78 ret = copy_to_kernel_nofault(dst: waddr, src: &insn, size: AARCH64_INSN_SIZE);
79
80 patch_unmap(fixmap: FIX_TEXT_POKE0);
81 raw_spin_unlock_irqrestore(&patch_lock, flags);
82
83 return ret;
84}
85
86int __kprobes aarch64_insn_write(void *addr, u32 insn)
87{
88 return __aarch64_insn_write(addr, cpu_to_le32(insn));
89}
90
91noinstr int aarch64_insn_write_literal_u64(void *addr, u64 val)
92{
93 u64 *waddr;
94 unsigned long flags;
95 int ret;
96
97 raw_spin_lock_irqsave(&patch_lock, flags);
98 waddr = patch_map(addr, fixmap: FIX_TEXT_POKE0);
99
100 ret = copy_to_kernel_nofault(dst: waddr, src: &val, size: sizeof(val));
101
102 patch_unmap(fixmap: FIX_TEXT_POKE0);
103 raw_spin_unlock_irqrestore(&patch_lock, flags);
104
105 return ret;
106}
107
108typedef void text_poke_f(void *dst, void *src, size_t patched, size_t len);
109
110static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len)
111{
112 unsigned long flags;
113 size_t patched = 0;
114 size_t size;
115 void *waddr;
116 void *ptr;
117
118 raw_spin_lock_irqsave(&patch_lock, flags);
119
120 while (patched < len) {
121 ptr = addr + patched;
122 size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr),
123 len - patched);
124
125 waddr = patch_map(addr: ptr, fixmap: FIX_TEXT_POKE0);
126 func(waddr, src, patched, size);
127 patch_unmap(fixmap: FIX_TEXT_POKE0);
128
129 patched += size;
130 }
131 raw_spin_unlock_irqrestore(&patch_lock, flags);
132
133 flush_icache_range(start: (uintptr_t)addr, end: (uintptr_t)addr + len);
134
135 return addr;
136}
137
138static void text_poke_memcpy(void *dst, void *src, size_t patched, size_t len)
139{
140 copy_to_kernel_nofault(dst, src: src + patched, size: len);
141}
142
143static void text_poke_memset(void *dst, void *src, size_t patched, size_t len)
144{
145 u32 c = *(u32 *)src;
146
147 memset32(s: dst, v: c, n: len / 4);
148}
149
150/**
151 * aarch64_insn_copy - Copy instructions into (an unused part of) RX memory
152 * @dst: address to modify
153 * @src: source of the copy
154 * @len: length to copy
155 *
156 * Useful for JITs to dump new code blocks into unused regions of RX memory.
157 */
158noinstr void *aarch64_insn_copy(void *dst, void *src, size_t len)
159{
160 /* A64 instructions must be word aligned */
161 if ((uintptr_t)dst & 0x3)
162 return NULL;
163
164 return __text_poke(func: text_poke_memcpy, addr: dst, src, len);
165}
166
167/**
168 * aarch64_insn_set - memset for RX memory regions.
169 * @dst: address to modify
170 * @insn: value to set
171 * @len: length of memory region.
172 *
173 * Useful for JITs to fill regions of RX memory with illegal instructions.
174 */
175noinstr void *aarch64_insn_set(void *dst, u32 insn, size_t len)
176{
177 if ((uintptr_t)dst & 0x3)
178 return NULL;
179
180 return __text_poke(func: text_poke_memset, addr: dst, src: &insn, len);
181}
182
183int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
184{
185 u32 *tp = addr;
186 int ret;
187
188 /* A64 instructions must be word aligned */
189 if ((uintptr_t)tp & 0x3)
190 return -EINVAL;
191
192 ret = aarch64_insn_write(addr: tp, insn);
193 if (ret == 0)
194 caches_clean_inval_pou((uintptr_t)tp,
195 (uintptr_t)tp + AARCH64_INSN_SIZE);
196
197 return ret;
198}
199
200struct aarch64_insn_patch {
201 void **text_addrs;
202 u32 *new_insns;
203 int insn_cnt;
204 atomic_t cpu_count;
205};
206
207static int __kprobes aarch64_insn_patch_text_cb(void *arg)
208{
209 int i, ret = 0;
210 struct aarch64_insn_patch *pp = arg;
211
212 /* The last CPU becomes master */
213 if (atomic_inc_return(v: &pp->cpu_count) == num_online_cpus()) {
214 for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
215 ret = aarch64_insn_patch_text_nosync(addr: pp->text_addrs[i],
216 insn: pp->new_insns[i]);
217 /* Notify other processors with an additional increment. */
218 atomic_inc(v: &pp->cpu_count);
219 } else {
220 while (atomic_read(v: &pp->cpu_count) <= num_online_cpus())
221 cpu_relax();
222 isb();
223 }
224
225 return ret;
226}
227
228int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
229{
230 struct aarch64_insn_patch patch = {
231 .text_addrs = addrs,
232 .new_insns = insns,
233 .insn_cnt = cnt,
234 .cpu_count = ATOMIC_INIT(0),
235 };
236
237 if (cnt <= 0)
238 return -EINVAL;
239
240 return stop_machine_cpuslocked(fn: aarch64_insn_patch_text_cb, data: &patch,
241 cpu_online_mask);
242}
243

source code of linux/arch/arm64/kernel/patching.c