1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * alternative runtime patching |
4 | * inspired by the x86 version |
5 | * |
6 | * Copyright (C) 2014 ARM Ltd. |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) "alternatives: " fmt |
10 | |
11 | #include <linux/init.h> |
12 | #include <linux/cpu.h> |
13 | #include <linux/elf.h> |
14 | #include <asm/cacheflush.h> |
15 | #include <asm/alternative.h> |
16 | #include <asm/cpufeature.h> |
17 | #include <asm/insn.h> |
18 | #include <asm/module.h> |
19 | #include <asm/sections.h> |
20 | #include <asm/vdso.h> |
21 | #include <linux/stop_machine.h> |
22 | |
23 | #define __ALT_PTR(a, f) ((void *)&(a)->f + (a)->f) |
24 | #define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset) |
25 | #define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset) |
26 | |
27 | #define ALT_CAP(a) ((a)->cpucap & ~ARM64_CB_BIT) |
28 | #define ALT_HAS_CB(a) ((a)->cpucap & ARM64_CB_BIT) |
29 | |
30 | /* Volatile, as we may be patching the guts of READ_ONCE() */ |
31 | static volatile int all_alternatives_applied; |
32 | |
33 | static DECLARE_BITMAP(applied_alternatives, ARM64_NCAPS); |
34 | |
35 | struct alt_region { |
36 | struct alt_instr *begin; |
37 | struct alt_instr *end; |
38 | }; |
39 | |
40 | bool alternative_is_applied(u16 cpucap) |
41 | { |
42 | if (WARN_ON(cpucap >= ARM64_NCAPS)) |
43 | return false; |
44 | |
45 | return test_bit(cpucap, applied_alternatives); |
46 | } |
47 | |
48 | /* |
49 | * Check if the target PC is within an alternative block. |
50 | */ |
51 | static __always_inline bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc) |
52 | { |
53 | unsigned long replptr = (unsigned long)ALT_REPL_PTR(alt); |
54 | return !(pc >= replptr && pc <= (replptr + alt->alt_len)); |
55 | } |
56 | |
57 | #define align_down(x, a) ((unsigned long)(x) & ~(((unsigned long)(a)) - 1)) |
58 | |
59 | static __always_inline u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr) |
60 | { |
61 | u32 insn; |
62 | |
63 | insn = le32_to_cpu(*altinsnptr); |
64 | |
65 | if (aarch64_insn_is_branch_imm(insn)) { |
66 | s32 offset = aarch64_get_branch_offset(insn); |
67 | unsigned long target; |
68 | |
69 | target = (unsigned long)altinsnptr + offset; |
70 | |
71 | /* |
72 | * If we're branching inside the alternate sequence, |
73 | * do not rewrite the instruction, as it is already |
74 | * correct. Otherwise, generate the new instruction. |
75 | */ |
76 | if (branch_insn_requires_update(alt, pc: target)) { |
77 | offset = target - (unsigned long)insnptr; |
78 | insn = aarch64_set_branch_offset(insn, offset); |
79 | } |
80 | } else if (aarch64_insn_is_adrp(insn)) { |
81 | s32 orig_offset, new_offset; |
82 | unsigned long target; |
83 | |
84 | /* |
85 | * If we're replacing an adrp instruction, which uses PC-relative |
86 | * immediate addressing, adjust the offset to reflect the new |
87 | * PC. adrp operates on 4K aligned addresses. |
88 | */ |
89 | orig_offset = aarch64_insn_adrp_get_offset(insn); |
90 | target = align_down(altinsnptr, SZ_4K) + orig_offset; |
91 | new_offset = target - align_down(insnptr, SZ_4K); |
92 | insn = aarch64_insn_adrp_set_offset(insn, new_offset); |
93 | } else if (aarch64_insn_uses_literal(insn)) { |
94 | /* |
95 | * Disallow patching unhandled instructions using PC relative |
96 | * literal addresses |
97 | */ |
98 | BUG(); |
99 | } |
100 | |
101 | return insn; |
102 | } |
103 | |
104 | static noinstr void patch_alternative(struct alt_instr *alt, |
105 | __le32 *origptr, __le32 *updptr, int nr_inst) |
106 | { |
107 | __le32 *replptr; |
108 | int i; |
109 | |
110 | replptr = ALT_REPL_PTR(alt); |
111 | for (i = 0; i < nr_inst; i++) { |
112 | u32 insn; |
113 | |
114 | insn = get_alt_insn(alt, insnptr: origptr + i, altinsnptr: replptr + i); |
115 | updptr[i] = cpu_to_le32(insn); |
116 | } |
117 | } |
118 | |
119 | /* |
120 | * We provide our own, private D-cache cleaning function so that we don't |
121 | * accidentally call into the cache.S code, which is patched by us at |
122 | * runtime. |
123 | */ |
124 | static noinstr void clean_dcache_range_nopatch(u64 start, u64 end) |
125 | { |
126 | u64 cur, d_size, ctr_el0; |
127 | |
128 | ctr_el0 = arm64_ftr_reg_ctrel0.sys_val; |
129 | d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0, |
130 | CTR_EL0_DminLine_SHIFT); |
131 | cur = start & ~(d_size - 1); |
132 | do { |
133 | /* |
134 | * We must clean+invalidate to the PoC in order to avoid |
135 | * Cortex-A53 errata 826319, 827319, 824069 and 819472 |
136 | * (this corresponds to ARM64_WORKAROUND_CLEAN_CACHE) |
137 | */ |
138 | asm volatile("dc civac, %0" : : "r" (cur) : "memory" ); |
139 | } while (cur += d_size, cur < end); |
140 | } |
141 | |
142 | static void __apply_alternatives(const struct alt_region *region, |
143 | bool is_module, |
144 | unsigned long *cpucap_mask) |
145 | { |
146 | struct alt_instr *alt; |
147 | __le32 *origptr, *updptr; |
148 | alternative_cb_t alt_cb; |
149 | |
150 | for (alt = region->begin; alt < region->end; alt++) { |
151 | int nr_inst; |
152 | int cap = ALT_CAP(alt); |
153 | |
154 | if (!test_bit(cap, cpucap_mask)) |
155 | continue; |
156 | |
157 | if (!cpus_have_cap(cap)) |
158 | continue; |
159 | |
160 | if (ALT_HAS_CB(alt)) |
161 | BUG_ON(alt->alt_len != 0); |
162 | else |
163 | BUG_ON(alt->alt_len != alt->orig_len); |
164 | |
165 | origptr = ALT_ORIG_PTR(alt); |
166 | updptr = is_module ? origptr : lm_alias(origptr); |
167 | nr_inst = alt->orig_len / AARCH64_INSN_SIZE; |
168 | |
169 | if (ALT_HAS_CB(alt)) |
170 | alt_cb = ALT_REPL_PTR(alt); |
171 | else |
172 | alt_cb = patch_alternative; |
173 | |
174 | alt_cb(alt, origptr, updptr, nr_inst); |
175 | |
176 | if (!is_module) { |
177 | clean_dcache_range_nopatch(start: (u64)origptr, |
178 | end: (u64)(origptr + nr_inst)); |
179 | } |
180 | } |
181 | |
182 | /* |
183 | * The core module code takes care of cache maintenance in |
184 | * flush_module_icache(). |
185 | */ |
186 | if (!is_module) { |
187 | dsb(ish); |
188 | icache_inval_all_pou(); |
189 | isb(); |
190 | |
191 | bitmap_or(dst: applied_alternatives, src1: applied_alternatives, |
192 | src2: cpucap_mask, nbits: ARM64_NCAPS); |
193 | bitmap_and(dst: applied_alternatives, src1: applied_alternatives, |
194 | src2: system_cpucaps, nbits: ARM64_NCAPS); |
195 | } |
196 | } |
197 | |
198 | static void __init apply_alternatives_vdso(void) |
199 | { |
200 | struct alt_region region; |
201 | const struct elf64_hdr *hdr; |
202 | const struct elf64_shdr *shdr; |
203 | const struct elf64_shdr *alt; |
204 | DECLARE_BITMAP(all_capabilities, ARM64_NCAPS); |
205 | |
206 | bitmap_fill(all_capabilities, ARM64_NCAPS); |
207 | |
208 | hdr = (struct elf64_hdr *)vdso_start; |
209 | shdr = (void *)hdr + hdr->e_shoff; |
210 | alt = find_section(hdr, shdr, ".altinstructions" ); |
211 | if (!alt) |
212 | return; |
213 | |
214 | region = (struct alt_region){ |
215 | .begin = (void *)hdr + alt->sh_offset, |
216 | .end = (void *)hdr + alt->sh_offset + alt->sh_size, |
217 | }; |
218 | |
219 | __apply_alternatives(region: ®ion, is_module: false, cpucap_mask: &all_capabilities[0]); |
220 | } |
221 | |
222 | static const struct alt_region kernel_alternatives __initconst = { |
223 | .begin = (struct alt_instr *)__alt_instructions, |
224 | .end = (struct alt_instr *)__alt_instructions_end, |
225 | }; |
226 | |
227 | /* |
228 | * We might be patching the stop_machine state machine, so implement a |
229 | * really simple polling protocol here. |
230 | */ |
231 | static int __init __apply_alternatives_multi_stop(void *unused) |
232 | { |
233 | /* We always have a CPU 0 at this point (__init) */ |
234 | if (smp_processor_id()) { |
235 | while (!all_alternatives_applied) |
236 | cpu_relax(); |
237 | isb(); |
238 | } else { |
239 | DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS); |
240 | |
241 | bitmap_complement(remaining_capabilities, boot_cpucaps, |
242 | ARM64_NCAPS); |
243 | |
244 | BUG_ON(all_alternatives_applied); |
245 | __apply_alternatives(region: &kernel_alternatives, is_module: false, |
246 | cpucap_mask: remaining_capabilities); |
247 | /* Barriers provided by the cache flushing */ |
248 | all_alternatives_applied = 1; |
249 | } |
250 | |
251 | return 0; |
252 | } |
253 | |
254 | void __init apply_alternatives_all(void) |
255 | { |
256 | pr_info("applying system-wide alternatives\n" ); |
257 | |
258 | apply_alternatives_vdso(); |
259 | /* better not try code patching on a live SMP system */ |
260 | stop_machine(fn: __apply_alternatives_multi_stop, NULL, cpu_online_mask); |
261 | } |
262 | |
263 | /* |
264 | * This is called very early in the boot process (directly after we run |
265 | * a feature detect on the boot CPU). No need to worry about other CPUs |
266 | * here. |
267 | */ |
268 | void __init apply_boot_alternatives(void) |
269 | { |
270 | /* If called on non-boot cpu things could go wrong */ |
271 | WARN_ON(smp_processor_id() != 0); |
272 | |
273 | pr_info("applying boot alternatives\n" ); |
274 | |
275 | __apply_alternatives(&kernel_alternatives, false, |
276 | &boot_cpucaps[0]); |
277 | } |
278 | |
279 | #ifdef CONFIG_MODULES |
280 | void apply_alternatives_module(void *start, size_t length) |
281 | { |
282 | struct alt_region region = { |
283 | .begin = start, |
284 | .end = start + length, |
285 | }; |
286 | DECLARE_BITMAP(all_capabilities, ARM64_NCAPS); |
287 | |
288 | bitmap_fill(all_capabilities, ARM64_NCAPS); |
289 | |
290 | __apply_alternatives(region: ®ion, is_module: true, cpucap_mask: &all_capabilities[0]); |
291 | } |
292 | #endif |
293 | |
294 | noinstr void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr, |
295 | __le32 *updptr, int nr_inst) |
296 | { |
297 | for (int i = 0; i < nr_inst; i++) |
298 | updptr[i] = cpu_to_le32(aarch64_insn_gen_nop()); |
299 | } |
300 | EXPORT_SYMBOL(alt_cb_patch_nops); |
301 | |