1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/mm.h> |
3 | #include <linux/module.h> |
4 | #include <asm/alternative.h> |
5 | #include <asm/cacheflush.h> |
6 | #include <asm/inst.h> |
7 | #include <asm/sections.h> |
8 | |
9 | int __read_mostly alternatives_patched; |
10 | |
11 | EXPORT_SYMBOL_GPL(alternatives_patched); |
12 | |
13 | #define MAX_PATCH_SIZE (((u8)(-1)) / LOONGARCH_INSN_SIZE) |
14 | |
15 | static int __initdata_or_module debug_alternative; |
16 | |
17 | static int __init debug_alt(char *str) |
18 | { |
19 | debug_alternative = 1; |
20 | return 1; |
21 | } |
22 | __setup("debug-alternative" , debug_alt); |
23 | |
24 | #define DPRINTK(fmt, args...) \ |
25 | do { \ |
26 | if (debug_alternative) \ |
27 | printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args); \ |
28 | } while (0) |
29 | |
30 | #define DUMP_WORDS(buf, count, fmt, args...) \ |
31 | do { \ |
32 | if (unlikely(debug_alternative)) { \ |
33 | int _j; \ |
34 | union loongarch_instruction *_buf = buf; \ |
35 | \ |
36 | if (!(count)) \ |
37 | break; \ |
38 | \ |
39 | printk(KERN_DEBUG fmt, ##args); \ |
40 | for (_j = 0; _j < count - 1; _j++) \ |
41 | printk(KERN_CONT "<%08x> ", _buf[_j].word); \ |
42 | printk(KERN_CONT "<%08x>\n", _buf[_j].word); \ |
43 | } \ |
44 | } while (0) |
45 | |
46 | /* Use this to add nops to a buffer, then text_poke the whole buffer. */ |
47 | static void __init_or_module add_nops(union loongarch_instruction *insn, int count) |
48 | { |
49 | while (count--) { |
50 | insn->word = INSN_NOP; |
51 | insn++; |
52 | } |
53 | } |
54 | |
55 | /* Is the jump addr in local .altinstructions */ |
56 | static inline bool in_alt_jump(unsigned long jump, void *start, void *end) |
57 | { |
58 | return jump >= (unsigned long)start && jump < (unsigned long)end; |
59 | } |
60 | |
61 | static void __init_or_module recompute_jump(union loongarch_instruction *buf, |
62 | union loongarch_instruction *dest, union loongarch_instruction *src, |
63 | void *start, void *end) |
64 | { |
65 | unsigned int si, si_l, si_h; |
66 | unsigned long cur_pc, jump_addr, pc; |
67 | long offset; |
68 | |
69 | cur_pc = (unsigned long)src; |
70 | pc = (unsigned long)dest; |
71 | |
72 | si_l = src->reg0i26_format.immediate_l; |
73 | si_h = src->reg0i26_format.immediate_h; |
74 | switch (src->reg0i26_format.opcode) { |
75 | case b_op: |
76 | case bl_op: |
77 | jump_addr = cur_pc + sign_extend64(value: (si_h << 16 | si_l) << 2, index: 27); |
78 | if (in_alt_jump(jump: jump_addr, start, end)) |
79 | return; |
80 | offset = jump_addr - pc; |
81 | BUG_ON(offset < -SZ_128M || offset >= SZ_128M); |
82 | offset >>= 2; |
83 | buf->reg0i26_format.immediate_h = offset >> 16; |
84 | buf->reg0i26_format.immediate_l = offset; |
85 | return; |
86 | } |
87 | |
88 | si_l = src->reg1i21_format.immediate_l; |
89 | si_h = src->reg1i21_format.immediate_h; |
90 | switch (src->reg1i21_format.opcode) { |
91 | case bceqz_op: /* bceqz_op = bcnez_op */ |
92 | BUG_ON(buf->reg1i21_format.rj & BIT(4)); |
93 | fallthrough; |
94 | case beqz_op: |
95 | case bnez_op: |
96 | jump_addr = cur_pc + sign_extend64(value: (si_h << 16 | si_l) << 2, index: 22); |
97 | if (in_alt_jump(jump: jump_addr, start, end)) |
98 | return; |
99 | offset = jump_addr - pc; |
100 | BUG_ON(offset < -SZ_4M || offset >= SZ_4M); |
101 | offset >>= 2; |
102 | buf->reg1i21_format.immediate_h = offset >> 16; |
103 | buf->reg1i21_format.immediate_l = offset; |
104 | return; |
105 | } |
106 | |
107 | si = src->reg2i16_format.immediate; |
108 | switch (src->reg2i16_format.opcode) { |
109 | case beq_op: |
110 | case bne_op: |
111 | case blt_op: |
112 | case bge_op: |
113 | case bltu_op: |
114 | case bgeu_op: |
115 | jump_addr = cur_pc + sign_extend64(value: si << 2, index: 17); |
116 | if (in_alt_jump(jump: jump_addr, start, end)) |
117 | return; |
118 | offset = jump_addr - pc; |
119 | BUG_ON(offset < -SZ_128K || offset >= SZ_128K); |
120 | offset >>= 2; |
121 | buf->reg2i16_format.immediate = offset; |
122 | return; |
123 | } |
124 | } |
125 | |
126 | static int __init_or_module copy_alt_insns(union loongarch_instruction *buf, |
127 | union loongarch_instruction *dest, union loongarch_instruction *src, int nr) |
128 | { |
129 | int i; |
130 | |
131 | for (i = 0; i < nr; i++) { |
132 | buf[i].word = src[i].word; |
133 | |
134 | if (is_pc_ins(&src[i])) { |
135 | pr_err("Not support pcrel instruction at present!" ); |
136 | return -EINVAL; |
137 | } |
138 | |
139 | if (is_branch_ins(&src[i]) && |
140 | src[i].reg2i16_format.opcode != jirl_op) { |
141 | recompute_jump(&buf[i], &dest[i], &src[i], src, src + nr); |
142 | } |
143 | } |
144 | |
145 | return 0; |
146 | } |
147 | |
148 | /* |
149 | * text_poke_early - Update instructions on a live kernel at boot time |
150 | * |
151 | * When you use this code to patch more than one byte of an instruction |
152 | * you need to make sure that other CPUs cannot execute this code in parallel. |
153 | * Also no thread must be currently preempted in the middle of these |
154 | * instructions. And on the local CPU you need to be protected again NMI or MCE |
155 | * handlers seeing an inconsistent instruction while you patch. |
156 | */ |
157 | static void *__init_or_module text_poke_early(union loongarch_instruction *insn, |
158 | union loongarch_instruction *buf, unsigned int nr) |
159 | { |
160 | int i; |
161 | unsigned long flags; |
162 | |
163 | local_irq_save(flags); |
164 | |
165 | for (i = 0; i < nr; i++) |
166 | insn[i].word = buf[i].word; |
167 | |
168 | local_irq_restore(flags); |
169 | |
170 | wbflush(); |
171 | flush_icache_range(start: (unsigned long)insn, end: (unsigned long)(insn + nr)); |
172 | |
173 | return insn; |
174 | } |
175 | |
176 | /* |
177 | * Replace instructions with better alternatives for this CPU type. This runs |
178 | * before SMP is initialized to avoid SMP problems with self modifying code. |
179 | * This implies that asymmetric systems where APs have less capabilities than |
180 | * the boot processor are not handled. Tough. Make sure you disable such |
181 | * features by hand. |
182 | */ |
183 | void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *end) |
184 | { |
185 | struct alt_instr *a; |
186 | unsigned int nr_instr, nr_repl, nr_insnbuf; |
187 | union loongarch_instruction *instr, *replacement; |
188 | union loongarch_instruction insnbuf[MAX_PATCH_SIZE]; |
189 | |
190 | DPRINTK("alt table %px, -> %px" , start, end); |
191 | /* |
192 | * The scan order should be from start to end. A later scanned |
193 | * alternative code can overwrite previously scanned alternative code. |
194 | * Some kernel functions (e.g. memcpy, memset, etc) use this order to |
195 | * patch code. |
196 | * |
197 | * So be careful if you want to change the scan order to any other |
198 | * order. |
199 | */ |
200 | for (a = start; a < end; a++) { |
201 | nr_insnbuf = 0; |
202 | |
203 | instr = (void *)&a->instr_offset + a->instr_offset; |
204 | replacement = (void *)&a->replace_offset + a->replace_offset; |
205 | |
206 | BUG_ON(a->instrlen > sizeof(insnbuf)); |
207 | BUG_ON(a->instrlen & 0x3); |
208 | BUG_ON(a->replacementlen & 0x3); |
209 | |
210 | nr_instr = a->instrlen / LOONGARCH_INSN_SIZE; |
211 | nr_repl = a->replacementlen / LOONGARCH_INSN_SIZE; |
212 | |
213 | if (!cpu_has(a->feature)) { |
214 | DPRINTK("feat not exist: %d, old: (%px len: %d), repl: (%px, len: %d)" , |
215 | a->feature, instr, a->instrlen, |
216 | replacement, a->replacementlen); |
217 | |
218 | continue; |
219 | } |
220 | |
221 | DPRINTK("feat: %d, old: (%px len: %d), repl: (%px, len: %d)" , |
222 | a->feature, instr, a->instrlen, |
223 | replacement, a->replacementlen); |
224 | |
225 | DUMP_WORDS(instr, nr_instr, "%px: old_insn: " , instr); |
226 | DUMP_WORDS(replacement, nr_repl, "%px: rpl_insn: " , replacement); |
227 | |
228 | copy_alt_insns(buf: insnbuf, dest: instr, src: replacement, nr: nr_repl); |
229 | nr_insnbuf = nr_repl; |
230 | |
231 | if (nr_instr > nr_repl) { |
232 | add_nops(insn: insnbuf + nr_repl, count: nr_instr - nr_repl); |
233 | nr_insnbuf += nr_instr - nr_repl; |
234 | } |
235 | DUMP_WORDS(insnbuf, nr_insnbuf, "%px: final_insn: " , instr); |
236 | |
237 | text_poke_early(insn: instr, buf: insnbuf, nr: nr_insnbuf); |
238 | } |
239 | } |
240 | |
241 | void __init alternative_instructions(void) |
242 | { |
243 | apply_alternatives(start: __alt_instructions, end: __alt_instructions_end); |
244 | |
245 | alternatives_patched = 1; |
246 | } |
247 | |