1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | // based on arch/arm/mm/alignment.c |
3 | |
4 | #include <linux/compiler.h> |
5 | #include <linux/errno.h> |
6 | #include <linux/kernel.h> |
7 | #include <linux/init.h> |
8 | #include <linux/perf_event.h> |
9 | #include <linux/uaccess.h> |
10 | |
11 | #include <asm/exception.h> |
12 | #include <asm/ptrace.h> |
13 | #include <asm/traps.h> |
14 | |
15 | /* |
16 | * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998 |
17 | * |
18 | * Speed optimisations and better fault handling by Russell King. |
19 | */ |
20 | #define CODING_BITS(i) (i & 0x0e000000) |
21 | |
22 | #define LDST_P_BIT(i) (i & (1 << 24)) /* Preindex */ |
23 | #define LDST_U_BIT(i) (i & (1 << 23)) /* Add offset */ |
24 | #define LDST_W_BIT(i) (i & (1 << 21)) /* Writeback */ |
25 | #define LDST_L_BIT(i) (i & (1 << 20)) /* Load */ |
26 | |
27 | #define LDST_P_EQ_U(i) ((((i) ^ ((i) >> 1)) & (1 << 23)) == 0) |
28 | |
29 | #define LDSTHD_I_BIT(i) (i & (1 << 22)) /* double/half-word immed */ |
30 | |
31 | #define RN_BITS(i) ((i >> 16) & 15) /* Rn */ |
32 | #define RD_BITS(i) ((i >> 12) & 15) /* Rd */ |
33 | #define RM_BITS(i) (i & 15) /* Rm */ |
34 | |
35 | #define REGMASK_BITS(i) (i & 0xffff) |
36 | |
37 | #define BAD_INSTR 0xdeadc0de |
38 | |
39 | /* Thumb-2 32 bit format per ARMv7 DDI0406A A6.3, either f800h,e800h,f800h */ |
40 | #define IS_T32(hi16) \ |
41 | (((hi16) & 0xe000) == 0xe000 && ((hi16) & 0x1800)) |
42 | |
43 | union offset_union { |
44 | unsigned long un; |
45 | signed long sn; |
46 | }; |
47 | |
48 | #define TYPE_ERROR 0 |
49 | #define TYPE_FAULT 1 |
50 | #define TYPE_LDST 2 |
51 | #define TYPE_DONE 3 |
52 | |
53 | static void |
54 | do_alignment_finish_ldst(unsigned long addr, u32 instr, struct pt_regs *regs, |
55 | union offset_union offset) |
56 | { |
57 | if (!LDST_U_BIT(instr)) |
58 | offset.un = -offset.un; |
59 | |
60 | if (!LDST_P_BIT(instr)) |
61 | addr += offset.un; |
62 | |
63 | if (!LDST_P_BIT(instr) || LDST_W_BIT(instr)) |
64 | regs->regs[RN_BITS(instr)] = addr; |
65 | } |
66 | |
67 | static int |
68 | do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs) |
69 | { |
70 | unsigned int rd = RD_BITS(instr); |
71 | unsigned int rd2; |
72 | int load; |
73 | |
74 | if ((instr & 0xfe000000) == 0xe8000000) { |
75 | /* ARMv7 Thumb-2 32-bit LDRD/STRD */ |
76 | rd2 = (instr >> 8) & 0xf; |
77 | load = !!(LDST_L_BIT(instr)); |
78 | } else if (((rd & 1) == 1) || (rd == 14)) { |
79 | return TYPE_ERROR; |
80 | } else { |
81 | load = ((instr & 0xf0) == 0xd0); |
82 | rd2 = rd + 1; |
83 | } |
84 | |
85 | if (load) { |
86 | unsigned int val, val2; |
87 | |
88 | if (get_user(val, (u32 __user *)addr) || |
89 | get_user(val2, (u32 __user *)(addr + 4))) |
90 | return TYPE_FAULT; |
91 | regs->regs[rd] = val; |
92 | regs->regs[rd2] = val2; |
93 | } else { |
94 | if (put_user(regs->regs[rd], (u32 __user *)addr) || |
95 | put_user(regs->regs[rd2], (u32 __user *)(addr + 4))) |
96 | return TYPE_FAULT; |
97 | } |
98 | return TYPE_LDST; |
99 | } |
100 | |
101 | /* |
102 | * LDM/STM alignment handler. |
103 | * |
104 | * There are 4 variants of this instruction: |
105 | * |
106 | * B = rn pointer before instruction, A = rn pointer after instruction |
107 | * ------ increasing address -----> |
108 | * | | r0 | r1 | ... | rx | | |
109 | * PU = 01 B A |
110 | * PU = 11 B A |
111 | * PU = 00 A B |
112 | * PU = 10 A B |
113 | */ |
114 | static int |
115 | do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs) |
116 | { |
117 | unsigned int rd, rn, nr_regs, regbits; |
118 | unsigned long eaddr, newaddr; |
119 | unsigned int val; |
120 | |
121 | /* count the number of registers in the mask to be transferred */ |
122 | nr_regs = hweight16(REGMASK_BITS(instr)) * 4; |
123 | |
124 | rn = RN_BITS(instr); |
125 | newaddr = eaddr = regs->regs[rn]; |
126 | |
127 | if (!LDST_U_BIT(instr)) |
128 | nr_regs = -nr_regs; |
129 | newaddr += nr_regs; |
130 | if (!LDST_U_BIT(instr)) |
131 | eaddr = newaddr; |
132 | |
133 | if (LDST_P_EQ_U(instr)) /* U = P */ |
134 | eaddr += 4; |
135 | |
136 | for (regbits = REGMASK_BITS(instr), rd = 0; regbits; |
137 | regbits >>= 1, rd += 1) |
138 | if (regbits & 1) { |
139 | if (LDST_L_BIT(instr)) { |
140 | if (get_user(val, (u32 __user *)eaddr)) |
141 | return TYPE_FAULT; |
142 | if (rd < 15) |
143 | regs->regs[rd] = val; |
144 | else |
145 | regs->pc = val; |
146 | } else { |
147 | /* |
148 | * The PC register has a bias of +8 in ARM mode |
149 | * and +4 in Thumb mode. This means that a read |
150 | * of the value of PC should account for this. |
151 | * Since Thumb does not permit STM instructions |
152 | * to refer to PC, just add 8 here. |
153 | */ |
154 | val = (rd < 15) ? regs->regs[rd] : regs->pc + 8; |
155 | if (put_user(val, (u32 __user *)eaddr)) |
156 | return TYPE_FAULT; |
157 | } |
158 | eaddr += 4; |
159 | } |
160 | |
161 | if (LDST_W_BIT(instr)) |
162 | regs->regs[rn] = newaddr; |
163 | |
164 | return TYPE_DONE; |
165 | } |
166 | |
167 | /* |
168 | * Convert Thumb multi-word load/store instruction forms to equivalent ARM |
169 | * instructions so we can reuse ARM userland alignment fault fixups for Thumb. |
170 | * |
171 | * This implementation was initially based on the algorithm found in |
172 | * gdb/sim/arm/thumbemu.c. It is basically just a code reduction of same |
173 | * to convert only Thumb ld/st instruction forms to equivalent ARM forms. |
174 | * |
175 | * NOTES: |
176 | * 1. Comments below refer to ARM ARM DDI0100E Thumb Instruction sections. |
177 | * 2. If for some reason we're passed an non-ld/st Thumb instruction to |
178 | * decode, we return 0xdeadc0de. This should never happen under normal |
179 | * circumstances but if it does, we've got other problems to deal with |
180 | * elsewhere and we obviously can't fix those problems here. |
181 | */ |
182 | |
183 | static unsigned long thumb2arm(u16 tinstr) |
184 | { |
185 | u32 L = (tinstr & (1<<11)) >> 11; |
186 | |
187 | switch ((tinstr & 0xf800) >> 11) { |
188 | /* 6.6.1 Format 1: */ |
189 | case 0xc000 >> 11: /* 7.1.51 STMIA */ |
190 | case 0xc800 >> 11: /* 7.1.25 LDMIA */ |
191 | { |
192 | u32 Rn = (tinstr & (7<<8)) >> 8; |
193 | u32 W = ((L<<Rn) & (tinstr&255)) ? 0 : 1<<21; |
194 | |
195 | return 0xe8800000 | W | (L<<20) | (Rn<<16) | |
196 | (tinstr&255); |
197 | } |
198 | |
199 | /* 6.6.1 Format 2: */ |
200 | case 0xb000 >> 11: /* 7.1.48 PUSH */ |
201 | case 0xb800 >> 11: /* 7.1.47 POP */ |
202 | if ((tinstr & (3 << 9)) == 0x0400) { |
203 | static const u32 subset[4] = { |
204 | 0xe92d0000, /* STMDB sp!,{registers} */ |
205 | 0xe92d4000, /* STMDB sp!,{registers,lr} */ |
206 | 0xe8bd0000, /* LDMIA sp!,{registers} */ |
207 | 0xe8bd8000 /* LDMIA sp!,{registers,pc} */ |
208 | }; |
209 | return subset[(L<<1) | ((tinstr & (1<<8)) >> 8)] | |
210 | (tinstr & 255); /* register_list */ |
211 | } |
212 | fallthrough; /* for illegal instruction case */ |
213 | |
214 | default: |
215 | return BAD_INSTR; |
216 | } |
217 | } |
218 | |
219 | /* |
220 | * Convert Thumb-2 32 bit LDM, STM, LDRD, STRD to equivalent instruction |
221 | * handlable by ARM alignment handler, also find the corresponding handler, |
222 | * so that we can reuse ARM userland alignment fault fixups for Thumb. |
223 | * |
224 | * @pinstr: original Thumb-2 instruction; returns new handlable instruction |
225 | * @regs: register context. |
226 | * @poffset: return offset from faulted addr for later writeback |
227 | * |
228 | * NOTES: |
229 | * 1. Comments below refer to ARMv7 DDI0406A Thumb Instruction sections. |
230 | * 2. Register name Rt from ARMv7 is same as Rd from ARMv6 (Rd is Rt) |
231 | */ |
232 | static void * |
233 | do_alignment_t32_to_handler(u32 *pinstr, struct pt_regs *regs, |
234 | union offset_union *poffset) |
235 | { |
236 | u32 instr = *pinstr; |
237 | u16 tinst1 = (instr >> 16) & 0xffff; |
238 | u16 tinst2 = instr & 0xffff; |
239 | |
240 | switch (tinst1 & 0xffe0) { |
241 | /* A6.3.5 Load/Store multiple */ |
242 | case 0xe880: /* STM/STMIA/STMEA,LDM/LDMIA, PUSH/POP T2 */ |
243 | case 0xe8a0: /* ...above writeback version */ |
244 | case 0xe900: /* STMDB/STMFD, LDMDB/LDMEA */ |
245 | case 0xe920: /* ...above writeback version */ |
246 | /* no need offset decision since handler calculates it */ |
247 | return do_alignment_ldmstm; |
248 | |
249 | case 0xf840: /* POP/PUSH T3 (single register) */ |
250 | if (RN_BITS(instr) == 13 && (tinst2 & 0x09ff) == 0x0904) { |
251 | u32 L = !!(LDST_L_BIT(instr)); |
252 | const u32 subset[2] = { |
253 | 0xe92d0000, /* STMDB sp!,{registers} */ |
254 | 0xe8bd0000, /* LDMIA sp!,{registers} */ |
255 | }; |
256 | *pinstr = subset[L] | (1<<RD_BITS(instr)); |
257 | return do_alignment_ldmstm; |
258 | } |
259 | /* Else fall through for illegal instruction case */ |
260 | break; |
261 | |
262 | /* A6.3.6 Load/store double, STRD/LDRD(immed, lit, reg) */ |
263 | case 0xe860: |
264 | case 0xe960: |
265 | case 0xe8e0: |
266 | case 0xe9e0: |
267 | poffset->un = (tinst2 & 0xff) << 2; |
268 | fallthrough; |
269 | |
270 | case 0xe940: |
271 | case 0xe9c0: |
272 | return do_alignment_ldrdstrd; |
273 | |
274 | /* |
275 | * No need to handle load/store instructions up to word size |
276 | * since ARMv6 and later CPUs can perform unaligned accesses. |
277 | */ |
278 | default: |
279 | break; |
280 | } |
281 | return NULL; |
282 | } |
283 | |
284 | static int alignment_get_arm(struct pt_regs *regs, __le32 __user *ip, u32 *inst) |
285 | { |
286 | __le32 instr = 0; |
287 | int fault; |
288 | |
289 | fault = get_user(instr, ip); |
290 | if (fault) |
291 | return fault; |
292 | |
293 | *inst = __le32_to_cpu(instr); |
294 | return 0; |
295 | } |
296 | |
297 | static int alignment_get_thumb(struct pt_regs *regs, __le16 __user *ip, u16 *inst) |
298 | { |
299 | __le16 instr = 0; |
300 | int fault; |
301 | |
302 | fault = get_user(instr, ip); |
303 | if (fault) |
304 | return fault; |
305 | |
306 | *inst = __le16_to_cpu(instr); |
307 | return 0; |
308 | } |
309 | |
310 | int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs) |
311 | { |
312 | union offset_union offset; |
313 | unsigned long instrptr; |
314 | int (*handler)(unsigned long addr, u32 instr, struct pt_regs *regs); |
315 | unsigned int type; |
316 | u32 instr = 0; |
317 | int isize = 4; |
318 | int thumb2_32b = 0; |
319 | |
320 | instrptr = instruction_pointer(regs); |
321 | |
322 | if (compat_thumb_mode(regs)) { |
323 | __le16 __user *ptr = (__le16 __user *)(instrptr & ~1); |
324 | u16 tinstr, tinst2; |
325 | |
326 | if (alignment_get_thumb(regs, ip: ptr, inst: &tinstr)) |
327 | return 1; |
328 | |
329 | if (IS_T32(tinstr)) { /* Thumb-2 32-bit */ |
330 | if (alignment_get_thumb(regs, ip: ptr + 1, inst: &tinst2)) |
331 | return 1; |
332 | instr = ((u32)tinstr << 16) | tinst2; |
333 | thumb2_32b = 1; |
334 | } else { |
335 | isize = 2; |
336 | instr = thumb2arm(tinstr); |
337 | } |
338 | } else { |
339 | if (alignment_get_arm(regs, ip: (__le32 __user *)instrptr, inst: &instr)) |
340 | return 1; |
341 | } |
342 | |
343 | switch (CODING_BITS(instr)) { |
344 | case 0x00000000: /* 3.13.4 load/store instruction extensions */ |
345 | if (LDSTHD_I_BIT(instr)) |
346 | offset.un = (instr & 0xf00) >> 4 | (instr & 15); |
347 | else |
348 | offset.un = regs->regs[RM_BITS(instr)]; |
349 | |
350 | if ((instr & 0x001000f0) == 0x000000d0 || /* LDRD */ |
351 | (instr & 0x001000f0) == 0x000000f0) /* STRD */ |
352 | handler = do_alignment_ldrdstrd; |
353 | else |
354 | return 1; |
355 | break; |
356 | |
357 | case 0x08000000: /* ldm or stm, or thumb-2 32bit instruction */ |
358 | if (thumb2_32b) { |
359 | offset.un = 0; |
360 | handler = do_alignment_t32_to_handler(pinstr: &instr, regs, poffset: &offset); |
361 | } else { |
362 | offset.un = 0; |
363 | handler = do_alignment_ldmstm; |
364 | } |
365 | break; |
366 | |
367 | default: |
368 | return 1; |
369 | } |
370 | |
371 | type = handler(addr, instr, regs); |
372 | |
373 | if (type == TYPE_ERROR || type == TYPE_FAULT) |
374 | return 1; |
375 | |
376 | if (type == TYPE_LDST) |
377 | do_alignment_finish_ldst(addr, instr, regs, offset); |
378 | |
379 | perf_sw_event(event_id: PERF_COUNT_SW_ALIGNMENT_FAULTS, nr: 1, regs, addr: regs->pc); |
380 | arm64_skip_faulting_instruction(regs, isize); |
381 | |
382 | return 0; |
383 | } |
384 | |