1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Handle unaligned accesses by emulation. |
4 | * |
5 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited |
6 | * |
7 | * Derived from MIPS: |
8 | * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle |
9 | * Copyright (C) 1999 Silicon Graphics, Inc. |
10 | * Copyright (C) 2014 Imagination Technologies Ltd. |
11 | */ |
12 | #include <linux/mm.h> |
13 | #include <linux/sched.h> |
14 | #include <linux/signal.h> |
15 | #include <linux/debugfs.h> |
16 | #include <linux/perf_event.h> |
17 | |
18 | #include <asm/asm.h> |
19 | #include <asm/branch.h> |
20 | #include <asm/fpu.h> |
21 | #include <asm/inst.h> |
22 | |
23 | #include "access-helper.h" |
24 | |
25 | #ifdef CONFIG_DEBUG_FS |
26 | static u32 unaligned_instructions_user; |
27 | static u32 unaligned_instructions_kernel; |
28 | #endif |
29 | |
30 | static inline unsigned long read_fpr(unsigned int idx) |
31 | { |
32 | #define READ_FPR(idx, __value) \ |
33 | __asm__ __volatile__("movfr2gr.d %0, $f"#idx"\n\t" : "=r"(__value)); |
34 | |
35 | unsigned long __value; |
36 | |
37 | switch (idx) { |
38 | case 0: |
39 | READ_FPR(0, __value); |
40 | break; |
41 | case 1: |
42 | READ_FPR(1, __value); |
43 | break; |
44 | case 2: |
45 | READ_FPR(2, __value); |
46 | break; |
47 | case 3: |
48 | READ_FPR(3, __value); |
49 | break; |
50 | case 4: |
51 | READ_FPR(4, __value); |
52 | break; |
53 | case 5: |
54 | READ_FPR(5, __value); |
55 | break; |
56 | case 6: |
57 | READ_FPR(6, __value); |
58 | break; |
59 | case 7: |
60 | READ_FPR(7, __value); |
61 | break; |
62 | case 8: |
63 | READ_FPR(8, __value); |
64 | break; |
65 | case 9: |
66 | READ_FPR(9, __value); |
67 | break; |
68 | case 10: |
69 | READ_FPR(10, __value); |
70 | break; |
71 | case 11: |
72 | READ_FPR(11, __value); |
73 | break; |
74 | case 12: |
75 | READ_FPR(12, __value); |
76 | break; |
77 | case 13: |
78 | READ_FPR(13, __value); |
79 | break; |
80 | case 14: |
81 | READ_FPR(14, __value); |
82 | break; |
83 | case 15: |
84 | READ_FPR(15, __value); |
85 | break; |
86 | case 16: |
87 | READ_FPR(16, __value); |
88 | break; |
89 | case 17: |
90 | READ_FPR(17, __value); |
91 | break; |
92 | case 18: |
93 | READ_FPR(18, __value); |
94 | break; |
95 | case 19: |
96 | READ_FPR(19, __value); |
97 | break; |
98 | case 20: |
99 | READ_FPR(20, __value); |
100 | break; |
101 | case 21: |
102 | READ_FPR(21, __value); |
103 | break; |
104 | case 22: |
105 | READ_FPR(22, __value); |
106 | break; |
107 | case 23: |
108 | READ_FPR(23, __value); |
109 | break; |
110 | case 24: |
111 | READ_FPR(24, __value); |
112 | break; |
113 | case 25: |
114 | READ_FPR(25, __value); |
115 | break; |
116 | case 26: |
117 | READ_FPR(26, __value); |
118 | break; |
119 | case 27: |
120 | READ_FPR(27, __value); |
121 | break; |
122 | case 28: |
123 | READ_FPR(28, __value); |
124 | break; |
125 | case 29: |
126 | READ_FPR(29, __value); |
127 | break; |
128 | case 30: |
129 | READ_FPR(30, __value); |
130 | break; |
131 | case 31: |
132 | READ_FPR(31, __value); |
133 | break; |
134 | default: |
135 | panic(fmt: "unexpected idx '%d'" , idx); |
136 | } |
137 | #undef READ_FPR |
138 | return __value; |
139 | } |
140 | |
141 | static inline void write_fpr(unsigned int idx, unsigned long value) |
142 | { |
143 | #define WRITE_FPR(idx, value) \ |
144 | __asm__ __volatile__("movgr2fr.d $f"#idx", %0\n\t" :: "r"(value)); |
145 | |
146 | switch (idx) { |
147 | case 0: |
148 | WRITE_FPR(0, value); |
149 | break; |
150 | case 1: |
151 | WRITE_FPR(1, value); |
152 | break; |
153 | case 2: |
154 | WRITE_FPR(2, value); |
155 | break; |
156 | case 3: |
157 | WRITE_FPR(3, value); |
158 | break; |
159 | case 4: |
160 | WRITE_FPR(4, value); |
161 | break; |
162 | case 5: |
163 | WRITE_FPR(5, value); |
164 | break; |
165 | case 6: |
166 | WRITE_FPR(6, value); |
167 | break; |
168 | case 7: |
169 | WRITE_FPR(7, value); |
170 | break; |
171 | case 8: |
172 | WRITE_FPR(8, value); |
173 | break; |
174 | case 9: |
175 | WRITE_FPR(9, value); |
176 | break; |
177 | case 10: |
178 | WRITE_FPR(10, value); |
179 | break; |
180 | case 11: |
181 | WRITE_FPR(11, value); |
182 | break; |
183 | case 12: |
184 | WRITE_FPR(12, value); |
185 | break; |
186 | case 13: |
187 | WRITE_FPR(13, value); |
188 | break; |
189 | case 14: |
190 | WRITE_FPR(14, value); |
191 | break; |
192 | case 15: |
193 | WRITE_FPR(15, value); |
194 | break; |
195 | case 16: |
196 | WRITE_FPR(16, value); |
197 | break; |
198 | case 17: |
199 | WRITE_FPR(17, value); |
200 | break; |
201 | case 18: |
202 | WRITE_FPR(18, value); |
203 | break; |
204 | case 19: |
205 | WRITE_FPR(19, value); |
206 | break; |
207 | case 20: |
208 | WRITE_FPR(20, value); |
209 | break; |
210 | case 21: |
211 | WRITE_FPR(21, value); |
212 | break; |
213 | case 22: |
214 | WRITE_FPR(22, value); |
215 | break; |
216 | case 23: |
217 | WRITE_FPR(23, value); |
218 | break; |
219 | case 24: |
220 | WRITE_FPR(24, value); |
221 | break; |
222 | case 25: |
223 | WRITE_FPR(25, value); |
224 | break; |
225 | case 26: |
226 | WRITE_FPR(26, value); |
227 | break; |
228 | case 27: |
229 | WRITE_FPR(27, value); |
230 | break; |
231 | case 28: |
232 | WRITE_FPR(28, value); |
233 | break; |
234 | case 29: |
235 | WRITE_FPR(29, value); |
236 | break; |
237 | case 30: |
238 | WRITE_FPR(30, value); |
239 | break; |
240 | case 31: |
241 | WRITE_FPR(31, value); |
242 | break; |
243 | default: |
244 | panic(fmt: "unexpected idx '%d'" , idx); |
245 | } |
246 | #undef WRITE_FPR |
247 | } |
248 | |
249 | void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int *pc) |
250 | { |
251 | bool fp = false; |
252 | bool sign, write; |
253 | bool user = user_mode(regs); |
254 | unsigned int res, size = 0; |
255 | unsigned long value = 0; |
256 | union loongarch_instruction insn; |
257 | |
258 | perf_sw_event(event_id: PERF_COUNT_SW_EMULATION_FAULTS, nr: 1, regs, addr: 0); |
259 | |
260 | __get_inst(i: &insn.word, p: pc, user); |
261 | |
262 | switch (insn.reg2i12_format.opcode) { |
263 | case ldh_op: |
264 | size = 2; |
265 | sign = true; |
266 | write = false; |
267 | break; |
268 | case ldhu_op: |
269 | size = 2; |
270 | sign = false; |
271 | write = false; |
272 | break; |
273 | case sth_op: |
274 | size = 2; |
275 | sign = true; |
276 | write = true; |
277 | break; |
278 | case ldw_op: |
279 | size = 4; |
280 | sign = true; |
281 | write = false; |
282 | break; |
283 | case ldwu_op: |
284 | size = 4; |
285 | sign = false; |
286 | write = false; |
287 | break; |
288 | case stw_op: |
289 | size = 4; |
290 | sign = true; |
291 | write = true; |
292 | break; |
293 | case ldd_op: |
294 | size = 8; |
295 | sign = true; |
296 | write = false; |
297 | break; |
298 | case std_op: |
299 | size = 8; |
300 | sign = true; |
301 | write = true; |
302 | break; |
303 | case flds_op: |
304 | size = 4; |
305 | fp = true; |
306 | sign = true; |
307 | write = false; |
308 | break; |
309 | case fsts_op: |
310 | size = 4; |
311 | fp = true; |
312 | sign = true; |
313 | write = true; |
314 | break; |
315 | case fldd_op: |
316 | size = 8; |
317 | fp = true; |
318 | sign = true; |
319 | write = false; |
320 | break; |
321 | case fstd_op: |
322 | size = 8; |
323 | fp = true; |
324 | sign = true; |
325 | write = true; |
326 | break; |
327 | } |
328 | |
329 | switch (insn.reg2i14_format.opcode) { |
330 | case ldptrw_op: |
331 | size = 4; |
332 | sign = true; |
333 | write = false; |
334 | break; |
335 | case stptrw_op: |
336 | size = 4; |
337 | sign = true; |
338 | write = true; |
339 | break; |
340 | case ldptrd_op: |
341 | size = 8; |
342 | sign = true; |
343 | write = false; |
344 | break; |
345 | case stptrd_op: |
346 | size = 8; |
347 | sign = true; |
348 | write = true; |
349 | break; |
350 | } |
351 | |
352 | switch (insn.reg3_format.opcode) { |
353 | case ldxh_op: |
354 | size = 2; |
355 | sign = true; |
356 | write = false; |
357 | break; |
358 | case ldxhu_op: |
359 | size = 2; |
360 | sign = false; |
361 | write = false; |
362 | break; |
363 | case stxh_op: |
364 | size = 2; |
365 | sign = true; |
366 | write = true; |
367 | break; |
368 | case ldxw_op: |
369 | size = 4; |
370 | sign = true; |
371 | write = false; |
372 | break; |
373 | case ldxwu_op: |
374 | size = 4; |
375 | sign = false; |
376 | write = false; |
377 | break; |
378 | case stxw_op: |
379 | size = 4; |
380 | sign = true; |
381 | write = true; |
382 | break; |
383 | case ldxd_op: |
384 | size = 8; |
385 | sign = true; |
386 | write = false; |
387 | break; |
388 | case stxd_op: |
389 | size = 8; |
390 | sign = true; |
391 | write = true; |
392 | break; |
393 | case fldxs_op: |
394 | size = 4; |
395 | fp = true; |
396 | sign = true; |
397 | write = false; |
398 | break; |
399 | case fstxs_op: |
400 | size = 4; |
401 | fp = true; |
402 | sign = true; |
403 | write = true; |
404 | break; |
405 | case fldxd_op: |
406 | size = 8; |
407 | fp = true; |
408 | sign = true; |
409 | write = false; |
410 | break; |
411 | case fstxd_op: |
412 | size = 8; |
413 | fp = true; |
414 | sign = true; |
415 | write = true; |
416 | break; |
417 | } |
418 | |
419 | if (!size) |
420 | goto sigbus; |
421 | if (user && !access_ok(addr, size)) |
422 | goto sigbus; |
423 | |
424 | if (!write) { |
425 | res = unaligned_read(addr, &value, size, sign); |
426 | if (res) |
427 | goto fault; |
428 | |
429 | /* Rd is the same field in any formats */ |
430 | if (!fp) |
431 | regs->regs[insn.reg3_format.rd] = value; |
432 | else { |
433 | if (is_fpu_owner()) |
434 | write_fpr(idx: insn.reg3_format.rd, value); |
435 | else |
436 | set_fpr64(¤t->thread.fpu.fpr[insn.reg3_format.rd], 0, value); |
437 | } |
438 | } else { |
439 | /* Rd is the same field in any formats */ |
440 | if (!fp) |
441 | value = regs->regs[insn.reg3_format.rd]; |
442 | else { |
443 | if (is_fpu_owner()) |
444 | value = read_fpr(idx: insn.reg3_format.rd); |
445 | else |
446 | value = get_fpr64(¤t->thread.fpu.fpr[insn.reg3_format.rd], 0); |
447 | } |
448 | |
449 | res = unaligned_write(addr, value, size); |
450 | if (res) |
451 | goto fault; |
452 | } |
453 | |
454 | #ifdef CONFIG_DEBUG_FS |
455 | if (user) |
456 | unaligned_instructions_user++; |
457 | else |
458 | unaligned_instructions_kernel++; |
459 | #endif |
460 | |
461 | compute_return_era(regs); |
462 | |
463 | return; |
464 | |
465 | fault: |
466 | /* Did we have an exception handler installed? */ |
467 | if (fixup_exception(regs)) |
468 | return; |
469 | |
470 | die_if_kernel("Unhandled kernel unaligned access" , regs); |
471 | force_sig(SIGSEGV); |
472 | |
473 | return; |
474 | |
475 | sigbus: |
476 | die_if_kernel("Unhandled kernel unaligned access" , regs); |
477 | force_sig(SIGBUS); |
478 | |
479 | return; |
480 | } |
481 | |
482 | #ifdef CONFIG_DEBUG_FS |
483 | static int __init debugfs_unaligned(void) |
484 | { |
485 | struct dentry *d; |
486 | |
487 | d = debugfs_create_dir(name: "loongarch" , NULL); |
488 | |
489 | debugfs_create_u32(name: "unaligned_instructions_user" , |
490 | S_IRUGO, parent: d, value: &unaligned_instructions_user); |
491 | debugfs_create_u32(name: "unaligned_instructions_kernel" , |
492 | S_IRUGO, parent: d, value: &unaligned_instructions_kernel); |
493 | |
494 | return 0; |
495 | } |
496 | arch_initcall(debugfs_unaligned); |
497 | #endif |
498 | |