1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2013 Huawei Ltd. |
4 | * Author: Jiang Liu <liuj97@gmail.com> |
5 | * |
6 | * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com> |
7 | */ |
8 | #include <linux/bitops.h> |
9 | #include <linux/bug.h> |
10 | #include <linux/printk.h> |
11 | #include <linux/sizes.h> |
12 | #include <linux/types.h> |
13 | |
14 | #include <asm/debug-monitors.h> |
15 | #include <asm/errno.h> |
16 | #include <asm/insn.h> |
17 | #include <asm/kprobes.h> |
18 | |
19 | #define AARCH64_INSN_SF_BIT BIT(31) |
20 | #define AARCH64_INSN_N_BIT BIT(22) |
21 | #define AARCH64_INSN_LSL_12 BIT(22) |
22 | |
23 | static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, |
24 | u32 *maskp, int *shiftp) |
25 | { |
26 | u32 mask; |
27 | int shift; |
28 | |
29 | switch (type) { |
30 | case AARCH64_INSN_IMM_26: |
31 | mask = BIT(26) - 1; |
32 | shift = 0; |
33 | break; |
34 | case AARCH64_INSN_IMM_19: |
35 | mask = BIT(19) - 1; |
36 | shift = 5; |
37 | break; |
38 | case AARCH64_INSN_IMM_16: |
39 | mask = BIT(16) - 1; |
40 | shift = 5; |
41 | break; |
42 | case AARCH64_INSN_IMM_14: |
43 | mask = BIT(14) - 1; |
44 | shift = 5; |
45 | break; |
46 | case AARCH64_INSN_IMM_12: |
47 | mask = BIT(12) - 1; |
48 | shift = 10; |
49 | break; |
50 | case AARCH64_INSN_IMM_9: |
51 | mask = BIT(9) - 1; |
52 | shift = 12; |
53 | break; |
54 | case AARCH64_INSN_IMM_7: |
55 | mask = BIT(7) - 1; |
56 | shift = 15; |
57 | break; |
58 | case AARCH64_INSN_IMM_6: |
59 | case AARCH64_INSN_IMM_S: |
60 | mask = BIT(6) - 1; |
61 | shift = 10; |
62 | break; |
63 | case AARCH64_INSN_IMM_R: |
64 | mask = BIT(6) - 1; |
65 | shift = 16; |
66 | break; |
67 | case AARCH64_INSN_IMM_N: |
68 | mask = 1; |
69 | shift = 22; |
70 | break; |
71 | default: |
72 | return -EINVAL; |
73 | } |
74 | |
75 | *maskp = mask; |
76 | *shiftp = shift; |
77 | |
78 | return 0; |
79 | } |
80 | |
81 | #define ADR_IMM_HILOSPLIT 2 |
82 | #define ADR_IMM_SIZE SZ_2M |
83 | #define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1) |
84 | #define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1) |
85 | #define ADR_IMM_LOSHIFT 29 |
86 | #define ADR_IMM_HISHIFT 5 |
87 | |
88 | u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn) |
89 | { |
90 | u32 immlo, immhi, mask; |
91 | int shift; |
92 | |
93 | switch (type) { |
94 | case AARCH64_INSN_IMM_ADR: |
95 | shift = 0; |
96 | immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK; |
97 | immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK; |
98 | insn = (immhi << ADR_IMM_HILOSPLIT) | immlo; |
99 | mask = ADR_IMM_SIZE - 1; |
100 | break; |
101 | default: |
102 | if (aarch64_get_imm_shift_mask(type: type, maskp: &mask, shiftp: &shift) < 0) { |
103 | pr_err("%s: unknown immediate encoding %d\n" , __func__, |
104 | type); |
105 | return 0; |
106 | } |
107 | } |
108 | |
109 | return (insn >> shift) & mask; |
110 | } |
111 | |
112 | u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, |
113 | u32 insn, u64 imm) |
114 | { |
115 | u32 immlo, immhi, mask; |
116 | int shift; |
117 | |
118 | if (insn == AARCH64_BREAK_FAULT) |
119 | return AARCH64_BREAK_FAULT; |
120 | |
121 | switch (type) { |
122 | case AARCH64_INSN_IMM_ADR: |
123 | shift = 0; |
124 | immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT; |
125 | imm >>= ADR_IMM_HILOSPLIT; |
126 | immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT; |
127 | imm = immlo | immhi; |
128 | mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) | |
129 | (ADR_IMM_HIMASK << ADR_IMM_HISHIFT)); |
130 | break; |
131 | default: |
132 | if (aarch64_get_imm_shift_mask(type: type, maskp: &mask, shiftp: &shift) < 0) { |
133 | pr_err("%s: unknown immediate encoding %d\n" , __func__, |
134 | type); |
135 | return AARCH64_BREAK_FAULT; |
136 | } |
137 | } |
138 | |
139 | /* Update the immediate field. */ |
140 | insn &= ~(mask << shift); |
141 | insn |= (imm & mask) << shift; |
142 | |
143 | return insn; |
144 | } |
145 | |
146 | u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type, |
147 | u32 insn) |
148 | { |
149 | int shift; |
150 | |
151 | switch (type) { |
152 | case AARCH64_INSN_REGTYPE_RT: |
153 | case AARCH64_INSN_REGTYPE_RD: |
154 | shift = 0; |
155 | break; |
156 | case AARCH64_INSN_REGTYPE_RN: |
157 | shift = 5; |
158 | break; |
159 | case AARCH64_INSN_REGTYPE_RT2: |
160 | case AARCH64_INSN_REGTYPE_RA: |
161 | shift = 10; |
162 | break; |
163 | case AARCH64_INSN_REGTYPE_RM: |
164 | shift = 16; |
165 | break; |
166 | default: |
167 | pr_err("%s: unknown register type encoding %d\n" , __func__, |
168 | type); |
169 | return 0; |
170 | } |
171 | |
172 | return (insn >> shift) & GENMASK(4, 0); |
173 | } |
174 | |
175 | static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type, |
176 | u32 insn, |
177 | enum aarch64_insn_register reg) |
178 | { |
179 | int shift; |
180 | |
181 | if (insn == AARCH64_BREAK_FAULT) |
182 | return AARCH64_BREAK_FAULT; |
183 | |
184 | if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) { |
185 | pr_err("%s: unknown register encoding %d\n" , __func__, reg); |
186 | return AARCH64_BREAK_FAULT; |
187 | } |
188 | |
189 | switch (type) { |
190 | case AARCH64_INSN_REGTYPE_RT: |
191 | case AARCH64_INSN_REGTYPE_RD: |
192 | shift = 0; |
193 | break; |
194 | case AARCH64_INSN_REGTYPE_RN: |
195 | shift = 5; |
196 | break; |
197 | case AARCH64_INSN_REGTYPE_RT2: |
198 | case AARCH64_INSN_REGTYPE_RA: |
199 | shift = 10; |
200 | break; |
201 | case AARCH64_INSN_REGTYPE_RM: |
202 | case AARCH64_INSN_REGTYPE_RS: |
203 | shift = 16; |
204 | break; |
205 | default: |
206 | pr_err("%s: unknown register type encoding %d\n" , __func__, |
207 | type); |
208 | return AARCH64_BREAK_FAULT; |
209 | } |
210 | |
211 | insn &= ~(GENMASK(4, 0) << shift); |
212 | insn |= reg << shift; |
213 | |
214 | return insn; |
215 | } |
216 | |
217 | static const u32 aarch64_insn_ldst_size[] = { |
218 | [AARCH64_INSN_SIZE_8] = 0, |
219 | [AARCH64_INSN_SIZE_16] = 1, |
220 | [AARCH64_INSN_SIZE_32] = 2, |
221 | [AARCH64_INSN_SIZE_64] = 3, |
222 | }; |
223 | |
224 | static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type, |
225 | u32 insn) |
226 | { |
227 | u32 size; |
228 | |
229 | if (type < AARCH64_INSN_SIZE_8 || type > AARCH64_INSN_SIZE_64) { |
230 | pr_err("%s: unknown size encoding %d\n" , __func__, type); |
231 | return AARCH64_BREAK_FAULT; |
232 | } |
233 | |
234 | size = aarch64_insn_ldst_size[type]; |
235 | insn &= ~GENMASK(31, 30); |
236 | insn |= size << 30; |
237 | |
238 | return insn; |
239 | } |
240 | |
241 | static inline long label_imm_common(unsigned long pc, unsigned long addr, |
242 | long range) |
243 | { |
244 | long offset; |
245 | |
246 | if ((pc & 0x3) || (addr & 0x3)) { |
247 | pr_err("%s: A64 instructions must be word aligned\n" , __func__); |
248 | return range; |
249 | } |
250 | |
251 | offset = ((long)addr - (long)pc); |
252 | |
253 | if (offset < -range || offset >= range) { |
254 | pr_err("%s: offset out of range\n" , __func__); |
255 | return range; |
256 | } |
257 | |
258 | return offset; |
259 | } |
260 | |
261 | u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, |
262 | enum aarch64_insn_branch_type type) |
263 | { |
264 | u32 insn; |
265 | long offset; |
266 | |
267 | /* |
268 | * B/BL support [-128M, 128M) offset |
269 | * ARM64 virtual address arrangement guarantees all kernel and module |
270 | * texts are within +/-128M. |
271 | */ |
272 | offset = label_imm_common(pc, addr, SZ_128M); |
273 | if (offset >= SZ_128M) |
274 | return AARCH64_BREAK_FAULT; |
275 | |
276 | switch (type) { |
277 | case AARCH64_INSN_BRANCH_LINK: |
278 | insn = aarch64_insn_get_bl_value(); |
279 | break; |
280 | case AARCH64_INSN_BRANCH_NOLINK: |
281 | insn = aarch64_insn_get_b_value(); |
282 | break; |
283 | default: |
284 | pr_err("%s: unknown branch encoding %d\n" , __func__, type); |
285 | return AARCH64_BREAK_FAULT; |
286 | } |
287 | |
288 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, |
289 | offset >> 2); |
290 | } |
291 | |
292 | u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, |
293 | enum aarch64_insn_register reg, |
294 | enum aarch64_insn_variant variant, |
295 | enum aarch64_insn_branch_type type) |
296 | { |
297 | u32 insn; |
298 | long offset; |
299 | |
300 | offset = label_imm_common(pc, addr, SZ_1M); |
301 | if (offset >= SZ_1M) |
302 | return AARCH64_BREAK_FAULT; |
303 | |
304 | switch (type) { |
305 | case AARCH64_INSN_BRANCH_COMP_ZERO: |
306 | insn = aarch64_insn_get_cbz_value(); |
307 | break; |
308 | case AARCH64_INSN_BRANCH_COMP_NONZERO: |
309 | insn = aarch64_insn_get_cbnz_value(); |
310 | break; |
311 | default: |
312 | pr_err("%s: unknown branch encoding %d\n" , __func__, type); |
313 | return AARCH64_BREAK_FAULT; |
314 | } |
315 | |
316 | switch (variant) { |
317 | case AARCH64_INSN_VARIANT_32BIT: |
318 | break; |
319 | case AARCH64_INSN_VARIANT_64BIT: |
320 | insn |= AARCH64_INSN_SF_BIT; |
321 | break; |
322 | default: |
323 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
324 | return AARCH64_BREAK_FAULT; |
325 | } |
326 | |
327 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); |
328 | |
329 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, |
330 | offset >> 2); |
331 | } |
332 | |
333 | u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, |
334 | enum aarch64_insn_condition cond) |
335 | { |
336 | u32 insn; |
337 | long offset; |
338 | |
339 | offset = label_imm_common(pc, addr, SZ_1M); |
340 | |
341 | insn = aarch64_insn_get_bcond_value(); |
342 | |
343 | if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) { |
344 | pr_err("%s: unknown condition encoding %d\n" , __func__, cond); |
345 | return AARCH64_BREAK_FAULT; |
346 | } |
347 | insn |= cond; |
348 | |
349 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, |
350 | offset >> 2); |
351 | } |
352 | |
353 | u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg, |
354 | enum aarch64_insn_branch_type type) |
355 | { |
356 | u32 insn; |
357 | |
358 | switch (type) { |
359 | case AARCH64_INSN_BRANCH_NOLINK: |
360 | insn = aarch64_insn_get_br_value(); |
361 | break; |
362 | case AARCH64_INSN_BRANCH_LINK: |
363 | insn = aarch64_insn_get_blr_value(); |
364 | break; |
365 | case AARCH64_INSN_BRANCH_RETURN: |
366 | insn = aarch64_insn_get_ret_value(); |
367 | break; |
368 | default: |
369 | pr_err("%s: unknown branch encoding %d\n" , __func__, type); |
370 | return AARCH64_BREAK_FAULT; |
371 | } |
372 | |
373 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg); |
374 | } |
375 | |
376 | u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg, |
377 | enum aarch64_insn_register base, |
378 | enum aarch64_insn_register offset, |
379 | enum aarch64_insn_size_type size, |
380 | enum aarch64_insn_ldst_type type) |
381 | { |
382 | u32 insn; |
383 | |
384 | switch (type) { |
385 | case AARCH64_INSN_LDST_LOAD_REG_OFFSET: |
386 | insn = aarch64_insn_get_ldr_reg_value(); |
387 | break; |
388 | case AARCH64_INSN_LDST_SIGNED_LOAD_REG_OFFSET: |
389 | insn = aarch64_insn_get_signed_ldr_reg_value(); |
390 | break; |
391 | case AARCH64_INSN_LDST_STORE_REG_OFFSET: |
392 | insn = aarch64_insn_get_str_reg_value(); |
393 | break; |
394 | default: |
395 | pr_err("%s: unknown load/store encoding %d\n" , __func__, type); |
396 | return AARCH64_BREAK_FAULT; |
397 | } |
398 | |
399 | insn = aarch64_insn_encode_ldst_size(type: size, insn); |
400 | |
401 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); |
402 | |
403 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, |
404 | base); |
405 | |
406 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, |
407 | offset); |
408 | } |
409 | |
410 | u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg, |
411 | enum aarch64_insn_register base, |
412 | unsigned int imm, |
413 | enum aarch64_insn_size_type size, |
414 | enum aarch64_insn_ldst_type type) |
415 | { |
416 | u32 insn; |
417 | u32 shift; |
418 | |
419 | if (size < AARCH64_INSN_SIZE_8 || size > AARCH64_INSN_SIZE_64) { |
420 | pr_err("%s: unknown size encoding %d\n" , __func__, type); |
421 | return AARCH64_BREAK_FAULT; |
422 | } |
423 | |
424 | shift = aarch64_insn_ldst_size[size]; |
425 | if (imm & ~(BIT(12 + shift) - BIT(shift))) { |
426 | pr_err("%s: invalid imm: %d\n" , __func__, imm); |
427 | return AARCH64_BREAK_FAULT; |
428 | } |
429 | |
430 | imm >>= shift; |
431 | |
432 | switch (type) { |
433 | case AARCH64_INSN_LDST_LOAD_IMM_OFFSET: |
434 | insn = aarch64_insn_get_ldr_imm_value(); |
435 | break; |
436 | case AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET: |
437 | insn = aarch64_insn_get_signed_load_imm_value(); |
438 | break; |
439 | case AARCH64_INSN_LDST_STORE_IMM_OFFSET: |
440 | insn = aarch64_insn_get_str_imm_value(); |
441 | break; |
442 | default: |
443 | pr_err("%s: unknown load/store encoding %d\n" , __func__, type); |
444 | return AARCH64_BREAK_FAULT; |
445 | } |
446 | |
447 | insn = aarch64_insn_encode_ldst_size(type: size, insn); |
448 | |
449 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); |
450 | |
451 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, |
452 | base); |
453 | |
454 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm); |
455 | } |
456 | |
457 | u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr, |
458 | enum aarch64_insn_register reg, |
459 | bool is64bit) |
460 | { |
461 | u32 insn; |
462 | long offset; |
463 | |
464 | offset = label_imm_common(pc, addr, SZ_1M); |
465 | if (offset >= SZ_1M) |
466 | return AARCH64_BREAK_FAULT; |
467 | |
468 | insn = aarch64_insn_get_ldr_lit_value(); |
469 | |
470 | if (is64bit) |
471 | insn |= BIT(30); |
472 | |
473 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); |
474 | |
475 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, |
476 | offset >> 2); |
477 | } |
478 | |
479 | u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, |
480 | enum aarch64_insn_register reg2, |
481 | enum aarch64_insn_register base, |
482 | int offset, |
483 | enum aarch64_insn_variant variant, |
484 | enum aarch64_insn_ldst_type type) |
485 | { |
486 | u32 insn; |
487 | int shift; |
488 | |
489 | switch (type) { |
490 | case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX: |
491 | insn = aarch64_insn_get_ldp_pre_value(); |
492 | break; |
493 | case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX: |
494 | insn = aarch64_insn_get_stp_pre_value(); |
495 | break; |
496 | case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX: |
497 | insn = aarch64_insn_get_ldp_post_value(); |
498 | break; |
499 | case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX: |
500 | insn = aarch64_insn_get_stp_post_value(); |
501 | break; |
502 | default: |
503 | pr_err("%s: unknown load/store encoding %d\n" , __func__, type); |
504 | return AARCH64_BREAK_FAULT; |
505 | } |
506 | |
507 | switch (variant) { |
508 | case AARCH64_INSN_VARIANT_32BIT: |
509 | if ((offset & 0x3) || (offset < -256) || (offset > 252)) { |
510 | pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n" , |
511 | __func__, offset); |
512 | return AARCH64_BREAK_FAULT; |
513 | } |
514 | shift = 2; |
515 | break; |
516 | case AARCH64_INSN_VARIANT_64BIT: |
517 | if ((offset & 0x7) || (offset < -512) || (offset > 504)) { |
518 | pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n" , |
519 | __func__, offset); |
520 | return AARCH64_BREAK_FAULT; |
521 | } |
522 | shift = 3; |
523 | insn |= AARCH64_INSN_SF_BIT; |
524 | break; |
525 | default: |
526 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
527 | return AARCH64_BREAK_FAULT; |
528 | } |
529 | |
530 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, |
531 | reg1); |
532 | |
533 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn, |
534 | reg2); |
535 | |
536 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, |
537 | base); |
538 | |
539 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn, |
540 | offset >> shift); |
541 | } |
542 | |
543 | u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg, |
544 | enum aarch64_insn_register base, |
545 | enum aarch64_insn_register state, |
546 | enum aarch64_insn_size_type size, |
547 | enum aarch64_insn_ldst_type type) |
548 | { |
549 | u32 insn; |
550 | |
551 | switch (type) { |
552 | case AARCH64_INSN_LDST_LOAD_EX: |
553 | case AARCH64_INSN_LDST_LOAD_ACQ_EX: |
554 | insn = aarch64_insn_get_load_ex_value(); |
555 | if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX) |
556 | insn |= BIT(15); |
557 | break; |
558 | case AARCH64_INSN_LDST_STORE_EX: |
559 | case AARCH64_INSN_LDST_STORE_REL_EX: |
560 | insn = aarch64_insn_get_store_ex_value(); |
561 | if (type == AARCH64_INSN_LDST_STORE_REL_EX) |
562 | insn |= BIT(15); |
563 | break; |
564 | default: |
565 | pr_err("%s: unknown load/store exclusive encoding %d\n" , __func__, type); |
566 | return AARCH64_BREAK_FAULT; |
567 | } |
568 | |
569 | insn = aarch64_insn_encode_ldst_size(type: size, insn); |
570 | |
571 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, |
572 | reg); |
573 | |
574 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, |
575 | base); |
576 | |
577 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn, |
578 | AARCH64_INSN_REG_ZR); |
579 | |
580 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn, |
581 | state); |
582 | } |
583 | |
584 | #ifdef CONFIG_ARM64_LSE_ATOMICS |
585 | static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type, |
586 | u32 insn) |
587 | { |
588 | u32 order; |
589 | |
590 | switch (type) { |
591 | case AARCH64_INSN_MEM_ORDER_NONE: |
592 | order = 0; |
593 | break; |
594 | case AARCH64_INSN_MEM_ORDER_ACQ: |
595 | order = 2; |
596 | break; |
597 | case AARCH64_INSN_MEM_ORDER_REL: |
598 | order = 1; |
599 | break; |
600 | case AARCH64_INSN_MEM_ORDER_ACQREL: |
601 | order = 3; |
602 | break; |
603 | default: |
604 | pr_err("%s: unknown mem order %d\n" , __func__, type); |
605 | return AARCH64_BREAK_FAULT; |
606 | } |
607 | |
608 | insn &= ~GENMASK(23, 22); |
609 | insn |= order << 22; |
610 | |
611 | return insn; |
612 | } |
613 | |
614 | u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result, |
615 | enum aarch64_insn_register address, |
616 | enum aarch64_insn_register value, |
617 | enum aarch64_insn_size_type size, |
618 | enum aarch64_insn_mem_atomic_op op, |
619 | enum aarch64_insn_mem_order_type order) |
620 | { |
621 | u32 insn; |
622 | |
623 | switch (op) { |
624 | case AARCH64_INSN_MEM_ATOMIC_ADD: |
625 | insn = aarch64_insn_get_ldadd_value(); |
626 | break; |
627 | case AARCH64_INSN_MEM_ATOMIC_CLR: |
628 | insn = aarch64_insn_get_ldclr_value(); |
629 | break; |
630 | case AARCH64_INSN_MEM_ATOMIC_EOR: |
631 | insn = aarch64_insn_get_ldeor_value(); |
632 | break; |
633 | case AARCH64_INSN_MEM_ATOMIC_SET: |
634 | insn = aarch64_insn_get_ldset_value(); |
635 | break; |
636 | case AARCH64_INSN_MEM_ATOMIC_SWP: |
637 | insn = aarch64_insn_get_swp_value(); |
638 | break; |
639 | default: |
640 | pr_err("%s: unimplemented mem atomic op %d\n" , __func__, op); |
641 | return AARCH64_BREAK_FAULT; |
642 | } |
643 | |
644 | switch (size) { |
645 | case AARCH64_INSN_SIZE_32: |
646 | case AARCH64_INSN_SIZE_64: |
647 | break; |
648 | default: |
649 | pr_err("%s: unimplemented size encoding %d\n" , __func__, size); |
650 | return AARCH64_BREAK_FAULT; |
651 | } |
652 | |
653 | insn = aarch64_insn_encode_ldst_size(size, insn); |
654 | |
655 | insn = aarch64_insn_encode_ldst_order(order, insn); |
656 | |
657 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, |
658 | result); |
659 | |
660 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, |
661 | address); |
662 | |
663 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn, |
664 | value); |
665 | } |
666 | |
667 | static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type, |
668 | u32 insn) |
669 | { |
670 | u32 order; |
671 | |
672 | switch (type) { |
673 | case AARCH64_INSN_MEM_ORDER_NONE: |
674 | order = 0; |
675 | break; |
676 | case AARCH64_INSN_MEM_ORDER_ACQ: |
677 | order = BIT(22); |
678 | break; |
679 | case AARCH64_INSN_MEM_ORDER_REL: |
680 | order = BIT(15); |
681 | break; |
682 | case AARCH64_INSN_MEM_ORDER_ACQREL: |
683 | order = BIT(15) | BIT(22); |
684 | break; |
685 | default: |
686 | pr_err("%s: unknown mem order %d\n" , __func__, type); |
687 | return AARCH64_BREAK_FAULT; |
688 | } |
689 | |
690 | insn &= ~(BIT(15) | BIT(22)); |
691 | insn |= order; |
692 | |
693 | return insn; |
694 | } |
695 | |
696 | u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, |
697 | enum aarch64_insn_register address, |
698 | enum aarch64_insn_register value, |
699 | enum aarch64_insn_size_type size, |
700 | enum aarch64_insn_mem_order_type order) |
701 | { |
702 | u32 insn; |
703 | |
704 | switch (size) { |
705 | case AARCH64_INSN_SIZE_32: |
706 | case AARCH64_INSN_SIZE_64: |
707 | break; |
708 | default: |
709 | pr_err("%s: unimplemented size encoding %d\n" , __func__, size); |
710 | return AARCH64_BREAK_FAULT; |
711 | } |
712 | |
713 | insn = aarch64_insn_get_cas_value(); |
714 | |
715 | insn = aarch64_insn_encode_ldst_size(size, insn); |
716 | |
717 | insn = aarch64_insn_encode_cas_order(order, insn); |
718 | |
719 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, |
720 | result); |
721 | |
722 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, |
723 | address); |
724 | |
725 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn, |
726 | value); |
727 | } |
728 | #endif |
729 | |
730 | u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst, |
731 | enum aarch64_insn_register src, |
732 | int imm, enum aarch64_insn_variant variant, |
733 | enum aarch64_insn_adsb_type type) |
734 | { |
735 | u32 insn; |
736 | |
737 | switch (type) { |
738 | case AARCH64_INSN_ADSB_ADD: |
739 | insn = aarch64_insn_get_add_imm_value(); |
740 | break; |
741 | case AARCH64_INSN_ADSB_SUB: |
742 | insn = aarch64_insn_get_sub_imm_value(); |
743 | break; |
744 | case AARCH64_INSN_ADSB_ADD_SETFLAGS: |
745 | insn = aarch64_insn_get_adds_imm_value(); |
746 | break; |
747 | case AARCH64_INSN_ADSB_SUB_SETFLAGS: |
748 | insn = aarch64_insn_get_subs_imm_value(); |
749 | break; |
750 | default: |
751 | pr_err("%s: unknown add/sub encoding %d\n" , __func__, type); |
752 | return AARCH64_BREAK_FAULT; |
753 | } |
754 | |
755 | switch (variant) { |
756 | case AARCH64_INSN_VARIANT_32BIT: |
757 | break; |
758 | case AARCH64_INSN_VARIANT_64BIT: |
759 | insn |= AARCH64_INSN_SF_BIT; |
760 | break; |
761 | default: |
762 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
763 | return AARCH64_BREAK_FAULT; |
764 | } |
765 | |
766 | /* We can't encode more than a 24bit value (12bit + 12bit shift) */ |
767 | if (imm & ~(BIT(24) - 1)) |
768 | goto out; |
769 | |
770 | /* If we have something in the top 12 bits... */ |
771 | if (imm & ~(SZ_4K - 1)) { |
772 | /* ... and in the low 12 bits -> error */ |
773 | if (imm & (SZ_4K - 1)) |
774 | goto out; |
775 | |
776 | imm >>= 12; |
777 | insn |= AARCH64_INSN_LSL_12; |
778 | } |
779 | |
780 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
781 | |
782 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); |
783 | |
784 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm); |
785 | |
786 | out: |
787 | pr_err("%s: invalid immediate encoding %d\n" , __func__, imm); |
788 | return AARCH64_BREAK_FAULT; |
789 | } |
790 | |
791 | u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst, |
792 | enum aarch64_insn_register src, |
793 | int immr, int imms, |
794 | enum aarch64_insn_variant variant, |
795 | enum aarch64_insn_bitfield_type type) |
796 | { |
797 | u32 insn; |
798 | u32 mask; |
799 | |
800 | switch (type) { |
801 | case AARCH64_INSN_BITFIELD_MOVE: |
802 | insn = aarch64_insn_get_bfm_value(); |
803 | break; |
804 | case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED: |
805 | insn = aarch64_insn_get_ubfm_value(); |
806 | break; |
807 | case AARCH64_INSN_BITFIELD_MOVE_SIGNED: |
808 | insn = aarch64_insn_get_sbfm_value(); |
809 | break; |
810 | default: |
811 | pr_err("%s: unknown bitfield encoding %d\n" , __func__, type); |
812 | return AARCH64_BREAK_FAULT; |
813 | } |
814 | |
815 | switch (variant) { |
816 | case AARCH64_INSN_VARIANT_32BIT: |
817 | mask = GENMASK(4, 0); |
818 | break; |
819 | case AARCH64_INSN_VARIANT_64BIT: |
820 | insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT; |
821 | mask = GENMASK(5, 0); |
822 | break; |
823 | default: |
824 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
825 | return AARCH64_BREAK_FAULT; |
826 | } |
827 | |
828 | if (immr & ~mask) { |
829 | pr_err("%s: invalid immr encoding %d\n" , __func__, immr); |
830 | return AARCH64_BREAK_FAULT; |
831 | } |
832 | if (imms & ~mask) { |
833 | pr_err("%s: invalid imms encoding %d\n" , __func__, imms); |
834 | return AARCH64_BREAK_FAULT; |
835 | } |
836 | |
837 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
838 | |
839 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); |
840 | |
841 | insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr); |
842 | |
843 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms); |
844 | } |
845 | |
846 | u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst, |
847 | int imm, int shift, |
848 | enum aarch64_insn_variant variant, |
849 | enum aarch64_insn_movewide_type type) |
850 | { |
851 | u32 insn; |
852 | |
853 | switch (type) { |
854 | case AARCH64_INSN_MOVEWIDE_ZERO: |
855 | insn = aarch64_insn_get_movz_value(); |
856 | break; |
857 | case AARCH64_INSN_MOVEWIDE_KEEP: |
858 | insn = aarch64_insn_get_movk_value(); |
859 | break; |
860 | case AARCH64_INSN_MOVEWIDE_INVERSE: |
861 | insn = aarch64_insn_get_movn_value(); |
862 | break; |
863 | default: |
864 | pr_err("%s: unknown movewide encoding %d\n" , __func__, type); |
865 | return AARCH64_BREAK_FAULT; |
866 | } |
867 | |
868 | if (imm & ~(SZ_64K - 1)) { |
869 | pr_err("%s: invalid immediate encoding %d\n" , __func__, imm); |
870 | return AARCH64_BREAK_FAULT; |
871 | } |
872 | |
873 | switch (variant) { |
874 | case AARCH64_INSN_VARIANT_32BIT: |
875 | if (shift != 0 && shift != 16) { |
876 | pr_err("%s: invalid shift encoding %d\n" , __func__, |
877 | shift); |
878 | return AARCH64_BREAK_FAULT; |
879 | } |
880 | break; |
881 | case AARCH64_INSN_VARIANT_64BIT: |
882 | insn |= AARCH64_INSN_SF_BIT; |
883 | if (shift != 0 && shift != 16 && shift != 32 && shift != 48) { |
884 | pr_err("%s: invalid shift encoding %d\n" , __func__, |
885 | shift); |
886 | return AARCH64_BREAK_FAULT; |
887 | } |
888 | break; |
889 | default: |
890 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
891 | return AARCH64_BREAK_FAULT; |
892 | } |
893 | |
894 | insn |= (shift >> 4) << 21; |
895 | |
896 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
897 | |
898 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); |
899 | } |
900 | |
901 | u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst, |
902 | enum aarch64_insn_register src, |
903 | enum aarch64_insn_register reg, |
904 | int shift, |
905 | enum aarch64_insn_variant variant, |
906 | enum aarch64_insn_adsb_type type) |
907 | { |
908 | u32 insn; |
909 | |
910 | switch (type) { |
911 | case AARCH64_INSN_ADSB_ADD: |
912 | insn = aarch64_insn_get_add_value(); |
913 | break; |
914 | case AARCH64_INSN_ADSB_SUB: |
915 | insn = aarch64_insn_get_sub_value(); |
916 | break; |
917 | case AARCH64_INSN_ADSB_ADD_SETFLAGS: |
918 | insn = aarch64_insn_get_adds_value(); |
919 | break; |
920 | case AARCH64_INSN_ADSB_SUB_SETFLAGS: |
921 | insn = aarch64_insn_get_subs_value(); |
922 | break; |
923 | default: |
924 | pr_err("%s: unknown add/sub encoding %d\n" , __func__, type); |
925 | return AARCH64_BREAK_FAULT; |
926 | } |
927 | |
928 | switch (variant) { |
929 | case AARCH64_INSN_VARIANT_32BIT: |
930 | if (shift & ~(SZ_32 - 1)) { |
931 | pr_err("%s: invalid shift encoding %d\n" , __func__, |
932 | shift); |
933 | return AARCH64_BREAK_FAULT; |
934 | } |
935 | break; |
936 | case AARCH64_INSN_VARIANT_64BIT: |
937 | insn |= AARCH64_INSN_SF_BIT; |
938 | if (shift & ~(SZ_64 - 1)) { |
939 | pr_err("%s: invalid shift encoding %d\n" , __func__, |
940 | shift); |
941 | return AARCH64_BREAK_FAULT; |
942 | } |
943 | break; |
944 | default: |
945 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
946 | return AARCH64_BREAK_FAULT; |
947 | } |
948 | |
949 | |
950 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
951 | |
952 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); |
953 | |
954 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); |
955 | |
956 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); |
957 | } |
958 | |
959 | u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst, |
960 | enum aarch64_insn_register src, |
961 | enum aarch64_insn_variant variant, |
962 | enum aarch64_insn_data1_type type) |
963 | { |
964 | u32 insn; |
965 | |
966 | switch (type) { |
967 | case AARCH64_INSN_DATA1_REVERSE_16: |
968 | insn = aarch64_insn_get_rev16_value(); |
969 | break; |
970 | case AARCH64_INSN_DATA1_REVERSE_32: |
971 | insn = aarch64_insn_get_rev32_value(); |
972 | break; |
973 | case AARCH64_INSN_DATA1_REVERSE_64: |
974 | if (variant != AARCH64_INSN_VARIANT_64BIT) { |
975 | pr_err("%s: invalid variant for reverse64 %d\n" , |
976 | __func__, variant); |
977 | return AARCH64_BREAK_FAULT; |
978 | } |
979 | insn = aarch64_insn_get_rev64_value(); |
980 | break; |
981 | default: |
982 | pr_err("%s: unknown data1 encoding %d\n" , __func__, type); |
983 | return AARCH64_BREAK_FAULT; |
984 | } |
985 | |
986 | switch (variant) { |
987 | case AARCH64_INSN_VARIANT_32BIT: |
988 | break; |
989 | case AARCH64_INSN_VARIANT_64BIT: |
990 | insn |= AARCH64_INSN_SF_BIT; |
991 | break; |
992 | default: |
993 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
994 | return AARCH64_BREAK_FAULT; |
995 | } |
996 | |
997 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
998 | |
999 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); |
1000 | } |
1001 | |
1002 | u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst, |
1003 | enum aarch64_insn_register src, |
1004 | enum aarch64_insn_register reg, |
1005 | enum aarch64_insn_variant variant, |
1006 | enum aarch64_insn_data2_type type) |
1007 | { |
1008 | u32 insn; |
1009 | |
1010 | switch (type) { |
1011 | case AARCH64_INSN_DATA2_UDIV: |
1012 | insn = aarch64_insn_get_udiv_value(); |
1013 | break; |
1014 | case AARCH64_INSN_DATA2_SDIV: |
1015 | insn = aarch64_insn_get_sdiv_value(); |
1016 | break; |
1017 | case AARCH64_INSN_DATA2_LSLV: |
1018 | insn = aarch64_insn_get_lslv_value(); |
1019 | break; |
1020 | case AARCH64_INSN_DATA2_LSRV: |
1021 | insn = aarch64_insn_get_lsrv_value(); |
1022 | break; |
1023 | case AARCH64_INSN_DATA2_ASRV: |
1024 | insn = aarch64_insn_get_asrv_value(); |
1025 | break; |
1026 | case AARCH64_INSN_DATA2_RORV: |
1027 | insn = aarch64_insn_get_rorv_value(); |
1028 | break; |
1029 | default: |
1030 | pr_err("%s: unknown data2 encoding %d\n" , __func__, type); |
1031 | return AARCH64_BREAK_FAULT; |
1032 | } |
1033 | |
1034 | switch (variant) { |
1035 | case AARCH64_INSN_VARIANT_32BIT: |
1036 | break; |
1037 | case AARCH64_INSN_VARIANT_64BIT: |
1038 | insn |= AARCH64_INSN_SF_BIT; |
1039 | break; |
1040 | default: |
1041 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
1042 | return AARCH64_BREAK_FAULT; |
1043 | } |
1044 | |
1045 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
1046 | |
1047 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); |
1048 | |
1049 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); |
1050 | } |
1051 | |
1052 | u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst, |
1053 | enum aarch64_insn_register src, |
1054 | enum aarch64_insn_register reg1, |
1055 | enum aarch64_insn_register reg2, |
1056 | enum aarch64_insn_variant variant, |
1057 | enum aarch64_insn_data3_type type) |
1058 | { |
1059 | u32 insn; |
1060 | |
1061 | switch (type) { |
1062 | case AARCH64_INSN_DATA3_MADD: |
1063 | insn = aarch64_insn_get_madd_value(); |
1064 | break; |
1065 | case AARCH64_INSN_DATA3_MSUB: |
1066 | insn = aarch64_insn_get_msub_value(); |
1067 | break; |
1068 | default: |
1069 | pr_err("%s: unknown data3 encoding %d\n" , __func__, type); |
1070 | return AARCH64_BREAK_FAULT; |
1071 | } |
1072 | |
1073 | switch (variant) { |
1074 | case AARCH64_INSN_VARIANT_32BIT: |
1075 | break; |
1076 | case AARCH64_INSN_VARIANT_64BIT: |
1077 | insn |= AARCH64_INSN_SF_BIT; |
1078 | break; |
1079 | default: |
1080 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
1081 | return AARCH64_BREAK_FAULT; |
1082 | } |
1083 | |
1084 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
1085 | |
1086 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src); |
1087 | |
1088 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, |
1089 | reg1); |
1090 | |
1091 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, |
1092 | reg2); |
1093 | } |
1094 | |
1095 | u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst, |
1096 | enum aarch64_insn_register src, |
1097 | enum aarch64_insn_register reg, |
1098 | int shift, |
1099 | enum aarch64_insn_variant variant, |
1100 | enum aarch64_insn_logic_type type) |
1101 | { |
1102 | u32 insn; |
1103 | |
1104 | switch (type) { |
1105 | case AARCH64_INSN_LOGIC_AND: |
1106 | insn = aarch64_insn_get_and_value(); |
1107 | break; |
1108 | case AARCH64_INSN_LOGIC_BIC: |
1109 | insn = aarch64_insn_get_bic_value(); |
1110 | break; |
1111 | case AARCH64_INSN_LOGIC_ORR: |
1112 | insn = aarch64_insn_get_orr_value(); |
1113 | break; |
1114 | case AARCH64_INSN_LOGIC_ORN: |
1115 | insn = aarch64_insn_get_orn_value(); |
1116 | break; |
1117 | case AARCH64_INSN_LOGIC_EOR: |
1118 | insn = aarch64_insn_get_eor_value(); |
1119 | break; |
1120 | case AARCH64_INSN_LOGIC_EON: |
1121 | insn = aarch64_insn_get_eon_value(); |
1122 | break; |
1123 | case AARCH64_INSN_LOGIC_AND_SETFLAGS: |
1124 | insn = aarch64_insn_get_ands_value(); |
1125 | break; |
1126 | case AARCH64_INSN_LOGIC_BIC_SETFLAGS: |
1127 | insn = aarch64_insn_get_bics_value(); |
1128 | break; |
1129 | default: |
1130 | pr_err("%s: unknown logical encoding %d\n" , __func__, type); |
1131 | return AARCH64_BREAK_FAULT; |
1132 | } |
1133 | |
1134 | switch (variant) { |
1135 | case AARCH64_INSN_VARIANT_32BIT: |
1136 | if (shift & ~(SZ_32 - 1)) { |
1137 | pr_err("%s: invalid shift encoding %d\n" , __func__, |
1138 | shift); |
1139 | return AARCH64_BREAK_FAULT; |
1140 | } |
1141 | break; |
1142 | case AARCH64_INSN_VARIANT_64BIT: |
1143 | insn |= AARCH64_INSN_SF_BIT; |
1144 | if (shift & ~(SZ_64 - 1)) { |
1145 | pr_err("%s: invalid shift encoding %d\n" , __func__, |
1146 | shift); |
1147 | return AARCH64_BREAK_FAULT; |
1148 | } |
1149 | break; |
1150 | default: |
1151 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
1152 | return AARCH64_BREAK_FAULT; |
1153 | } |
1154 | |
1155 | |
1156 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); |
1157 | |
1158 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); |
1159 | |
1160 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); |
1161 | |
1162 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); |
1163 | } |
1164 | |
1165 | /* |
1166 | * MOV (register) is architecturally an alias of ORR (shifted register) where |
1167 | * MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m> |
1168 | */ |
1169 | u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst, |
1170 | enum aarch64_insn_register src, |
1171 | enum aarch64_insn_variant variant) |
1172 | { |
1173 | return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR, |
1174 | src, 0, variant, |
1175 | AARCH64_INSN_LOGIC_ORR); |
1176 | } |
1177 | |
1178 | u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr, |
1179 | enum aarch64_insn_register reg, |
1180 | enum aarch64_insn_adr_type type) |
1181 | { |
1182 | u32 insn; |
1183 | s32 offset; |
1184 | |
1185 | switch (type) { |
1186 | case AARCH64_INSN_ADR_TYPE_ADR: |
1187 | insn = aarch64_insn_get_adr_value(); |
1188 | offset = addr - pc; |
1189 | break; |
1190 | case AARCH64_INSN_ADR_TYPE_ADRP: |
1191 | insn = aarch64_insn_get_adrp_value(); |
1192 | offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12; |
1193 | break; |
1194 | default: |
1195 | pr_err("%s: unknown adr encoding %d\n" , __func__, type); |
1196 | return AARCH64_BREAK_FAULT; |
1197 | } |
1198 | |
1199 | if (offset < -SZ_1M || offset >= SZ_1M) |
1200 | return AARCH64_BREAK_FAULT; |
1201 | |
1202 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg); |
1203 | |
1204 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset); |
1205 | } |
1206 | |
1207 | /* |
1208 | * Decode the imm field of a branch, and return the byte offset as a |
1209 | * signed value (so it can be used when computing a new branch |
1210 | * target). |
1211 | */ |
1212 | s32 aarch64_get_branch_offset(u32 insn) |
1213 | { |
1214 | s32 imm; |
1215 | |
1216 | if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) { |
1217 | imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn); |
1218 | return (imm << 6) >> 4; |
1219 | } |
1220 | |
1221 | if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || |
1222 | aarch64_insn_is_bcond(insn)) { |
1223 | imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn); |
1224 | return (imm << 13) >> 11; |
1225 | } |
1226 | |
1227 | if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) { |
1228 | imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn); |
1229 | return (imm << 18) >> 16; |
1230 | } |
1231 | |
1232 | /* Unhandled instruction */ |
1233 | BUG(); |
1234 | } |
1235 | |
1236 | /* |
1237 | * Encode the displacement of a branch in the imm field and return the |
1238 | * updated instruction. |
1239 | */ |
1240 | u32 aarch64_set_branch_offset(u32 insn, s32 offset) |
1241 | { |
1242 | if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) |
1243 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, |
1244 | offset >> 2); |
1245 | |
1246 | if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || |
1247 | aarch64_insn_is_bcond(insn)) |
1248 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, |
1249 | offset >> 2); |
1250 | |
1251 | if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) |
1252 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn, |
1253 | offset >> 2); |
1254 | |
1255 | /* Unhandled instruction */ |
1256 | BUG(); |
1257 | } |
1258 | |
1259 | s32 aarch64_insn_adrp_get_offset(u32 insn) |
1260 | { |
1261 | BUG_ON(!aarch64_insn_is_adrp(insn)); |
1262 | return aarch64_insn_decode_immediate(AARCH64_INSN_IMM_ADR, insn) << 12; |
1263 | } |
1264 | |
1265 | u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset) |
1266 | { |
1267 | BUG_ON(!aarch64_insn_is_adrp(insn)); |
1268 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, |
1269 | offset >> 12); |
1270 | } |
1271 | |
1272 | /* |
1273 | * Extract the Op/CR data from a msr/mrs instruction. |
1274 | */ |
1275 | u32 (u32 insn) |
1276 | { |
1277 | return (insn & 0x1FFFE0) >> 5; |
1278 | } |
1279 | |
1280 | bool aarch32_insn_is_wide(u32 insn) |
1281 | { |
1282 | return insn >= 0xe800; |
1283 | } |
1284 | |
1285 | /* |
1286 | * Macros/defines for extracting register numbers from instruction. |
1287 | */ |
1288 | u32 (u32 insn, int offset) |
1289 | { |
1290 | return (insn & (0xf << offset)) >> offset; |
1291 | } |
1292 | |
1293 | #define OPC2_MASK 0x7 |
1294 | #define OPC2_OFFSET 5 |
1295 | u32 (u32 insn) |
1296 | { |
1297 | return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET; |
1298 | } |
1299 | |
1300 | #define CRM_MASK 0xf |
1301 | u32 (u32 insn) |
1302 | { |
1303 | return insn & CRM_MASK; |
1304 | } |
1305 | |
1306 | static bool range_of_ones(u64 val) |
1307 | { |
1308 | /* Doesn't handle full ones or full zeroes */ |
1309 | u64 sval = val >> __ffs64(word: val); |
1310 | |
1311 | /* One of Sean Eron Anderson's bithack tricks */ |
1312 | return ((sval + 1) & (sval)) == 0; |
1313 | } |
1314 | |
1315 | static u32 aarch64_encode_immediate(u64 imm, |
1316 | enum aarch64_insn_variant variant, |
1317 | u32 insn) |
1318 | { |
1319 | unsigned int immr, imms, n, ones, ror, esz, tmp; |
1320 | u64 mask; |
1321 | |
1322 | switch (variant) { |
1323 | case AARCH64_INSN_VARIANT_32BIT: |
1324 | esz = 32; |
1325 | break; |
1326 | case AARCH64_INSN_VARIANT_64BIT: |
1327 | insn |= AARCH64_INSN_SF_BIT; |
1328 | esz = 64; |
1329 | break; |
1330 | default: |
1331 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
1332 | return AARCH64_BREAK_FAULT; |
1333 | } |
1334 | |
1335 | mask = GENMASK(esz - 1, 0); |
1336 | |
1337 | /* Can't encode full zeroes, full ones, or value wider than the mask */ |
1338 | if (!imm || imm == mask || imm & ~mask) |
1339 | return AARCH64_BREAK_FAULT; |
1340 | |
1341 | /* |
1342 | * Inverse of Replicate(). Try to spot a repeating pattern |
1343 | * with a pow2 stride. |
1344 | */ |
1345 | for (tmp = esz / 2; tmp >= 2; tmp /= 2) { |
1346 | u64 emask = BIT(tmp) - 1; |
1347 | |
1348 | if ((imm & emask) != ((imm >> tmp) & emask)) |
1349 | break; |
1350 | |
1351 | esz = tmp; |
1352 | mask = emask; |
1353 | } |
1354 | |
1355 | /* N is only set if we're encoding a 64bit value */ |
1356 | n = esz == 64; |
1357 | |
1358 | /* Trim imm to the element size */ |
1359 | imm &= mask; |
1360 | |
1361 | /* That's how many ones we need to encode */ |
1362 | ones = hweight64(imm); |
1363 | |
1364 | /* |
1365 | * imms is set to (ones - 1), prefixed with a string of ones |
1366 | * and a zero if they fit. Cap it to 6 bits. |
1367 | */ |
1368 | imms = ones - 1; |
1369 | imms |= 0xf << ffs(esz); |
1370 | imms &= BIT(6) - 1; |
1371 | |
1372 | /* Compute the rotation */ |
1373 | if (range_of_ones(val: imm)) { |
1374 | /* |
1375 | * Pattern: 0..01..10..0 |
1376 | * |
1377 | * Compute how many rotate we need to align it right |
1378 | */ |
1379 | ror = __ffs64(word: imm); |
1380 | } else { |
1381 | /* |
1382 | * Pattern: 0..01..10..01..1 |
1383 | * |
1384 | * Fill the unused top bits with ones, and check if |
1385 | * the result is a valid immediate (all ones with a |
1386 | * contiguous ranges of zeroes). |
1387 | */ |
1388 | imm |= ~mask; |
1389 | if (!range_of_ones(~imm)) |
1390 | return AARCH64_BREAK_FAULT; |
1391 | |
1392 | /* |
1393 | * Compute the rotation to get a continuous set of |
1394 | * ones, with the first bit set at position 0 |
1395 | */ |
1396 | ror = fls64(x: ~imm); |
1397 | } |
1398 | |
1399 | /* |
1400 | * immr is the number of bits we need to rotate back to the |
1401 | * original set of ones. Note that this is relative to the |
1402 | * element size... |
1403 | */ |
1404 | immr = (esz - ror) % esz; |
1405 | |
1406 | insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, n); |
1407 | insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr); |
1408 | return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms); |
1409 | } |
1410 | |
1411 | u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type, |
1412 | enum aarch64_insn_variant variant, |
1413 | enum aarch64_insn_register Rn, |
1414 | enum aarch64_insn_register Rd, |
1415 | u64 imm) |
1416 | { |
1417 | u32 insn; |
1418 | |
1419 | switch (type) { |
1420 | case AARCH64_INSN_LOGIC_AND: |
1421 | insn = aarch64_insn_get_and_imm_value(); |
1422 | break; |
1423 | case AARCH64_INSN_LOGIC_ORR: |
1424 | insn = aarch64_insn_get_orr_imm_value(); |
1425 | break; |
1426 | case AARCH64_INSN_LOGIC_EOR: |
1427 | insn = aarch64_insn_get_eor_imm_value(); |
1428 | break; |
1429 | case AARCH64_INSN_LOGIC_AND_SETFLAGS: |
1430 | insn = aarch64_insn_get_ands_imm_value(); |
1431 | break; |
1432 | default: |
1433 | pr_err("%s: unknown logical encoding %d\n" , __func__, type); |
1434 | return AARCH64_BREAK_FAULT; |
1435 | } |
1436 | |
1437 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd); |
1438 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn); |
1439 | return aarch64_encode_immediate(imm, variant: variant, insn); |
1440 | } |
1441 | |
1442 | u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant, |
1443 | enum aarch64_insn_register Rm, |
1444 | enum aarch64_insn_register Rn, |
1445 | enum aarch64_insn_register Rd, |
1446 | u8 lsb) |
1447 | { |
1448 | u32 insn; |
1449 | |
1450 | insn = aarch64_insn_get_extr_value(); |
1451 | |
1452 | switch (variant) { |
1453 | case AARCH64_INSN_VARIANT_32BIT: |
1454 | if (lsb > 31) |
1455 | return AARCH64_BREAK_FAULT; |
1456 | break; |
1457 | case AARCH64_INSN_VARIANT_64BIT: |
1458 | if (lsb > 63) |
1459 | return AARCH64_BREAK_FAULT; |
1460 | insn |= AARCH64_INSN_SF_BIT; |
1461 | insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, 1); |
1462 | break; |
1463 | default: |
1464 | pr_err("%s: unknown variant encoding %d\n" , __func__, variant); |
1465 | return AARCH64_BREAK_FAULT; |
1466 | } |
1467 | |
1468 | insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, lsb); |
1469 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd); |
1470 | insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn); |
1471 | return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm); |
1472 | } |
1473 | |
1474 | u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) |
1475 | { |
1476 | u32 opt; |
1477 | u32 insn; |
1478 | |
1479 | switch (type) { |
1480 | case AARCH64_INSN_MB_SY: |
1481 | opt = 0xf; |
1482 | break; |
1483 | case AARCH64_INSN_MB_ST: |
1484 | opt = 0xe; |
1485 | break; |
1486 | case AARCH64_INSN_MB_LD: |
1487 | opt = 0xd; |
1488 | break; |
1489 | case AARCH64_INSN_MB_ISH: |
1490 | opt = 0xb; |
1491 | break; |
1492 | case AARCH64_INSN_MB_ISHST: |
1493 | opt = 0xa; |
1494 | break; |
1495 | case AARCH64_INSN_MB_ISHLD: |
1496 | opt = 0x9; |
1497 | break; |
1498 | case AARCH64_INSN_MB_NSH: |
1499 | opt = 0x7; |
1500 | break; |
1501 | case AARCH64_INSN_MB_NSHST: |
1502 | opt = 0x6; |
1503 | break; |
1504 | case AARCH64_INSN_MB_NSHLD: |
1505 | opt = 0x5; |
1506 | break; |
1507 | default: |
1508 | pr_err("%s: unknown dmb type %d\n" , __func__, type); |
1509 | return AARCH64_BREAK_FAULT; |
1510 | } |
1511 | |
1512 | insn = aarch64_insn_get_dmb_value(); |
1513 | insn &= ~GENMASK(11, 8); |
1514 | insn |= (opt << 8); |
1515 | |
1516 | return insn; |
1517 | } |
1518 | |