1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * Common functionality for RV32 and RV64 BPF JIT compilers |
4 | * |
5 | * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> |
6 | * |
7 | */ |
8 | |
9 | #ifndef _BPF_JIT_H |
10 | #define _BPF_JIT_H |
11 | |
12 | #include <linux/bpf.h> |
13 | #include <linux/filter.h> |
14 | #include <asm/cacheflush.h> |
15 | |
16 | static inline bool rvc_enabled(void) |
17 | { |
18 | return IS_ENABLED(CONFIG_RISCV_ISA_C); |
19 | } |
20 | |
21 | static inline bool rvzbb_enabled(void) |
22 | { |
23 | return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB); |
24 | } |
25 | |
26 | enum { |
27 | RV_REG_ZERO = 0, /* The constant value 0 */ |
28 | RV_REG_RA = 1, /* Return address */ |
29 | RV_REG_SP = 2, /* Stack pointer */ |
30 | RV_REG_GP = 3, /* Global pointer */ |
31 | RV_REG_TP = 4, /* Thread pointer */ |
32 | RV_REG_T0 = 5, /* Temporaries */ |
33 | RV_REG_T1 = 6, |
34 | RV_REG_T2 = 7, |
35 | RV_REG_FP = 8, /* Saved register/frame pointer */ |
36 | RV_REG_S1 = 9, /* Saved register */ |
37 | RV_REG_A0 = 10, /* Function argument/return values */ |
38 | RV_REG_A1 = 11, /* Function arguments */ |
39 | RV_REG_A2 = 12, |
40 | RV_REG_A3 = 13, |
41 | RV_REG_A4 = 14, |
42 | RV_REG_A5 = 15, |
43 | RV_REG_A6 = 16, |
44 | RV_REG_A7 = 17, |
45 | RV_REG_S2 = 18, /* Saved registers */ |
46 | RV_REG_S3 = 19, |
47 | RV_REG_S4 = 20, |
48 | RV_REG_S5 = 21, |
49 | RV_REG_S6 = 22, |
50 | RV_REG_S7 = 23, |
51 | RV_REG_S8 = 24, |
52 | RV_REG_S9 = 25, |
53 | RV_REG_S10 = 26, |
54 | RV_REG_S11 = 27, |
55 | RV_REG_T3 = 28, /* Temporaries */ |
56 | RV_REG_T4 = 29, |
57 | RV_REG_T5 = 30, |
58 | RV_REG_T6 = 31, |
59 | }; |
60 | |
61 | static inline bool is_creg(u8 reg) |
62 | { |
63 | return (1 << reg) & (BIT(RV_REG_FP) | |
64 | BIT(RV_REG_S1) | |
65 | BIT(RV_REG_A0) | |
66 | BIT(RV_REG_A1) | |
67 | BIT(RV_REG_A2) | |
68 | BIT(RV_REG_A3) | |
69 | BIT(RV_REG_A4) | |
70 | BIT(RV_REG_A5)); |
71 | } |
72 | |
73 | struct rv_jit_context { |
74 | struct bpf_prog *prog; |
75 | u16 *insns; /* RV insns */ |
76 | u16 *ro_insns; |
77 | int ninsns; |
78 | int prologue_len; |
79 | int epilogue_offset; |
80 | int *offset; /* BPF to RV */ |
81 | int nexentries; |
82 | unsigned long flags; |
83 | int stack_size; |
84 | }; |
85 | |
86 | /* Convert from ninsns to bytes. */ |
87 | static inline int ninsns_rvoff(int ninsns) |
88 | { |
89 | return ninsns << 1; |
90 | } |
91 | |
92 | struct rv_jit_data { |
93 | struct bpf_binary_header *; |
94 | struct bpf_binary_header *; |
95 | u8 *image; |
96 | u8 *ro_image; |
97 | struct rv_jit_context ctx; |
98 | }; |
99 | |
100 | static inline void bpf_fill_ill_insns(void *area, unsigned int size) |
101 | { |
102 | memset(area, 0, size); |
103 | } |
104 | |
105 | static inline void bpf_flush_icache(void *start, void *end) |
106 | { |
107 | flush_icache_range(start: (unsigned long)start, end: (unsigned long)end); |
108 | } |
109 | |
110 | /* Emit a 4-byte riscv instruction. */ |
111 | static inline void emit(const u32 insn, struct rv_jit_context *ctx) |
112 | { |
113 | if (ctx->insns) { |
114 | ctx->insns[ctx->ninsns] = insn; |
115 | ctx->insns[ctx->ninsns + 1] = (insn >> 16); |
116 | } |
117 | |
118 | ctx->ninsns += 2; |
119 | } |
120 | |
121 | /* Emit a 2-byte riscv compressed instruction. */ |
122 | static inline void emitc(const u16 insn, struct rv_jit_context *ctx) |
123 | { |
124 | BUILD_BUG_ON(!rvc_enabled()); |
125 | |
126 | if (ctx->insns) |
127 | ctx->insns[ctx->ninsns] = insn; |
128 | |
129 | ctx->ninsns++; |
130 | } |
131 | |
132 | static inline int epilogue_offset(struct rv_jit_context *ctx) |
133 | { |
134 | int to = ctx->epilogue_offset, from = ctx->ninsns; |
135 | |
136 | return ninsns_rvoff(ninsns: to - from); |
137 | } |
138 | |
139 | /* Return -1 or inverted cond. */ |
140 | static inline int invert_bpf_cond(u8 cond) |
141 | { |
142 | switch (cond) { |
143 | case BPF_JEQ: |
144 | return BPF_JNE; |
145 | case BPF_JGT: |
146 | return BPF_JLE; |
147 | case BPF_JLT: |
148 | return BPF_JGE; |
149 | case BPF_JGE: |
150 | return BPF_JLT; |
151 | case BPF_JLE: |
152 | return BPF_JGT; |
153 | case BPF_JNE: |
154 | return BPF_JEQ; |
155 | case BPF_JSGT: |
156 | return BPF_JSLE; |
157 | case BPF_JSLT: |
158 | return BPF_JSGE; |
159 | case BPF_JSGE: |
160 | return BPF_JSLT; |
161 | case BPF_JSLE: |
162 | return BPF_JSGT; |
163 | } |
164 | return -1; |
165 | } |
166 | |
167 | static inline bool is_6b_int(long val) |
168 | { |
169 | return -(1L << 5) <= val && val < (1L << 5); |
170 | } |
171 | |
172 | static inline bool is_7b_uint(unsigned long val) |
173 | { |
174 | return val < (1UL << 7); |
175 | } |
176 | |
177 | static inline bool is_8b_uint(unsigned long val) |
178 | { |
179 | return val < (1UL << 8); |
180 | } |
181 | |
182 | static inline bool is_9b_uint(unsigned long val) |
183 | { |
184 | return val < (1UL << 9); |
185 | } |
186 | |
187 | static inline bool is_10b_int(long val) |
188 | { |
189 | return -(1L << 9) <= val && val < (1L << 9); |
190 | } |
191 | |
192 | static inline bool is_10b_uint(unsigned long val) |
193 | { |
194 | return val < (1UL << 10); |
195 | } |
196 | |
197 | static inline bool is_12b_int(long val) |
198 | { |
199 | return -(1L << 11) <= val && val < (1L << 11); |
200 | } |
201 | |
202 | static inline int is_12b_check(int off, int insn) |
203 | { |
204 | if (!is_12b_int(val: off)) { |
205 | pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n" , |
206 | insn, (int)off); |
207 | return -1; |
208 | } |
209 | return 0; |
210 | } |
211 | |
212 | static inline bool is_13b_int(long val) |
213 | { |
214 | return -(1L << 12) <= val && val < (1L << 12); |
215 | } |
216 | |
217 | static inline bool is_21b_int(long val) |
218 | { |
219 | return -(1L << 20) <= val && val < (1L << 20); |
220 | } |
221 | |
222 | static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) |
223 | { |
224 | int from, to; |
225 | |
226 | off++; /* BPF branch is from PC+1, RV is from PC */ |
227 | from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len; |
228 | to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len; |
229 | return ninsns_rvoff(ninsns: to - from); |
230 | } |
231 | |
232 | /* Instruction formats. */ |
233 | |
234 | static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, |
235 | u8 opcode) |
236 | { |
237 | return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | |
238 | (rd << 7) | opcode; |
239 | } |
240 | |
241 | static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) |
242 | { |
243 | return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | |
244 | opcode; |
245 | } |
246 | |
247 | static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) |
248 | { |
249 | u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; |
250 | |
251 | return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | |
252 | (imm4_0 << 7) | opcode; |
253 | } |
254 | |
255 | static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) |
256 | { |
257 | u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); |
258 | u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); |
259 | |
260 | return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | |
261 | (imm4_1 << 7) | opcode; |
262 | } |
263 | |
264 | static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) |
265 | { |
266 | return (imm31_12 << 12) | (rd << 7) | opcode; |
267 | } |
268 | |
269 | static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) |
270 | { |
271 | u32 imm; |
272 | |
273 | imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | |
274 | ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); |
275 | |
276 | return (imm << 12) | (rd << 7) | opcode; |
277 | } |
278 | |
279 | static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, |
280 | u8 funct3, u8 rd, u8 opcode) |
281 | { |
282 | u8 funct7 = (funct5 << 2) | (aq << 1) | rl; |
283 | |
284 | return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); |
285 | } |
286 | |
287 | /* RISC-V compressed instruction formats. */ |
288 | |
289 | static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) |
290 | { |
291 | return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; |
292 | } |
293 | |
294 | static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) |
295 | { |
296 | u32 imm; |
297 | |
298 | imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); |
299 | return (funct3 << 13) | (rd << 7) | op | imm; |
300 | } |
301 | |
302 | static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) |
303 | { |
304 | return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; |
305 | } |
306 | |
307 | static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) |
308 | { |
309 | return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; |
310 | } |
311 | |
312 | static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, |
313 | u8 op) |
314 | { |
315 | return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | |
316 | (imm_lo << 5) | ((rd & 0x7) << 2) | op; |
317 | } |
318 | |
319 | static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, |
320 | u8 op) |
321 | { |
322 | return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | |
323 | (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; |
324 | } |
325 | |
326 | static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) |
327 | { |
328 | return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | |
329 | ((rs2 & 0x7) << 2) | op; |
330 | } |
331 | |
332 | static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) |
333 | { |
334 | u32 imm; |
335 | |
336 | imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); |
337 | return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; |
338 | } |
339 | |
340 | /* Instructions shared by both RV32 and RV64. */ |
341 | |
342 | static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) |
343 | { |
344 | return rv_i_insn(imm11_0, rs1, funct3: 0, rd, opcode: 0x13); |
345 | } |
346 | |
347 | static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) |
348 | { |
349 | return rv_i_insn(imm11_0, rs1, funct3: 7, rd, opcode: 0x13); |
350 | } |
351 | |
352 | static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) |
353 | { |
354 | return rv_i_insn(imm11_0, rs1, funct3: 6, rd, opcode: 0x13); |
355 | } |
356 | |
357 | static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) |
358 | { |
359 | return rv_i_insn(imm11_0, rs1, funct3: 4, rd, opcode: 0x13); |
360 | } |
361 | |
362 | static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) |
363 | { |
364 | return rv_i_insn(imm11_0, rs1, funct3: 1, rd, opcode: 0x13); |
365 | } |
366 | |
367 | static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) |
368 | { |
369 | return rv_i_insn(imm11_0, rs1, funct3: 5, rd, opcode: 0x13); |
370 | } |
371 | |
372 | static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) |
373 | { |
374 | return rv_i_insn(imm11_0: 0x400 | imm11_0, rs1, funct3: 5, rd, opcode: 0x13); |
375 | } |
376 | |
377 | static inline u32 rv_lui(u8 rd, u32 imm31_12) |
378 | { |
379 | return rv_u_insn(imm31_12, rd, opcode: 0x37); |
380 | } |
381 | |
382 | static inline u32 rv_auipc(u8 rd, u32 imm31_12) |
383 | { |
384 | return rv_u_insn(imm31_12, rd, opcode: 0x17); |
385 | } |
386 | |
387 | static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) |
388 | { |
389 | return rv_r_insn(funct7: 0, rs2, rs1, funct3: 0, rd, opcode: 0x33); |
390 | } |
391 | |
392 | static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) |
393 | { |
394 | return rv_r_insn(funct7: 0x20, rs2, rs1, funct3: 0, rd, opcode: 0x33); |
395 | } |
396 | |
397 | static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) |
398 | { |
399 | return rv_r_insn(funct7: 0, rs2, rs1, funct3: 3, rd, opcode: 0x33); |
400 | } |
401 | |
402 | static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) |
403 | { |
404 | return rv_r_insn(funct7: 0, rs2, rs1, funct3: 7, rd, opcode: 0x33); |
405 | } |
406 | |
407 | static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) |
408 | { |
409 | return rv_r_insn(funct7: 0, rs2, rs1, funct3: 6, rd, opcode: 0x33); |
410 | } |
411 | |
412 | static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) |
413 | { |
414 | return rv_r_insn(funct7: 0, rs2, rs1, funct3: 4, rd, opcode: 0x33); |
415 | } |
416 | |
417 | static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) |
418 | { |
419 | return rv_r_insn(funct7: 0, rs2, rs1, funct3: 1, rd, opcode: 0x33); |
420 | } |
421 | |
422 | static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) |
423 | { |
424 | return rv_r_insn(funct7: 0, rs2, rs1, funct3: 5, rd, opcode: 0x33); |
425 | } |
426 | |
427 | static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) |
428 | { |
429 | return rv_r_insn(funct7: 0x20, rs2, rs1, funct3: 5, rd, opcode: 0x33); |
430 | } |
431 | |
432 | static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) |
433 | { |
434 | return rv_r_insn(funct7: 1, rs2, rs1, funct3: 0, rd, opcode: 0x33); |
435 | } |
436 | |
437 | static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) |
438 | { |
439 | return rv_r_insn(funct7: 1, rs2, rs1, funct3: 3, rd, opcode: 0x33); |
440 | } |
441 | |
442 | static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2) |
443 | { |
444 | return rv_r_insn(funct7: 1, rs2, rs1, funct3: 4, rd, opcode: 0x33); |
445 | } |
446 | |
447 | static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) |
448 | { |
449 | return rv_r_insn(funct7: 1, rs2, rs1, funct3: 5, rd, opcode: 0x33); |
450 | } |
451 | |
452 | static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2) |
453 | { |
454 | return rv_r_insn(funct7: 1, rs2, rs1, funct3: 6, rd, opcode: 0x33); |
455 | } |
456 | |
457 | static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) |
458 | { |
459 | return rv_r_insn(funct7: 1, rs2, rs1, funct3: 7, rd, opcode: 0x33); |
460 | } |
461 | |
462 | static inline u32 rv_jal(u8 rd, u32 imm20_1) |
463 | { |
464 | return rv_j_insn(imm20_1, rd, opcode: 0x6f); |
465 | } |
466 | |
467 | static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) |
468 | { |
469 | return rv_i_insn(imm11_0, rs1, funct3: 0, rd, opcode: 0x67); |
470 | } |
471 | |
472 | static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) |
473 | { |
474 | return rv_b_insn(imm12_1, rs2, rs1, funct3: 0, opcode: 0x63); |
475 | } |
476 | |
477 | static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) |
478 | { |
479 | return rv_b_insn(imm12_1, rs2, rs1, funct3: 1, opcode: 0x63); |
480 | } |
481 | |
482 | static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) |
483 | { |
484 | return rv_b_insn(imm12_1, rs2, rs1, funct3: 6, opcode: 0x63); |
485 | } |
486 | |
487 | static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) |
488 | { |
489 | return rv_bltu(rs1: rs2, rs2: rs1, imm12_1); |
490 | } |
491 | |
492 | static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) |
493 | { |
494 | return rv_b_insn(imm12_1, rs2, rs1, funct3: 7, opcode: 0x63); |
495 | } |
496 | |
497 | static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) |
498 | { |
499 | return rv_bgeu(rs1: rs2, rs2: rs1, imm12_1); |
500 | } |
501 | |
502 | static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) |
503 | { |
504 | return rv_b_insn(imm12_1, rs2, rs1, funct3: 4, opcode: 0x63); |
505 | } |
506 | |
507 | static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) |
508 | { |
509 | return rv_blt(rs1: rs2, rs2: rs1, imm12_1); |
510 | } |
511 | |
512 | static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) |
513 | { |
514 | return rv_b_insn(imm12_1, rs2, rs1, funct3: 5, opcode: 0x63); |
515 | } |
516 | |
517 | static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) |
518 | { |
519 | return rv_bge(rs1: rs2, rs2: rs1, imm12_1); |
520 | } |
521 | |
522 | static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1) |
523 | { |
524 | return rv_i_insn(imm11_0, rs1, funct3: 0, rd, opcode: 0x03); |
525 | } |
526 | |
527 | static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1) |
528 | { |
529 | return rv_i_insn(imm11_0, rs1, funct3: 1, rd, opcode: 0x03); |
530 | } |
531 | |
532 | static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) |
533 | { |
534 | return rv_i_insn(imm11_0, rs1, funct3: 2, rd, opcode: 0x03); |
535 | } |
536 | |
537 | static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) |
538 | { |
539 | return rv_i_insn(imm11_0, rs1, funct3: 4, rd, opcode: 0x03); |
540 | } |
541 | |
542 | static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) |
543 | { |
544 | return rv_i_insn(imm11_0, rs1, funct3: 5, rd, opcode: 0x03); |
545 | } |
546 | |
547 | static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) |
548 | { |
549 | return rv_s_insn(imm11_0, rs2, rs1, funct3: 0, opcode: 0x23); |
550 | } |
551 | |
552 | static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) |
553 | { |
554 | return rv_s_insn(imm11_0, rs2, rs1, funct3: 1, opcode: 0x23); |
555 | } |
556 | |
557 | static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) |
558 | { |
559 | return rv_s_insn(imm11_0, rs2, rs1, funct3: 2, opcode: 0x23); |
560 | } |
561 | |
562 | static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
563 | { |
564 | return rv_amo_insn(funct5: 0, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f); |
565 | } |
566 | |
567 | static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
568 | { |
569 | return rv_amo_insn(funct5: 0xc, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f); |
570 | } |
571 | |
572 | static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
573 | { |
574 | return rv_amo_insn(funct5: 0x8, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f); |
575 | } |
576 | |
577 | static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
578 | { |
579 | return rv_amo_insn(funct5: 0x4, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f); |
580 | } |
581 | |
582 | static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
583 | { |
584 | return rv_amo_insn(funct5: 0x1, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f); |
585 | } |
586 | |
587 | static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
588 | { |
589 | return rv_amo_insn(funct5: 0x2, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f); |
590 | } |
591 | |
592 | static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
593 | { |
594 | return rv_amo_insn(funct5: 0x3, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f); |
595 | } |
596 | |
597 | static inline u32 rv_fence(u8 pred, u8 succ) |
598 | { |
599 | u16 imm11_0 = pred << 4 | succ; |
600 | |
601 | return rv_i_insn(imm11_0, rs1: 0, funct3: 0, rd: 0, opcode: 0xf); |
602 | } |
603 | |
604 | static inline u32 rv_nop(void) |
605 | { |
606 | return rv_i_insn(imm11_0: 0, rs1: 0, funct3: 0, rd: 0, opcode: 0x13); |
607 | } |
608 | |
609 | /* RVC instrutions. */ |
610 | |
611 | static inline u16 rvc_addi4spn(u8 rd, u32 imm10) |
612 | { |
613 | u32 imm; |
614 | |
615 | imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | |
616 | ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); |
617 | return rv_ciw_insn(funct3: 0x0, uimm: imm, rd, op: 0x0); |
618 | } |
619 | |
620 | static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) |
621 | { |
622 | u32 imm_hi, imm_lo; |
623 | |
624 | imm_hi = (imm7 & 0x38) >> 3; |
625 | imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); |
626 | return rv_cl_insn(funct3: 0x2, imm_hi, rs1, imm_lo, rd, op: 0x0); |
627 | } |
628 | |
629 | static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) |
630 | { |
631 | u32 imm_hi, imm_lo; |
632 | |
633 | imm_hi = (imm7 & 0x38) >> 3; |
634 | imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); |
635 | return rv_cs_insn(funct3: 0x6, imm_hi, rs1, imm_lo, rs2, op: 0x0); |
636 | } |
637 | |
638 | static inline u16 rvc_addi(u8 rd, u32 imm6) |
639 | { |
640 | return rv_ci_insn(funct3: 0, imm6, rd, op: 0x1); |
641 | } |
642 | |
643 | static inline u16 rvc_li(u8 rd, u32 imm6) |
644 | { |
645 | return rv_ci_insn(funct3: 0x2, imm6, rd, op: 0x1); |
646 | } |
647 | |
648 | static inline u16 rvc_addi16sp(u32 imm10) |
649 | { |
650 | u32 imm; |
651 | |
652 | imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | |
653 | ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); |
654 | return rv_ci_insn(funct3: 0x3, imm6: imm, rd: RV_REG_SP, op: 0x1); |
655 | } |
656 | |
657 | static inline u16 rvc_lui(u8 rd, u32 imm6) |
658 | { |
659 | return rv_ci_insn(funct3: 0x3, imm6, rd, op: 0x1); |
660 | } |
661 | |
662 | static inline u16 rvc_srli(u8 rd, u32 imm6) |
663 | { |
664 | return rv_cb_insn(funct3: 0x4, imm6, funct2: 0, rd, op: 0x1); |
665 | } |
666 | |
667 | static inline u16 rvc_srai(u8 rd, u32 imm6) |
668 | { |
669 | return rv_cb_insn(funct3: 0x4, imm6, funct2: 0x1, rd, op: 0x1); |
670 | } |
671 | |
672 | static inline u16 rvc_andi(u8 rd, u32 imm6) |
673 | { |
674 | return rv_cb_insn(funct3: 0x4, imm6, funct2: 0x2, rd, op: 0x1); |
675 | } |
676 | |
677 | static inline u16 rvc_sub(u8 rd, u8 rs) |
678 | { |
679 | return rv_ca_insn(funct6: 0x23, rd, funct2: 0, rs2: rs, op: 0x1); |
680 | } |
681 | |
682 | static inline u16 rvc_xor(u8 rd, u8 rs) |
683 | { |
684 | return rv_ca_insn(funct6: 0x23, rd, funct2: 0x1, rs2: rs, op: 0x1); |
685 | } |
686 | |
687 | static inline u16 rvc_or(u8 rd, u8 rs) |
688 | { |
689 | return rv_ca_insn(funct6: 0x23, rd, funct2: 0x2, rs2: rs, op: 0x1); |
690 | } |
691 | |
692 | static inline u16 rvc_and(u8 rd, u8 rs) |
693 | { |
694 | return rv_ca_insn(funct6: 0x23, rd, funct2: 0x3, rs2: rs, op: 0x1); |
695 | } |
696 | |
697 | static inline u16 rvc_slli(u8 rd, u32 imm6) |
698 | { |
699 | return rv_ci_insn(funct3: 0, imm6, rd, op: 0x2); |
700 | } |
701 | |
702 | static inline u16 rvc_lwsp(u8 rd, u32 imm8) |
703 | { |
704 | u32 imm; |
705 | |
706 | imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); |
707 | return rv_ci_insn(funct3: 0x2, imm6: imm, rd, op: 0x2); |
708 | } |
709 | |
710 | static inline u16 rvc_jr(u8 rs1) |
711 | { |
712 | return rv_cr_insn(funct4: 0x8, rd: rs1, rs2: RV_REG_ZERO, op: 0x2); |
713 | } |
714 | |
715 | static inline u16 rvc_mv(u8 rd, u8 rs) |
716 | { |
717 | return rv_cr_insn(funct4: 0x8, rd, rs2: rs, op: 0x2); |
718 | } |
719 | |
720 | static inline u16 rvc_jalr(u8 rs1) |
721 | { |
722 | return rv_cr_insn(funct4: 0x9, rd: rs1, rs2: RV_REG_ZERO, op: 0x2); |
723 | } |
724 | |
725 | static inline u16 rvc_add(u8 rd, u8 rs) |
726 | { |
727 | return rv_cr_insn(funct4: 0x9, rd, rs2: rs, op: 0x2); |
728 | } |
729 | |
730 | static inline u16 rvc_swsp(u32 imm8, u8 rs2) |
731 | { |
732 | u32 imm; |
733 | |
734 | imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); |
735 | return rv_css_insn(funct3: 0x6, uimm: imm, rs2, op: 0x2); |
736 | } |
737 | |
738 | /* RVZBB instrutions. */ |
739 | static inline u32 rvzbb_sextb(u8 rd, u8 rs1) |
740 | { |
741 | return rv_i_insn(imm11_0: 0x604, rs1, funct3: 1, rd, opcode: 0x13); |
742 | } |
743 | |
744 | static inline u32 rvzbb_sexth(u8 rd, u8 rs1) |
745 | { |
746 | return rv_i_insn(imm11_0: 0x605, rs1, funct3: 1, rd, opcode: 0x13); |
747 | } |
748 | |
749 | static inline u32 rvzbb_zexth(u8 rd, u8 rs) |
750 | { |
751 | if (IS_ENABLED(CONFIG_64BIT)) |
752 | return rv_i_insn(imm11_0: 0x80, rs1: rs, funct3: 4, rd, opcode: 0x3b); |
753 | |
754 | return rv_i_insn(imm11_0: 0x80, rs1: rs, funct3: 4, rd, opcode: 0x33); |
755 | } |
756 | |
757 | static inline u32 rvzbb_rev8(u8 rd, u8 rs) |
758 | { |
759 | if (IS_ENABLED(CONFIG_64BIT)) |
760 | return rv_i_insn(imm11_0: 0x6b8, rs1: rs, funct3: 5, rd, opcode: 0x13); |
761 | |
762 | return rv_i_insn(imm11_0: 0x698, rs1: rs, funct3: 5, rd, opcode: 0x13); |
763 | } |
764 | |
765 | /* |
766 | * RV64-only instructions. |
767 | * |
768 | * These instructions are not available on RV32. Wrap them below a #if to |
769 | * ensure that the RV32 JIT doesn't emit any of these instructions. |
770 | */ |
771 | |
772 | #if __riscv_xlen == 64 |
773 | |
774 | static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) |
775 | { |
776 | return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); |
777 | } |
778 | |
779 | static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) |
780 | { |
781 | return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); |
782 | } |
783 | |
784 | static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) |
785 | { |
786 | return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); |
787 | } |
788 | |
789 | static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) |
790 | { |
791 | return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); |
792 | } |
793 | |
794 | static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) |
795 | { |
796 | return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); |
797 | } |
798 | |
799 | static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) |
800 | { |
801 | return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); |
802 | } |
803 | |
804 | static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) |
805 | { |
806 | return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); |
807 | } |
808 | |
809 | static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) |
810 | { |
811 | return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); |
812 | } |
813 | |
814 | static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) |
815 | { |
816 | return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); |
817 | } |
818 | |
819 | static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) |
820 | { |
821 | return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); |
822 | } |
823 | |
824 | static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2) |
825 | { |
826 | return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b); |
827 | } |
828 | |
829 | static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) |
830 | { |
831 | return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); |
832 | } |
833 | |
834 | static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2) |
835 | { |
836 | return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b); |
837 | } |
838 | |
839 | static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) |
840 | { |
841 | return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); |
842 | } |
843 | |
844 | static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) |
845 | { |
846 | return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); |
847 | } |
848 | |
849 | static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) |
850 | { |
851 | return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); |
852 | } |
853 | |
854 | static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) |
855 | { |
856 | return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); |
857 | } |
858 | |
859 | static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
860 | { |
861 | return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); |
862 | } |
863 | |
864 | static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
865 | { |
866 | return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f); |
867 | } |
868 | |
869 | static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
870 | { |
871 | return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f); |
872 | } |
873 | |
874 | static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
875 | { |
876 | return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f); |
877 | } |
878 | |
879 | static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
880 | { |
881 | return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f); |
882 | } |
883 | |
884 | static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
885 | { |
886 | return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f); |
887 | } |
888 | |
889 | static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) |
890 | { |
891 | return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f); |
892 | } |
893 | |
894 | /* RV64-only RVC instructions. */ |
895 | |
896 | static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) |
897 | { |
898 | u32 imm_hi, imm_lo; |
899 | |
900 | imm_hi = (imm8 & 0x38) >> 3; |
901 | imm_lo = (imm8 & 0xc0) >> 6; |
902 | return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); |
903 | } |
904 | |
905 | static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) |
906 | { |
907 | u32 imm_hi, imm_lo; |
908 | |
909 | imm_hi = (imm8 & 0x38) >> 3; |
910 | imm_lo = (imm8 & 0xc0) >> 6; |
911 | return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); |
912 | } |
913 | |
914 | static inline u16 rvc_subw(u8 rd, u8 rs) |
915 | { |
916 | return rv_ca_insn(0x27, rd, 0, rs, 0x1); |
917 | } |
918 | |
919 | static inline u16 rvc_addiw(u8 rd, u32 imm6) |
920 | { |
921 | return rv_ci_insn(0x1, imm6, rd, 0x1); |
922 | } |
923 | |
924 | static inline u16 rvc_ldsp(u8 rd, u32 imm9) |
925 | { |
926 | u32 imm; |
927 | |
928 | imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); |
929 | return rv_ci_insn(0x3, imm, rd, 0x2); |
930 | } |
931 | |
932 | static inline u16 rvc_sdsp(u32 imm9, u8 rs2) |
933 | { |
934 | u32 imm; |
935 | |
936 | imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); |
937 | return rv_css_insn(0x7, imm, rs2, 0x2); |
938 | } |
939 | |
940 | #endif /* __riscv_xlen == 64 */ |
941 | |
942 | /* Helper functions that emit RVC instructions when possible. */ |
943 | |
944 | static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) |
945 | { |
946 | if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) |
947 | emitc(insn: rvc_jalr(rs1: rs), ctx); |
948 | else if (rvc_enabled() && !rd && rs && !imm) |
949 | emitc(insn: rvc_jr(rs1: rs), ctx); |
950 | else |
951 | emit(insn: rv_jalr(rd, rs1: rs, imm11_0: imm), ctx); |
952 | } |
953 | |
954 | static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) |
955 | { |
956 | if (rvc_enabled() && rd && rs) |
957 | emitc(insn: rvc_mv(rd, rs), ctx); |
958 | else |
959 | emit(insn: rv_addi(rd, rs1: rs, imm11_0: 0), ctx); |
960 | } |
961 | |
962 | static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) |
963 | { |
964 | if (rvc_enabled() && rd && rd == rs1 && rs2) |
965 | emitc(insn: rvc_add(rd, rs: rs2), ctx); |
966 | else |
967 | emit(insn: rv_add(rd, rs1, rs2), ctx); |
968 | } |
969 | |
970 | static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) |
971 | { |
972 | if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(val: imm) && imm && !(imm & 0xf)) |
973 | emitc(insn: rvc_addi16sp(imm10: imm), ctx); |
974 | else if (rvc_enabled() && is_creg(reg: rd) && rs == RV_REG_SP && is_10b_uint(val: imm) && |
975 | !(imm & 0x3) && imm) |
976 | emitc(insn: rvc_addi4spn(rd, imm10: imm), ctx); |
977 | else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(val: imm)) |
978 | emitc(insn: rvc_addi(rd, imm6: imm), ctx); |
979 | else |
980 | emit(insn: rv_addi(rd, rs1: rs, imm11_0: imm), ctx); |
981 | } |
982 | |
983 | static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) |
984 | { |
985 | if (rvc_enabled() && rd && is_6b_int(val: imm)) |
986 | emitc(insn: rvc_li(rd, imm6: imm), ctx); |
987 | else |
988 | emit(insn: rv_addi(rd, rs1: RV_REG_ZERO, imm11_0: imm), ctx); |
989 | } |
990 | |
991 | static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) |
992 | { |
993 | if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(val: imm) && imm) |
994 | emitc(insn: rvc_lui(rd, imm6: imm), ctx); |
995 | else |
996 | emit(insn: rv_lui(rd, imm31_12: imm), ctx); |
997 | } |
998 | |
999 | static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) |
1000 | { |
1001 | if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) |
1002 | emitc(insn: rvc_slli(rd, imm6: imm), ctx); |
1003 | else |
1004 | emit(insn: rv_slli(rd, rs1: rs, imm11_0: imm), ctx); |
1005 | } |
1006 | |
1007 | static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) |
1008 | { |
1009 | if (rvc_enabled() && is_creg(reg: rd) && rd == rs && is_6b_int(val: imm)) |
1010 | emitc(insn: rvc_andi(rd, imm6: imm), ctx); |
1011 | else |
1012 | emit(insn: rv_andi(rd, rs1: rs, imm11_0: imm), ctx); |
1013 | } |
1014 | |
1015 | static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) |
1016 | { |
1017 | if (rvc_enabled() && is_creg(reg: rd) && rd == rs && imm && (u32)imm < __riscv_xlen) |
1018 | emitc(insn: rvc_srli(rd, imm6: imm), ctx); |
1019 | else |
1020 | emit(insn: rv_srli(rd, rs1: rs, imm11_0: imm), ctx); |
1021 | } |
1022 | |
1023 | static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) |
1024 | { |
1025 | if (rvc_enabled() && is_creg(reg: rd) && rd == rs && imm && (u32)imm < __riscv_xlen) |
1026 | emitc(insn: rvc_srai(rd, imm6: imm), ctx); |
1027 | else |
1028 | emit(insn: rv_srai(rd, rs1: rs, imm11_0: imm), ctx); |
1029 | } |
1030 | |
1031 | static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) |
1032 | { |
1033 | if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2)) |
1034 | emitc(insn: rvc_sub(rd, rs: rs2), ctx); |
1035 | else |
1036 | emit(insn: rv_sub(rd, rs1, rs2), ctx); |
1037 | } |
1038 | |
1039 | static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) |
1040 | { |
1041 | if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2)) |
1042 | emitc(insn: rvc_or(rd, rs: rs2), ctx); |
1043 | else |
1044 | emit(insn: rv_or(rd, rs1, rs2), ctx); |
1045 | } |
1046 | |
1047 | static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) |
1048 | { |
1049 | if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2)) |
1050 | emitc(insn: rvc_and(rd, rs: rs2), ctx); |
1051 | else |
1052 | emit(insn: rv_and(rd, rs1, rs2), ctx); |
1053 | } |
1054 | |
1055 | static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) |
1056 | { |
1057 | if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2)) |
1058 | emitc(insn: rvc_xor(rd, rs: rs2), ctx); |
1059 | else |
1060 | emit(insn: rv_xor(rd, rs1, rs2), ctx); |
1061 | } |
1062 | |
1063 | static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) |
1064 | { |
1065 | if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(val: off) && !(off & 0x3)) |
1066 | emitc(insn: rvc_lwsp(rd, imm8: off), ctx); |
1067 | else if (rvc_enabled() && is_creg(reg: rd) && is_creg(reg: rs1) && is_7b_uint(val: off) && !(off & 0x3)) |
1068 | emitc(insn: rvc_lw(rd, imm7: off, rs1), ctx); |
1069 | else |
1070 | emit(insn: rv_lw(rd, imm11_0: off, rs1), ctx); |
1071 | } |
1072 | |
1073 | static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) |
1074 | { |
1075 | if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(val: off) && !(off & 0x3)) |
1076 | emitc(insn: rvc_swsp(imm8: off, rs2), ctx); |
1077 | else if (rvc_enabled() && is_creg(reg: rs1) && is_creg(reg: rs2) && is_7b_uint(val: off) && !(off & 0x3)) |
1078 | emitc(insn: rvc_sw(rs1, imm7: off, rs2), ctx); |
1079 | else |
1080 | emit(insn: rv_sw(rs1, imm11_0: off, rs2), ctx); |
1081 | } |
1082 | |
1083 | /* RV64-only helper functions. */ |
1084 | #if __riscv_xlen == 64 |
1085 | |
1086 | static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) |
1087 | { |
1088 | if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) |
1089 | emitc(rvc_addiw(rd, imm), ctx); |
1090 | else |
1091 | emit(rv_addiw(rd, rs, imm), ctx); |
1092 | } |
1093 | |
1094 | static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) |
1095 | { |
1096 | if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) |
1097 | emitc(rvc_ldsp(rd, off), ctx); |
1098 | else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) |
1099 | emitc(rvc_ld(rd, off, rs1), ctx); |
1100 | else |
1101 | emit(rv_ld(rd, off, rs1), ctx); |
1102 | } |
1103 | |
1104 | static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) |
1105 | { |
1106 | if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) |
1107 | emitc(rvc_sdsp(off, rs2), ctx); |
1108 | else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) |
1109 | emitc(rvc_sd(rs1, off, rs2), ctx); |
1110 | else |
1111 | emit(rv_sd(rs1, off, rs2), ctx); |
1112 | } |
1113 | |
1114 | static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) |
1115 | { |
1116 | if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) |
1117 | emitc(rvc_subw(rd, rs2), ctx); |
1118 | else |
1119 | emit(rv_subw(rd, rs1, rs2), ctx); |
1120 | } |
1121 | |
1122 | static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx) |
1123 | { |
1124 | if (rvzbb_enabled()) { |
1125 | emit(rvzbb_sextb(rd, rs), ctx); |
1126 | return; |
1127 | } |
1128 | |
1129 | emit_slli(rd, rs, 56, ctx); |
1130 | emit_srai(rd, rd, 56, ctx); |
1131 | } |
1132 | |
1133 | static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx) |
1134 | { |
1135 | if (rvzbb_enabled()) { |
1136 | emit(rvzbb_sexth(rd, rs), ctx); |
1137 | return; |
1138 | } |
1139 | |
1140 | emit_slli(rd, rs, 48, ctx); |
1141 | emit_srai(rd, rd, 48, ctx); |
1142 | } |
1143 | |
1144 | static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx) |
1145 | { |
1146 | emit_addiw(rd, rs, 0, ctx); |
1147 | } |
1148 | |
1149 | static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) |
1150 | { |
1151 | if (rvzbb_enabled()) { |
1152 | emit(rvzbb_zexth(rd, rs), ctx); |
1153 | return; |
1154 | } |
1155 | |
1156 | emit_slli(rd, rs, 48, ctx); |
1157 | emit_srli(rd, rd, 48, ctx); |
1158 | } |
1159 | |
1160 | static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) |
1161 | { |
1162 | emit_slli(rd, rs, 32, ctx); |
1163 | emit_srli(rd, rd, 32, ctx); |
1164 | } |
1165 | |
1166 | static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx) |
1167 | { |
1168 | if (rvzbb_enabled()) { |
1169 | int bits = 64 - imm; |
1170 | |
1171 | emit(rvzbb_rev8(rd, rd), ctx); |
1172 | if (bits) |
1173 | emit_srli(rd, rd, bits, ctx); |
1174 | return; |
1175 | } |
1176 | |
1177 | emit_li(RV_REG_T2, 0, ctx); |
1178 | |
1179 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1180 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1181 | emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); |
1182 | emit_srli(rd, rd, 8, ctx); |
1183 | if (imm == 16) |
1184 | goto out_be; |
1185 | |
1186 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1187 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1188 | emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); |
1189 | emit_srli(rd, rd, 8, ctx); |
1190 | |
1191 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1192 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1193 | emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); |
1194 | emit_srli(rd, rd, 8, ctx); |
1195 | if (imm == 32) |
1196 | goto out_be; |
1197 | |
1198 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1199 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1200 | emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); |
1201 | emit_srli(rd, rd, 8, ctx); |
1202 | |
1203 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1204 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1205 | emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); |
1206 | emit_srli(rd, rd, 8, ctx); |
1207 | |
1208 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1209 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1210 | emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); |
1211 | emit_srli(rd, rd, 8, ctx); |
1212 | |
1213 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1214 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1215 | emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); |
1216 | emit_srli(rd, rd, 8, ctx); |
1217 | out_be: |
1218 | emit_andi(RV_REG_T1, rd, 0xff, ctx); |
1219 | emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); |
1220 | |
1221 | emit_mv(rd, RV_REG_T2, ctx); |
1222 | } |
1223 | |
1224 | #endif /* __riscv_xlen == 64 */ |
1225 | |
1226 | void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog); |
1227 | void bpf_jit_build_epilogue(struct rv_jit_context *ctx); |
1228 | |
1229 | int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, |
1230 | bool ); |
1231 | |
1232 | #endif /* _BPF_JIT_H */ |
1233 | |