1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Common functionality for RV32 and RV64 BPF JIT compilers
4 *
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6 *
7 */
8
9#ifndef _BPF_JIT_H
10#define _BPF_JIT_H
11
12#include <linux/bpf.h>
13#include <linux/filter.h>
14#include <asm/cacheflush.h>
15
16static inline bool rvc_enabled(void)
17{
18 return IS_ENABLED(CONFIG_RISCV_ISA_C);
19}
20
21static inline bool rvzbb_enabled(void)
22{
23 return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB);
24}
25
26enum {
27 RV_REG_ZERO = 0, /* The constant value 0 */
28 RV_REG_RA = 1, /* Return address */
29 RV_REG_SP = 2, /* Stack pointer */
30 RV_REG_GP = 3, /* Global pointer */
31 RV_REG_TP = 4, /* Thread pointer */
32 RV_REG_T0 = 5, /* Temporaries */
33 RV_REG_T1 = 6,
34 RV_REG_T2 = 7,
35 RV_REG_FP = 8, /* Saved register/frame pointer */
36 RV_REG_S1 = 9, /* Saved register */
37 RV_REG_A0 = 10, /* Function argument/return values */
38 RV_REG_A1 = 11, /* Function arguments */
39 RV_REG_A2 = 12,
40 RV_REG_A3 = 13,
41 RV_REG_A4 = 14,
42 RV_REG_A5 = 15,
43 RV_REG_A6 = 16,
44 RV_REG_A7 = 17,
45 RV_REG_S2 = 18, /* Saved registers */
46 RV_REG_S3 = 19,
47 RV_REG_S4 = 20,
48 RV_REG_S5 = 21,
49 RV_REG_S6 = 22,
50 RV_REG_S7 = 23,
51 RV_REG_S8 = 24,
52 RV_REG_S9 = 25,
53 RV_REG_S10 = 26,
54 RV_REG_S11 = 27,
55 RV_REG_T3 = 28, /* Temporaries */
56 RV_REG_T4 = 29,
57 RV_REG_T5 = 30,
58 RV_REG_T6 = 31,
59};
60
61static inline bool is_creg(u8 reg)
62{
63 return (1 << reg) & (BIT(RV_REG_FP) |
64 BIT(RV_REG_S1) |
65 BIT(RV_REG_A0) |
66 BIT(RV_REG_A1) |
67 BIT(RV_REG_A2) |
68 BIT(RV_REG_A3) |
69 BIT(RV_REG_A4) |
70 BIT(RV_REG_A5));
71}
72
73struct rv_jit_context {
74 struct bpf_prog *prog;
75 u16 *insns; /* RV insns */
76 u16 *ro_insns;
77 int ninsns;
78 int prologue_len;
79 int epilogue_offset;
80 int *offset; /* BPF to RV */
81 int nexentries;
82 unsigned long flags;
83 int stack_size;
84};
85
86/* Convert from ninsns to bytes. */
87static inline int ninsns_rvoff(int ninsns)
88{
89 return ninsns << 1;
90}
91
92struct rv_jit_data {
93 struct bpf_binary_header *header;
94 struct bpf_binary_header *ro_header;
95 u8 *image;
96 u8 *ro_image;
97 struct rv_jit_context ctx;
98};
99
100static inline void bpf_fill_ill_insns(void *area, unsigned int size)
101{
102 memset(area, 0, size);
103}
104
105static inline void bpf_flush_icache(void *start, void *end)
106{
107 flush_icache_range(start: (unsigned long)start, end: (unsigned long)end);
108}
109
110/* Emit a 4-byte riscv instruction. */
111static inline void emit(const u32 insn, struct rv_jit_context *ctx)
112{
113 if (ctx->insns) {
114 ctx->insns[ctx->ninsns] = insn;
115 ctx->insns[ctx->ninsns + 1] = (insn >> 16);
116 }
117
118 ctx->ninsns += 2;
119}
120
121/* Emit a 2-byte riscv compressed instruction. */
122static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
123{
124 BUILD_BUG_ON(!rvc_enabled());
125
126 if (ctx->insns)
127 ctx->insns[ctx->ninsns] = insn;
128
129 ctx->ninsns++;
130}
131
132static inline int epilogue_offset(struct rv_jit_context *ctx)
133{
134 int to = ctx->epilogue_offset, from = ctx->ninsns;
135
136 return ninsns_rvoff(ninsns: to - from);
137}
138
139/* Return -1 or inverted cond. */
140static inline int invert_bpf_cond(u8 cond)
141{
142 switch (cond) {
143 case BPF_JEQ:
144 return BPF_JNE;
145 case BPF_JGT:
146 return BPF_JLE;
147 case BPF_JLT:
148 return BPF_JGE;
149 case BPF_JGE:
150 return BPF_JLT;
151 case BPF_JLE:
152 return BPF_JGT;
153 case BPF_JNE:
154 return BPF_JEQ;
155 case BPF_JSGT:
156 return BPF_JSLE;
157 case BPF_JSLT:
158 return BPF_JSGE;
159 case BPF_JSGE:
160 return BPF_JSLT;
161 case BPF_JSLE:
162 return BPF_JSGT;
163 }
164 return -1;
165}
166
167static inline bool is_6b_int(long val)
168{
169 return -(1L << 5) <= val && val < (1L << 5);
170}
171
172static inline bool is_7b_uint(unsigned long val)
173{
174 return val < (1UL << 7);
175}
176
177static inline bool is_8b_uint(unsigned long val)
178{
179 return val < (1UL << 8);
180}
181
182static inline bool is_9b_uint(unsigned long val)
183{
184 return val < (1UL << 9);
185}
186
187static inline bool is_10b_int(long val)
188{
189 return -(1L << 9) <= val && val < (1L << 9);
190}
191
192static inline bool is_10b_uint(unsigned long val)
193{
194 return val < (1UL << 10);
195}
196
197static inline bool is_12b_int(long val)
198{
199 return -(1L << 11) <= val && val < (1L << 11);
200}
201
202static inline int is_12b_check(int off, int insn)
203{
204 if (!is_12b_int(val: off)) {
205 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
206 insn, (int)off);
207 return -1;
208 }
209 return 0;
210}
211
212static inline bool is_13b_int(long val)
213{
214 return -(1L << 12) <= val && val < (1L << 12);
215}
216
217static inline bool is_21b_int(long val)
218{
219 return -(1L << 20) <= val && val < (1L << 20);
220}
221
222static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
223{
224 int from, to;
225
226 off++; /* BPF branch is from PC+1, RV is from PC */
227 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len;
228 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len;
229 return ninsns_rvoff(ninsns: to - from);
230}
231
232/* Instruction formats. */
233
234static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
235 u8 opcode)
236{
237 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
238 (rd << 7) | opcode;
239}
240
241static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
242{
243 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
244 opcode;
245}
246
247static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
248{
249 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
250
251 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
252 (imm4_0 << 7) | opcode;
253}
254
255static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
256{
257 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
258 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
259
260 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
261 (imm4_1 << 7) | opcode;
262}
263
264static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
265{
266 return (imm31_12 << 12) | (rd << 7) | opcode;
267}
268
269static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
270{
271 u32 imm;
272
273 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
274 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
275
276 return (imm << 12) | (rd << 7) | opcode;
277}
278
279static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
280 u8 funct3, u8 rd, u8 opcode)
281{
282 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
283
284 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
285}
286
287/* RISC-V compressed instruction formats. */
288
289static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
290{
291 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
292}
293
294static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
295{
296 u32 imm;
297
298 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
299 return (funct3 << 13) | (rd << 7) | op | imm;
300}
301
302static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
303{
304 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
305}
306
307static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
308{
309 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
310}
311
312static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
313 u8 op)
314{
315 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
316 (imm_lo << 5) | ((rd & 0x7) << 2) | op;
317}
318
319static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
320 u8 op)
321{
322 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
323 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
324}
325
326static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
327{
328 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
329 ((rs2 & 0x7) << 2) | op;
330}
331
332static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
333{
334 u32 imm;
335
336 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
337 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
338}
339
340/* Instructions shared by both RV32 and RV64. */
341
342static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
343{
344 return rv_i_insn(imm11_0, rs1, funct3: 0, rd, opcode: 0x13);
345}
346
347static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
348{
349 return rv_i_insn(imm11_0, rs1, funct3: 7, rd, opcode: 0x13);
350}
351
352static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
353{
354 return rv_i_insn(imm11_0, rs1, funct3: 6, rd, opcode: 0x13);
355}
356
357static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
358{
359 return rv_i_insn(imm11_0, rs1, funct3: 4, rd, opcode: 0x13);
360}
361
362static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
363{
364 return rv_i_insn(imm11_0, rs1, funct3: 1, rd, opcode: 0x13);
365}
366
367static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
368{
369 return rv_i_insn(imm11_0, rs1, funct3: 5, rd, opcode: 0x13);
370}
371
372static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
373{
374 return rv_i_insn(imm11_0: 0x400 | imm11_0, rs1, funct3: 5, rd, opcode: 0x13);
375}
376
377static inline u32 rv_lui(u8 rd, u32 imm31_12)
378{
379 return rv_u_insn(imm31_12, rd, opcode: 0x37);
380}
381
382static inline u32 rv_auipc(u8 rd, u32 imm31_12)
383{
384 return rv_u_insn(imm31_12, rd, opcode: 0x17);
385}
386
387static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
388{
389 return rv_r_insn(funct7: 0, rs2, rs1, funct3: 0, rd, opcode: 0x33);
390}
391
392static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
393{
394 return rv_r_insn(funct7: 0x20, rs2, rs1, funct3: 0, rd, opcode: 0x33);
395}
396
397static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
398{
399 return rv_r_insn(funct7: 0, rs2, rs1, funct3: 3, rd, opcode: 0x33);
400}
401
402static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
403{
404 return rv_r_insn(funct7: 0, rs2, rs1, funct3: 7, rd, opcode: 0x33);
405}
406
407static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
408{
409 return rv_r_insn(funct7: 0, rs2, rs1, funct3: 6, rd, opcode: 0x33);
410}
411
412static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
413{
414 return rv_r_insn(funct7: 0, rs2, rs1, funct3: 4, rd, opcode: 0x33);
415}
416
417static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
418{
419 return rv_r_insn(funct7: 0, rs2, rs1, funct3: 1, rd, opcode: 0x33);
420}
421
422static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
423{
424 return rv_r_insn(funct7: 0, rs2, rs1, funct3: 5, rd, opcode: 0x33);
425}
426
427static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
428{
429 return rv_r_insn(funct7: 0x20, rs2, rs1, funct3: 5, rd, opcode: 0x33);
430}
431
432static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
433{
434 return rv_r_insn(funct7: 1, rs2, rs1, funct3: 0, rd, opcode: 0x33);
435}
436
437static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
438{
439 return rv_r_insn(funct7: 1, rs2, rs1, funct3: 3, rd, opcode: 0x33);
440}
441
442static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)
443{
444 return rv_r_insn(funct7: 1, rs2, rs1, funct3: 4, rd, opcode: 0x33);
445}
446
447static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
448{
449 return rv_r_insn(funct7: 1, rs2, rs1, funct3: 5, rd, opcode: 0x33);
450}
451
452static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)
453{
454 return rv_r_insn(funct7: 1, rs2, rs1, funct3: 6, rd, opcode: 0x33);
455}
456
457static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
458{
459 return rv_r_insn(funct7: 1, rs2, rs1, funct3: 7, rd, opcode: 0x33);
460}
461
462static inline u32 rv_jal(u8 rd, u32 imm20_1)
463{
464 return rv_j_insn(imm20_1, rd, opcode: 0x6f);
465}
466
467static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
468{
469 return rv_i_insn(imm11_0, rs1, funct3: 0, rd, opcode: 0x67);
470}
471
472static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
473{
474 return rv_b_insn(imm12_1, rs2, rs1, funct3: 0, opcode: 0x63);
475}
476
477static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
478{
479 return rv_b_insn(imm12_1, rs2, rs1, funct3: 1, opcode: 0x63);
480}
481
482static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
483{
484 return rv_b_insn(imm12_1, rs2, rs1, funct3: 6, opcode: 0x63);
485}
486
487static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
488{
489 return rv_bltu(rs1: rs2, rs2: rs1, imm12_1);
490}
491
492static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
493{
494 return rv_b_insn(imm12_1, rs2, rs1, funct3: 7, opcode: 0x63);
495}
496
497static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
498{
499 return rv_bgeu(rs1: rs2, rs2: rs1, imm12_1);
500}
501
502static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
503{
504 return rv_b_insn(imm12_1, rs2, rs1, funct3: 4, opcode: 0x63);
505}
506
507static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
508{
509 return rv_blt(rs1: rs2, rs2: rs1, imm12_1);
510}
511
512static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
513{
514 return rv_b_insn(imm12_1, rs2, rs1, funct3: 5, opcode: 0x63);
515}
516
517static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
518{
519 return rv_bge(rs1: rs2, rs2: rs1, imm12_1);
520}
521
522static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)
523{
524 return rv_i_insn(imm11_0, rs1, funct3: 0, rd, opcode: 0x03);
525}
526
527static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)
528{
529 return rv_i_insn(imm11_0, rs1, funct3: 1, rd, opcode: 0x03);
530}
531
532static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
533{
534 return rv_i_insn(imm11_0, rs1, funct3: 2, rd, opcode: 0x03);
535}
536
537static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
538{
539 return rv_i_insn(imm11_0, rs1, funct3: 4, rd, opcode: 0x03);
540}
541
542static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
543{
544 return rv_i_insn(imm11_0, rs1, funct3: 5, rd, opcode: 0x03);
545}
546
547static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
548{
549 return rv_s_insn(imm11_0, rs2, rs1, funct3: 0, opcode: 0x23);
550}
551
552static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
553{
554 return rv_s_insn(imm11_0, rs2, rs1, funct3: 1, opcode: 0x23);
555}
556
557static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
558{
559 return rv_s_insn(imm11_0, rs2, rs1, funct3: 2, opcode: 0x23);
560}
561
562static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
563{
564 return rv_amo_insn(funct5: 0, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f);
565}
566
567static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
568{
569 return rv_amo_insn(funct5: 0xc, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f);
570}
571
572static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
573{
574 return rv_amo_insn(funct5: 0x8, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f);
575}
576
577static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
578{
579 return rv_amo_insn(funct5: 0x4, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f);
580}
581
582static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
583{
584 return rv_amo_insn(funct5: 0x1, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f);
585}
586
587static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
588{
589 return rv_amo_insn(funct5: 0x2, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f);
590}
591
592static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
593{
594 return rv_amo_insn(funct5: 0x3, aq, rl, rs2, rs1, funct3: 2, rd, opcode: 0x2f);
595}
596
597static inline u32 rv_fence(u8 pred, u8 succ)
598{
599 u16 imm11_0 = pred << 4 | succ;
600
601 return rv_i_insn(imm11_0, rs1: 0, funct3: 0, rd: 0, opcode: 0xf);
602}
603
604static inline u32 rv_nop(void)
605{
606 return rv_i_insn(imm11_0: 0, rs1: 0, funct3: 0, rd: 0, opcode: 0x13);
607}
608
609/* RVC instrutions. */
610
611static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
612{
613 u32 imm;
614
615 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
616 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
617 return rv_ciw_insn(funct3: 0x0, uimm: imm, rd, op: 0x0);
618}
619
620static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
621{
622 u32 imm_hi, imm_lo;
623
624 imm_hi = (imm7 & 0x38) >> 3;
625 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
626 return rv_cl_insn(funct3: 0x2, imm_hi, rs1, imm_lo, rd, op: 0x0);
627}
628
629static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
630{
631 u32 imm_hi, imm_lo;
632
633 imm_hi = (imm7 & 0x38) >> 3;
634 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
635 return rv_cs_insn(funct3: 0x6, imm_hi, rs1, imm_lo, rs2, op: 0x0);
636}
637
638static inline u16 rvc_addi(u8 rd, u32 imm6)
639{
640 return rv_ci_insn(funct3: 0, imm6, rd, op: 0x1);
641}
642
643static inline u16 rvc_li(u8 rd, u32 imm6)
644{
645 return rv_ci_insn(funct3: 0x2, imm6, rd, op: 0x1);
646}
647
648static inline u16 rvc_addi16sp(u32 imm10)
649{
650 u32 imm;
651
652 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
653 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
654 return rv_ci_insn(funct3: 0x3, imm6: imm, rd: RV_REG_SP, op: 0x1);
655}
656
657static inline u16 rvc_lui(u8 rd, u32 imm6)
658{
659 return rv_ci_insn(funct3: 0x3, imm6, rd, op: 0x1);
660}
661
662static inline u16 rvc_srli(u8 rd, u32 imm6)
663{
664 return rv_cb_insn(funct3: 0x4, imm6, funct2: 0, rd, op: 0x1);
665}
666
667static inline u16 rvc_srai(u8 rd, u32 imm6)
668{
669 return rv_cb_insn(funct3: 0x4, imm6, funct2: 0x1, rd, op: 0x1);
670}
671
672static inline u16 rvc_andi(u8 rd, u32 imm6)
673{
674 return rv_cb_insn(funct3: 0x4, imm6, funct2: 0x2, rd, op: 0x1);
675}
676
677static inline u16 rvc_sub(u8 rd, u8 rs)
678{
679 return rv_ca_insn(funct6: 0x23, rd, funct2: 0, rs2: rs, op: 0x1);
680}
681
682static inline u16 rvc_xor(u8 rd, u8 rs)
683{
684 return rv_ca_insn(funct6: 0x23, rd, funct2: 0x1, rs2: rs, op: 0x1);
685}
686
687static inline u16 rvc_or(u8 rd, u8 rs)
688{
689 return rv_ca_insn(funct6: 0x23, rd, funct2: 0x2, rs2: rs, op: 0x1);
690}
691
692static inline u16 rvc_and(u8 rd, u8 rs)
693{
694 return rv_ca_insn(funct6: 0x23, rd, funct2: 0x3, rs2: rs, op: 0x1);
695}
696
697static inline u16 rvc_slli(u8 rd, u32 imm6)
698{
699 return rv_ci_insn(funct3: 0, imm6, rd, op: 0x2);
700}
701
702static inline u16 rvc_lwsp(u8 rd, u32 imm8)
703{
704 u32 imm;
705
706 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
707 return rv_ci_insn(funct3: 0x2, imm6: imm, rd, op: 0x2);
708}
709
710static inline u16 rvc_jr(u8 rs1)
711{
712 return rv_cr_insn(funct4: 0x8, rd: rs1, rs2: RV_REG_ZERO, op: 0x2);
713}
714
715static inline u16 rvc_mv(u8 rd, u8 rs)
716{
717 return rv_cr_insn(funct4: 0x8, rd, rs2: rs, op: 0x2);
718}
719
720static inline u16 rvc_jalr(u8 rs1)
721{
722 return rv_cr_insn(funct4: 0x9, rd: rs1, rs2: RV_REG_ZERO, op: 0x2);
723}
724
725static inline u16 rvc_add(u8 rd, u8 rs)
726{
727 return rv_cr_insn(funct4: 0x9, rd, rs2: rs, op: 0x2);
728}
729
730static inline u16 rvc_swsp(u32 imm8, u8 rs2)
731{
732 u32 imm;
733
734 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
735 return rv_css_insn(funct3: 0x6, uimm: imm, rs2, op: 0x2);
736}
737
738/* RVZBB instrutions. */
739static inline u32 rvzbb_sextb(u8 rd, u8 rs1)
740{
741 return rv_i_insn(imm11_0: 0x604, rs1, funct3: 1, rd, opcode: 0x13);
742}
743
744static inline u32 rvzbb_sexth(u8 rd, u8 rs1)
745{
746 return rv_i_insn(imm11_0: 0x605, rs1, funct3: 1, rd, opcode: 0x13);
747}
748
749static inline u32 rvzbb_zexth(u8 rd, u8 rs)
750{
751 if (IS_ENABLED(CONFIG_64BIT))
752 return rv_i_insn(imm11_0: 0x80, rs1: rs, funct3: 4, rd, opcode: 0x3b);
753
754 return rv_i_insn(imm11_0: 0x80, rs1: rs, funct3: 4, rd, opcode: 0x33);
755}
756
757static inline u32 rvzbb_rev8(u8 rd, u8 rs)
758{
759 if (IS_ENABLED(CONFIG_64BIT))
760 return rv_i_insn(imm11_0: 0x6b8, rs1: rs, funct3: 5, rd, opcode: 0x13);
761
762 return rv_i_insn(imm11_0: 0x698, rs1: rs, funct3: 5, rd, opcode: 0x13);
763}
764
765/*
766 * RV64-only instructions.
767 *
768 * These instructions are not available on RV32. Wrap them below a #if to
769 * ensure that the RV32 JIT doesn't emit any of these instructions.
770 */
771
772#if __riscv_xlen == 64
773
774static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
775{
776 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
777}
778
779static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
780{
781 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
782}
783
784static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
785{
786 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
787}
788
789static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
790{
791 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
792}
793
794static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
795{
796 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
797}
798
799static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
800{
801 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
802}
803
804static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
805{
806 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
807}
808
809static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
810{
811 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
812}
813
814static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
815{
816 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
817}
818
819static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
820{
821 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
822}
823
824static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)
825{
826 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);
827}
828
829static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
830{
831 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
832}
833
834static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)
835{
836 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);
837}
838
839static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
840{
841 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
842}
843
844static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
845{
846 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
847}
848
849static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
850{
851 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
852}
853
854static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
855{
856 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
857}
858
859static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
860{
861 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
862}
863
864static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
865{
866 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);
867}
868
869static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
870{
871 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);
872}
873
874static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
875{
876 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);
877}
878
879static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
880{
881 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);
882}
883
884static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
885{
886 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);
887}
888
889static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
890{
891 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);
892}
893
894/* RV64-only RVC instructions. */
895
896static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
897{
898 u32 imm_hi, imm_lo;
899
900 imm_hi = (imm8 & 0x38) >> 3;
901 imm_lo = (imm8 & 0xc0) >> 6;
902 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
903}
904
905static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
906{
907 u32 imm_hi, imm_lo;
908
909 imm_hi = (imm8 & 0x38) >> 3;
910 imm_lo = (imm8 & 0xc0) >> 6;
911 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
912}
913
914static inline u16 rvc_subw(u8 rd, u8 rs)
915{
916 return rv_ca_insn(0x27, rd, 0, rs, 0x1);
917}
918
919static inline u16 rvc_addiw(u8 rd, u32 imm6)
920{
921 return rv_ci_insn(0x1, imm6, rd, 0x1);
922}
923
924static inline u16 rvc_ldsp(u8 rd, u32 imm9)
925{
926 u32 imm;
927
928 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
929 return rv_ci_insn(0x3, imm, rd, 0x2);
930}
931
932static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
933{
934 u32 imm;
935
936 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
937 return rv_css_insn(0x7, imm, rs2, 0x2);
938}
939
940#endif /* __riscv_xlen == 64 */
941
942/* Helper functions that emit RVC instructions when possible. */
943
944static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
945{
946 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
947 emitc(insn: rvc_jalr(rs1: rs), ctx);
948 else if (rvc_enabled() && !rd && rs && !imm)
949 emitc(insn: rvc_jr(rs1: rs), ctx);
950 else
951 emit(insn: rv_jalr(rd, rs1: rs, imm11_0: imm), ctx);
952}
953
954static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
955{
956 if (rvc_enabled() && rd && rs)
957 emitc(insn: rvc_mv(rd, rs), ctx);
958 else
959 emit(insn: rv_addi(rd, rs1: rs, imm11_0: 0), ctx);
960}
961
962static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
963{
964 if (rvc_enabled() && rd && rd == rs1 && rs2)
965 emitc(insn: rvc_add(rd, rs: rs2), ctx);
966 else
967 emit(insn: rv_add(rd, rs1, rs2), ctx);
968}
969
970static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
971{
972 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(val: imm) && imm && !(imm & 0xf))
973 emitc(insn: rvc_addi16sp(imm10: imm), ctx);
974 else if (rvc_enabled() && is_creg(reg: rd) && rs == RV_REG_SP && is_10b_uint(val: imm) &&
975 !(imm & 0x3) && imm)
976 emitc(insn: rvc_addi4spn(rd, imm10: imm), ctx);
977 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(val: imm))
978 emitc(insn: rvc_addi(rd, imm6: imm), ctx);
979 else
980 emit(insn: rv_addi(rd, rs1: rs, imm11_0: imm), ctx);
981}
982
983static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
984{
985 if (rvc_enabled() && rd && is_6b_int(val: imm))
986 emitc(insn: rvc_li(rd, imm6: imm), ctx);
987 else
988 emit(insn: rv_addi(rd, rs1: RV_REG_ZERO, imm11_0: imm), ctx);
989}
990
991static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
992{
993 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(val: imm) && imm)
994 emitc(insn: rvc_lui(rd, imm6: imm), ctx);
995 else
996 emit(insn: rv_lui(rd, imm31_12: imm), ctx);
997}
998
999static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1000{
1001 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
1002 emitc(insn: rvc_slli(rd, imm6: imm), ctx);
1003 else
1004 emit(insn: rv_slli(rd, rs1: rs, imm11_0: imm), ctx);
1005}
1006
1007static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1008{
1009 if (rvc_enabled() && is_creg(reg: rd) && rd == rs && is_6b_int(val: imm))
1010 emitc(insn: rvc_andi(rd, imm6: imm), ctx);
1011 else
1012 emit(insn: rv_andi(rd, rs1: rs, imm11_0: imm), ctx);
1013}
1014
1015static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1016{
1017 if (rvc_enabled() && is_creg(reg: rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
1018 emitc(insn: rvc_srli(rd, imm6: imm), ctx);
1019 else
1020 emit(insn: rv_srli(rd, rs1: rs, imm11_0: imm), ctx);
1021}
1022
1023static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1024{
1025 if (rvc_enabled() && is_creg(reg: rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
1026 emitc(insn: rvc_srai(rd, imm6: imm), ctx);
1027 else
1028 emit(insn: rv_srai(rd, rs1: rs, imm11_0: imm), ctx);
1029}
1030
1031static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1032{
1033 if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2))
1034 emitc(insn: rvc_sub(rd, rs: rs2), ctx);
1035 else
1036 emit(insn: rv_sub(rd, rs1, rs2), ctx);
1037}
1038
1039static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1040{
1041 if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2))
1042 emitc(insn: rvc_or(rd, rs: rs2), ctx);
1043 else
1044 emit(insn: rv_or(rd, rs1, rs2), ctx);
1045}
1046
1047static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1048{
1049 if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2))
1050 emitc(insn: rvc_and(rd, rs: rs2), ctx);
1051 else
1052 emit(insn: rv_and(rd, rs1, rs2), ctx);
1053}
1054
1055static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1056{
1057 if (rvc_enabled() && is_creg(reg: rd) && rd == rs1 && is_creg(reg: rs2))
1058 emitc(insn: rvc_xor(rd, rs: rs2), ctx);
1059 else
1060 emit(insn: rv_xor(rd, rs1, rs2), ctx);
1061}
1062
1063static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1064{
1065 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(val: off) && !(off & 0x3))
1066 emitc(insn: rvc_lwsp(rd, imm8: off), ctx);
1067 else if (rvc_enabled() && is_creg(reg: rd) && is_creg(reg: rs1) && is_7b_uint(val: off) && !(off & 0x3))
1068 emitc(insn: rvc_lw(rd, imm7: off, rs1), ctx);
1069 else
1070 emit(insn: rv_lw(rd, imm11_0: off, rs1), ctx);
1071}
1072
1073static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1074{
1075 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(val: off) && !(off & 0x3))
1076 emitc(insn: rvc_swsp(imm8: off, rs2), ctx);
1077 else if (rvc_enabled() && is_creg(reg: rs1) && is_creg(reg: rs2) && is_7b_uint(val: off) && !(off & 0x3))
1078 emitc(insn: rvc_sw(rs1, imm7: off, rs2), ctx);
1079 else
1080 emit(insn: rv_sw(rs1, imm11_0: off, rs2), ctx);
1081}
1082
1083/* RV64-only helper functions. */
1084#if __riscv_xlen == 64
1085
1086static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1087{
1088 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
1089 emitc(rvc_addiw(rd, imm), ctx);
1090 else
1091 emit(rv_addiw(rd, rs, imm), ctx);
1092}
1093
1094static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1095{
1096 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
1097 emitc(rvc_ldsp(rd, off), ctx);
1098 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
1099 emitc(rvc_ld(rd, off, rs1), ctx);
1100 else
1101 emit(rv_ld(rd, off, rs1), ctx);
1102}
1103
1104static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1105{
1106 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
1107 emitc(rvc_sdsp(off, rs2), ctx);
1108 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
1109 emitc(rvc_sd(rs1, off, rs2), ctx);
1110 else
1111 emit(rv_sd(rs1, off, rs2), ctx);
1112}
1113
1114static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1115{
1116 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1117 emitc(rvc_subw(rd, rs2), ctx);
1118 else
1119 emit(rv_subw(rd, rs1, rs2), ctx);
1120}
1121
1122static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx)
1123{
1124 if (rvzbb_enabled()) {
1125 emit(rvzbb_sextb(rd, rs), ctx);
1126 return;
1127 }
1128
1129 emit_slli(rd, rs, 56, ctx);
1130 emit_srai(rd, rd, 56, ctx);
1131}
1132
1133static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
1134{
1135 if (rvzbb_enabled()) {
1136 emit(rvzbb_sexth(rd, rs), ctx);
1137 return;
1138 }
1139
1140 emit_slli(rd, rs, 48, ctx);
1141 emit_srai(rd, rd, 48, ctx);
1142}
1143
1144static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
1145{
1146 emit_addiw(rd, rs, 0, ctx);
1147}
1148
1149static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
1150{
1151 if (rvzbb_enabled()) {
1152 emit(rvzbb_zexth(rd, rs), ctx);
1153 return;
1154 }
1155
1156 emit_slli(rd, rs, 48, ctx);
1157 emit_srli(rd, rd, 48, ctx);
1158}
1159
1160static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
1161{
1162 emit_slli(rd, rs, 32, ctx);
1163 emit_srli(rd, rd, 32, ctx);
1164}
1165
1166static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx)
1167{
1168 if (rvzbb_enabled()) {
1169 int bits = 64 - imm;
1170
1171 emit(rvzbb_rev8(rd, rd), ctx);
1172 if (bits)
1173 emit_srli(rd, rd, bits, ctx);
1174 return;
1175 }
1176
1177 emit_li(RV_REG_T2, 0, ctx);
1178
1179 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1180 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1181 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1182 emit_srli(rd, rd, 8, ctx);
1183 if (imm == 16)
1184 goto out_be;
1185
1186 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1187 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1188 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1189 emit_srli(rd, rd, 8, ctx);
1190
1191 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1192 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1193 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1194 emit_srli(rd, rd, 8, ctx);
1195 if (imm == 32)
1196 goto out_be;
1197
1198 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1199 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1200 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1201 emit_srli(rd, rd, 8, ctx);
1202
1203 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1204 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1205 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1206 emit_srli(rd, rd, 8, ctx);
1207
1208 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1209 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1210 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1211 emit_srli(rd, rd, 8, ctx);
1212
1213 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1214 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1215 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1216 emit_srli(rd, rd, 8, ctx);
1217out_be:
1218 emit_andi(RV_REG_T1, rd, 0xff, ctx);
1219 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1220
1221 emit_mv(rd, RV_REG_T2, ctx);
1222}
1223
1224#endif /* __riscv_xlen == 64 */
1225
1226void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog);
1227void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
1228
1229int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1230 bool extra_pass);
1231
1232#endif /* _BPF_JIT_H */
1233

source code of linux/arch/riscv/net/bpf_jit.h