1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * BPF JIT compiler for LoongArch |
4 | * |
5 | * Copyright (C) 2022 Loongson Technology Corporation Limited |
6 | */ |
7 | #include "bpf_jit.h" |
8 | |
9 | #define REG_TCC LOONGARCH_GPR_A6 |
10 | #define TCC_SAVED LOONGARCH_GPR_S5 |
11 | |
12 | #define SAVE_RA BIT(0) |
13 | #define SAVE_TCC BIT(1) |
14 | |
15 | static const int regmap[] = { |
16 | /* return value from in-kernel function, and exit value for eBPF program */ |
17 | [BPF_REG_0] = LOONGARCH_GPR_A5, |
18 | /* arguments from eBPF program to in-kernel function */ |
19 | [BPF_REG_1] = LOONGARCH_GPR_A0, |
20 | [BPF_REG_2] = LOONGARCH_GPR_A1, |
21 | [BPF_REG_3] = LOONGARCH_GPR_A2, |
22 | [BPF_REG_4] = LOONGARCH_GPR_A3, |
23 | [BPF_REG_5] = LOONGARCH_GPR_A4, |
24 | /* callee saved registers that in-kernel function will preserve */ |
25 | [BPF_REG_6] = LOONGARCH_GPR_S0, |
26 | [BPF_REG_7] = LOONGARCH_GPR_S1, |
27 | [BPF_REG_8] = LOONGARCH_GPR_S2, |
28 | [BPF_REG_9] = LOONGARCH_GPR_S3, |
29 | /* read-only frame pointer to access stack */ |
30 | [BPF_REG_FP] = LOONGARCH_GPR_S4, |
31 | /* temporary register for blinding constants */ |
32 | [BPF_REG_AX] = LOONGARCH_GPR_T0, |
33 | }; |
34 | |
35 | static void mark_call(struct jit_ctx *ctx) |
36 | { |
37 | ctx->flags |= SAVE_RA; |
38 | } |
39 | |
40 | static void mark_tail_call(struct jit_ctx *ctx) |
41 | { |
42 | ctx->flags |= SAVE_TCC; |
43 | } |
44 | |
45 | static bool seen_call(struct jit_ctx *ctx) |
46 | { |
47 | return (ctx->flags & SAVE_RA); |
48 | } |
49 | |
50 | static bool seen_tail_call(struct jit_ctx *ctx) |
51 | { |
52 | return (ctx->flags & SAVE_TCC); |
53 | } |
54 | |
55 | static u8 tail_call_reg(struct jit_ctx *ctx) |
56 | { |
57 | if (seen_call(ctx)) |
58 | return TCC_SAVED; |
59 | |
60 | return REG_TCC; |
61 | } |
62 | |
63 | /* |
64 | * eBPF prog stack layout: |
65 | * |
66 | * high |
67 | * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP |
68 | * | $ra | |
69 | * +-------------------------+ |
70 | * | $fp | |
71 | * +-------------------------+ |
72 | * | $s0 | |
73 | * +-------------------------+ |
74 | * | $s1 | |
75 | * +-------------------------+ |
76 | * | $s2 | |
77 | * +-------------------------+ |
78 | * | $s3 | |
79 | * +-------------------------+ |
80 | * | $s4 | |
81 | * +-------------------------+ |
82 | * | $s5 | |
83 | * +-------------------------+ <--BPF_REG_FP |
84 | * | prog->aux->stack_depth | |
85 | * | (optional) | |
86 | * current $sp -------------> +-------------------------+ |
87 | * low |
88 | */ |
89 | static void build_prologue(struct jit_ctx *ctx) |
90 | { |
91 | int stack_adjust = 0, store_offset, bpf_stack_adjust; |
92 | |
93 | bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); |
94 | |
95 | /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */ |
96 | stack_adjust += sizeof(long) * 8; |
97 | |
98 | stack_adjust = round_up(stack_adjust, 16); |
99 | stack_adjust += bpf_stack_adjust; |
100 | |
101 | /* |
102 | * First instruction initializes the tail call count (TCC). |
103 | * On tail call we skip this instruction, and the TCC is |
104 | * passed in REG_TCC from the caller. |
105 | */ |
106 | emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); |
107 | |
108 | emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); |
109 | |
110 | store_offset = stack_adjust - sizeof(long); |
111 | emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); |
112 | |
113 | store_offset -= sizeof(long); |
114 | emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); |
115 | |
116 | store_offset -= sizeof(long); |
117 | emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); |
118 | |
119 | store_offset -= sizeof(long); |
120 | emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); |
121 | |
122 | store_offset -= sizeof(long); |
123 | emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); |
124 | |
125 | store_offset -= sizeof(long); |
126 | emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); |
127 | |
128 | store_offset -= sizeof(long); |
129 | emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); |
130 | |
131 | store_offset -= sizeof(long); |
132 | emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); |
133 | |
134 | emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); |
135 | |
136 | if (bpf_stack_adjust) |
137 | emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); |
138 | |
139 | /* |
140 | * Program contains calls and tail calls, so REG_TCC need |
141 | * to be saved across calls. |
142 | */ |
143 | if (seen_tail_call(ctx) && seen_call(ctx)) |
144 | move_reg(ctx, TCC_SAVED, REG_TCC); |
145 | |
146 | ctx->stack_size = stack_adjust; |
147 | } |
148 | |
149 | static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) |
150 | { |
151 | int stack_adjust = ctx->stack_size; |
152 | int load_offset; |
153 | |
154 | load_offset = stack_adjust - sizeof(long); |
155 | emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); |
156 | |
157 | load_offset -= sizeof(long); |
158 | emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); |
159 | |
160 | load_offset -= sizeof(long); |
161 | emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); |
162 | |
163 | load_offset -= sizeof(long); |
164 | emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); |
165 | |
166 | load_offset -= sizeof(long); |
167 | emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); |
168 | |
169 | load_offset -= sizeof(long); |
170 | emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); |
171 | |
172 | load_offset -= sizeof(long); |
173 | emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); |
174 | |
175 | load_offset -= sizeof(long); |
176 | emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); |
177 | |
178 | emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); |
179 | |
180 | if (!is_tail_call) { |
181 | /* Set return value */ |
182 | move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]); |
183 | /* Return to the caller */ |
184 | emit_insn(ctx, jirl, LOONGARCH_GPR_RA, LOONGARCH_GPR_ZERO, 0); |
185 | } else { |
186 | /* |
187 | * Call the next bpf prog and skip the first instruction |
188 | * of TCC initialization. |
189 | */ |
190 | emit_insn(ctx, jirl, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, 1); |
191 | } |
192 | } |
193 | |
194 | static void build_epilogue(struct jit_ctx *ctx) |
195 | { |
196 | __build_epilogue(ctx, is_tail_call: false); |
197 | } |
198 | |
199 | bool bpf_jit_supports_kfunc_call(void) |
200 | { |
201 | return true; |
202 | } |
203 | |
204 | bool bpf_jit_supports_far_kfunc_call(void) |
205 | { |
206 | return true; |
207 | } |
208 | |
209 | /* initialized on the first pass of build_body() */ |
210 | static int out_offset = -1; |
211 | static int emit_bpf_tail_call(struct jit_ctx *ctx) |
212 | { |
213 | int off; |
214 | u8 tcc = tail_call_reg(ctx); |
215 | u8 a1 = LOONGARCH_GPR_A1; |
216 | u8 a2 = LOONGARCH_GPR_A2; |
217 | u8 t1 = LOONGARCH_GPR_T1; |
218 | u8 t2 = LOONGARCH_GPR_T2; |
219 | u8 t3 = LOONGARCH_GPR_T3; |
220 | const int idx0 = ctx->idx; |
221 | |
222 | #define cur_offset (ctx->idx - idx0) |
223 | #define jmp_offset (out_offset - (cur_offset)) |
224 | |
225 | /* |
226 | * a0: &ctx |
227 | * a1: &array |
228 | * a2: index |
229 | * |
230 | * if (index >= array->map.max_entries) |
231 | * goto out; |
232 | */ |
233 | off = offsetof(struct bpf_array, map.max_entries); |
234 | emit_insn(ctx, ldwu, t1, a1, off); |
235 | /* bgeu $a2, $t1, jmp_offset */ |
236 | if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) |
237 | goto toofar; |
238 | |
239 | /* |
240 | * if (--TCC < 0) |
241 | * goto out; |
242 | */ |
243 | emit_insn(ctx, addid, REG_TCC, tcc, -1); |
244 | if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0) |
245 | goto toofar; |
246 | |
247 | /* |
248 | * prog = array->ptrs[index]; |
249 | * if (!prog) |
250 | * goto out; |
251 | */ |
252 | emit_insn(ctx, alsld, t2, a2, a1, 2); |
253 | off = offsetof(struct bpf_array, ptrs); |
254 | emit_insn(ctx, ldd, t2, t2, off); |
255 | /* beq $t2, $zero, jmp_offset */ |
256 | if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) |
257 | goto toofar; |
258 | |
259 | /* goto *(prog->bpf_func + 4); */ |
260 | off = offsetof(struct bpf_prog, bpf_func); |
261 | emit_insn(ctx, ldd, t3, t2, off); |
262 | __build_epilogue(ctx, is_tail_call: true); |
263 | |
264 | /* out: */ |
265 | if (out_offset == -1) |
266 | out_offset = cur_offset; |
267 | if (cur_offset != out_offset) { |
268 | pr_err_once("tail_call out_offset = %d, expected %d!\n" , |
269 | cur_offset, out_offset); |
270 | return -1; |
271 | } |
272 | |
273 | return 0; |
274 | |
275 | toofar: |
276 | pr_info_once("tail_call: jump too far\n" ); |
277 | return -1; |
278 | #undef cur_offset |
279 | #undef jmp_offset |
280 | } |
281 | |
282 | static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) |
283 | { |
284 | const u8 t1 = LOONGARCH_GPR_T1; |
285 | const u8 t2 = LOONGARCH_GPR_T2; |
286 | const u8 t3 = LOONGARCH_GPR_T3; |
287 | const u8 r0 = regmap[BPF_REG_0]; |
288 | const u8 src = regmap[insn->src_reg]; |
289 | const u8 dst = regmap[insn->dst_reg]; |
290 | const s16 off = insn->off; |
291 | const s32 imm = insn->imm; |
292 | const bool isdw = BPF_SIZE(insn->code) == BPF_DW; |
293 | |
294 | move_imm(ctx, t1, off, false); |
295 | emit_insn(ctx, addd, t1, dst, t1); |
296 | move_reg(ctx, t3, src); |
297 | |
298 | switch (imm) { |
299 | /* lock *(size *)(dst + off) <op>= src */ |
300 | case BPF_ADD: |
301 | if (isdw) |
302 | emit_insn(ctx, amaddd, t2, t1, src); |
303 | else |
304 | emit_insn(ctx, amaddw, t2, t1, src); |
305 | break; |
306 | case BPF_AND: |
307 | if (isdw) |
308 | emit_insn(ctx, amandd, t2, t1, src); |
309 | else |
310 | emit_insn(ctx, amandw, t2, t1, src); |
311 | break; |
312 | case BPF_OR: |
313 | if (isdw) |
314 | emit_insn(ctx, amord, t2, t1, src); |
315 | else |
316 | emit_insn(ctx, amorw, t2, t1, src); |
317 | break; |
318 | case BPF_XOR: |
319 | if (isdw) |
320 | emit_insn(ctx, amxord, t2, t1, src); |
321 | else |
322 | emit_insn(ctx, amxorw, t2, t1, src); |
323 | break; |
324 | /* src = atomic_fetch_<op>(dst + off, src) */ |
325 | case BPF_ADD | BPF_FETCH: |
326 | if (isdw) { |
327 | emit_insn(ctx, amaddd, src, t1, t3); |
328 | } else { |
329 | emit_insn(ctx, amaddw, src, t1, t3); |
330 | emit_zext_32(ctx, src, true); |
331 | } |
332 | break; |
333 | case BPF_AND | BPF_FETCH: |
334 | if (isdw) { |
335 | emit_insn(ctx, amandd, src, t1, t3); |
336 | } else { |
337 | emit_insn(ctx, amandw, src, t1, t3); |
338 | emit_zext_32(ctx, src, true); |
339 | } |
340 | break; |
341 | case BPF_OR | BPF_FETCH: |
342 | if (isdw) { |
343 | emit_insn(ctx, amord, src, t1, t3); |
344 | } else { |
345 | emit_insn(ctx, amorw, src, t1, t3); |
346 | emit_zext_32(ctx, src, true); |
347 | } |
348 | break; |
349 | case BPF_XOR | BPF_FETCH: |
350 | if (isdw) { |
351 | emit_insn(ctx, amxord, src, t1, t3); |
352 | } else { |
353 | emit_insn(ctx, amxorw, src, t1, t3); |
354 | emit_zext_32(ctx, src, true); |
355 | } |
356 | break; |
357 | /* src = atomic_xchg(dst + off, src); */ |
358 | case BPF_XCHG: |
359 | if (isdw) { |
360 | emit_insn(ctx, amswapd, src, t1, t3); |
361 | } else { |
362 | emit_insn(ctx, amswapw, src, t1, t3); |
363 | emit_zext_32(ctx, src, true); |
364 | } |
365 | break; |
366 | /* r0 = atomic_cmpxchg(dst + off, r0, src); */ |
367 | case BPF_CMPXCHG: |
368 | move_reg(ctx, t2, r0); |
369 | if (isdw) { |
370 | emit_insn(ctx, lld, r0, t1, 0); |
371 | emit_insn(ctx, bne, t2, r0, 4); |
372 | move_reg(ctx, t3, src); |
373 | emit_insn(ctx, scd, t3, t1, 0); |
374 | emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); |
375 | } else { |
376 | emit_insn(ctx, llw, r0, t1, 0); |
377 | emit_zext_32(ctx, t2, true); |
378 | emit_zext_32(ctx, r0, true); |
379 | emit_insn(ctx, bne, t2, r0, 4); |
380 | move_reg(ctx, t3, src); |
381 | emit_insn(ctx, scw, t3, t1, 0); |
382 | emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); |
383 | emit_zext_32(ctx, r0, true); |
384 | } |
385 | break; |
386 | } |
387 | } |
388 | |
389 | static bool is_signed_bpf_cond(u8 cond) |
390 | { |
391 | return cond == BPF_JSGT || cond == BPF_JSLT || |
392 | cond == BPF_JSGE || cond == BPF_JSLE; |
393 | } |
394 | |
395 | #define BPF_FIXUP_REG_MASK GENMASK(31, 27) |
396 | #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) |
397 | |
398 | bool ex_handler_bpf(const struct exception_table_entry *ex, |
399 | struct pt_regs *regs) |
400 | { |
401 | int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); |
402 | off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); |
403 | |
404 | regs->regs[dst_reg] = 0; |
405 | regs->csr_era = (unsigned long)&ex->fixup - offset; |
406 | |
407 | return true; |
408 | } |
409 | |
410 | /* For accesses to BTF pointers, add an entry to the exception table */ |
411 | static int add_exception_handler(const struct bpf_insn *insn, |
412 | struct jit_ctx *ctx, |
413 | int dst_reg) |
414 | { |
415 | unsigned long pc; |
416 | off_t offset; |
417 | struct exception_table_entry *ex; |
418 | |
419 | if (!ctx->image || !ctx->prog->aux->extable) |
420 | return 0; |
421 | |
422 | if (BPF_MODE(insn->code) != BPF_PROBE_MEM && |
423 | BPF_MODE(insn->code) != BPF_PROBE_MEMSX) |
424 | return 0; |
425 | |
426 | if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) |
427 | return -EINVAL; |
428 | |
429 | ex = &ctx->prog->aux->extable[ctx->num_exentries]; |
430 | pc = (unsigned long)&ctx->image[ctx->idx - 1]; |
431 | |
432 | offset = pc - (long)&ex->insn; |
433 | if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) |
434 | return -ERANGE; |
435 | |
436 | ex->insn = offset; |
437 | |
438 | /* |
439 | * Since the extable follows the program, the fixup offset is always |
440 | * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value |
441 | * to keep things simple, and put the destination register in the upper |
442 | * bits. We don't need to worry about buildtime or runtime sort |
443 | * modifying the upper bits because the table is already sorted, and |
444 | * isn't part of the main exception table. |
445 | */ |
446 | offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); |
447 | if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) |
448 | return -ERANGE; |
449 | |
450 | ex->type = EX_TYPE_BPF; |
451 | ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); |
452 | |
453 | ctx->num_exentries++; |
454 | |
455 | return 0; |
456 | } |
457 | |
458 | static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ) |
459 | { |
460 | u8 tm = -1; |
461 | u64 func_addr; |
462 | bool func_addr_fixed, sign_extend; |
463 | int i = insn - ctx->prog->insnsi; |
464 | int ret, jmp_offset; |
465 | const u8 code = insn->code; |
466 | const u8 cond = BPF_OP(code); |
467 | const u8 t1 = LOONGARCH_GPR_T1; |
468 | const u8 t2 = LOONGARCH_GPR_T2; |
469 | const u8 src = regmap[insn->src_reg]; |
470 | const u8 dst = regmap[insn->dst_reg]; |
471 | const s16 off = insn->off; |
472 | const s32 imm = insn->imm; |
473 | const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; |
474 | |
475 | switch (code) { |
476 | /* dst = src */ |
477 | case BPF_ALU | BPF_MOV | BPF_X: |
478 | case BPF_ALU64 | BPF_MOV | BPF_X: |
479 | switch (off) { |
480 | case 0: |
481 | move_reg(ctx, dst, src); |
482 | emit_zext_32(ctx, dst, is32); |
483 | break; |
484 | case 8: |
485 | move_reg(ctx, t1, src); |
486 | emit_insn(ctx, extwb, dst, t1); |
487 | emit_zext_32(ctx, dst, is32); |
488 | break; |
489 | case 16: |
490 | move_reg(ctx, t1, src); |
491 | emit_insn(ctx, extwh, dst, t1); |
492 | emit_zext_32(ctx, dst, is32); |
493 | break; |
494 | case 32: |
495 | emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); |
496 | break; |
497 | } |
498 | break; |
499 | |
500 | /* dst = imm */ |
501 | case BPF_ALU | BPF_MOV | BPF_K: |
502 | case BPF_ALU64 | BPF_MOV | BPF_K: |
503 | move_imm(ctx, dst, imm, is32); |
504 | break; |
505 | |
506 | /* dst = dst + src */ |
507 | case BPF_ALU | BPF_ADD | BPF_X: |
508 | case BPF_ALU64 | BPF_ADD | BPF_X: |
509 | emit_insn(ctx, addd, dst, dst, src); |
510 | emit_zext_32(ctx, dst, is32); |
511 | break; |
512 | |
513 | /* dst = dst + imm */ |
514 | case BPF_ALU | BPF_ADD | BPF_K: |
515 | case BPF_ALU64 | BPF_ADD | BPF_K: |
516 | if (is_signed_imm12(imm)) { |
517 | emit_insn(ctx, addid, dst, dst, imm); |
518 | } else { |
519 | move_imm(ctx, t1, imm, is32); |
520 | emit_insn(ctx, addd, dst, dst, t1); |
521 | } |
522 | emit_zext_32(ctx, dst, is32); |
523 | break; |
524 | |
525 | /* dst = dst - src */ |
526 | case BPF_ALU | BPF_SUB | BPF_X: |
527 | case BPF_ALU64 | BPF_SUB | BPF_X: |
528 | emit_insn(ctx, subd, dst, dst, src); |
529 | emit_zext_32(ctx, dst, is32); |
530 | break; |
531 | |
532 | /* dst = dst - imm */ |
533 | case BPF_ALU | BPF_SUB | BPF_K: |
534 | case BPF_ALU64 | BPF_SUB | BPF_K: |
535 | if (is_signed_imm12(-imm)) { |
536 | emit_insn(ctx, addid, dst, dst, -imm); |
537 | } else { |
538 | move_imm(ctx, t1, imm, is32); |
539 | emit_insn(ctx, subd, dst, dst, t1); |
540 | } |
541 | emit_zext_32(ctx, dst, is32); |
542 | break; |
543 | |
544 | /* dst = dst * src */ |
545 | case BPF_ALU | BPF_MUL | BPF_X: |
546 | case BPF_ALU64 | BPF_MUL | BPF_X: |
547 | emit_insn(ctx, muld, dst, dst, src); |
548 | emit_zext_32(ctx, dst, is32); |
549 | break; |
550 | |
551 | /* dst = dst * imm */ |
552 | case BPF_ALU | BPF_MUL | BPF_K: |
553 | case BPF_ALU64 | BPF_MUL | BPF_K: |
554 | move_imm(ctx, t1, imm, is32); |
555 | emit_insn(ctx, muld, dst, dst, t1); |
556 | emit_zext_32(ctx, dst, is32); |
557 | break; |
558 | |
559 | /* dst = dst / src */ |
560 | case BPF_ALU | BPF_DIV | BPF_X: |
561 | case BPF_ALU64 | BPF_DIV | BPF_X: |
562 | if (!off) { |
563 | emit_zext_32(ctx, dst, is32); |
564 | move_reg(ctx, t1, src); |
565 | emit_zext_32(ctx, t1, is32); |
566 | emit_insn(ctx, divdu, dst, dst, t1); |
567 | emit_zext_32(ctx, dst, is32); |
568 | } else { |
569 | emit_sext_32(ctx, dst, is32); |
570 | move_reg(ctx, t1, src); |
571 | emit_sext_32(ctx, t1, is32); |
572 | emit_insn(ctx, divd, dst, dst, t1); |
573 | emit_sext_32(ctx, dst, is32); |
574 | } |
575 | break; |
576 | |
577 | /* dst = dst / imm */ |
578 | case BPF_ALU | BPF_DIV | BPF_K: |
579 | case BPF_ALU64 | BPF_DIV | BPF_K: |
580 | if (!off) { |
581 | move_imm(ctx, t1, imm, is32); |
582 | emit_zext_32(ctx, dst, is32); |
583 | emit_insn(ctx, divdu, dst, dst, t1); |
584 | emit_zext_32(ctx, dst, is32); |
585 | } else { |
586 | move_imm(ctx, t1, imm, false); |
587 | emit_sext_32(ctx, t1, is32); |
588 | emit_sext_32(ctx, dst, is32); |
589 | emit_insn(ctx, divd, dst, dst, t1); |
590 | emit_sext_32(ctx, dst, is32); |
591 | } |
592 | break; |
593 | |
594 | /* dst = dst % src */ |
595 | case BPF_ALU | BPF_MOD | BPF_X: |
596 | case BPF_ALU64 | BPF_MOD | BPF_X: |
597 | if (!off) { |
598 | emit_zext_32(ctx, dst, is32); |
599 | move_reg(ctx, t1, src); |
600 | emit_zext_32(ctx, t1, is32); |
601 | emit_insn(ctx, moddu, dst, dst, t1); |
602 | emit_zext_32(ctx, dst, is32); |
603 | } else { |
604 | emit_sext_32(ctx, dst, is32); |
605 | move_reg(ctx, t1, src); |
606 | emit_sext_32(ctx, t1, is32); |
607 | emit_insn(ctx, modd, dst, dst, t1); |
608 | emit_sext_32(ctx, dst, is32); |
609 | } |
610 | break; |
611 | |
612 | /* dst = dst % imm */ |
613 | case BPF_ALU | BPF_MOD | BPF_K: |
614 | case BPF_ALU64 | BPF_MOD | BPF_K: |
615 | if (!off) { |
616 | move_imm(ctx, t1, imm, is32); |
617 | emit_zext_32(ctx, dst, is32); |
618 | emit_insn(ctx, moddu, dst, dst, t1); |
619 | emit_zext_32(ctx, dst, is32); |
620 | } else { |
621 | move_imm(ctx, t1, imm, false); |
622 | emit_sext_32(ctx, t1, is32); |
623 | emit_sext_32(ctx, dst, is32); |
624 | emit_insn(ctx, modd, dst, dst, t1); |
625 | emit_sext_32(ctx, dst, is32); |
626 | } |
627 | break; |
628 | |
629 | /* dst = -dst */ |
630 | case BPF_ALU | BPF_NEG: |
631 | case BPF_ALU64 | BPF_NEG: |
632 | move_imm(ctx, t1, imm, is32); |
633 | emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); |
634 | emit_zext_32(ctx, dst, is32); |
635 | break; |
636 | |
637 | /* dst = dst & src */ |
638 | case BPF_ALU | BPF_AND | BPF_X: |
639 | case BPF_ALU64 | BPF_AND | BPF_X: |
640 | emit_insn(ctx, and, dst, dst, src); |
641 | emit_zext_32(ctx, dst, is32); |
642 | break; |
643 | |
644 | /* dst = dst & imm */ |
645 | case BPF_ALU | BPF_AND | BPF_K: |
646 | case BPF_ALU64 | BPF_AND | BPF_K: |
647 | if (is_unsigned_imm12(imm)) { |
648 | emit_insn(ctx, andi, dst, dst, imm); |
649 | } else { |
650 | move_imm(ctx, t1, imm, is32); |
651 | emit_insn(ctx, and, dst, dst, t1); |
652 | } |
653 | emit_zext_32(ctx, dst, is32); |
654 | break; |
655 | |
656 | /* dst = dst | src */ |
657 | case BPF_ALU | BPF_OR | BPF_X: |
658 | case BPF_ALU64 | BPF_OR | BPF_X: |
659 | emit_insn(ctx, or, dst, dst, src); |
660 | emit_zext_32(ctx, dst, is32); |
661 | break; |
662 | |
663 | /* dst = dst | imm */ |
664 | case BPF_ALU | BPF_OR | BPF_K: |
665 | case BPF_ALU64 | BPF_OR | BPF_K: |
666 | if (is_unsigned_imm12(imm)) { |
667 | emit_insn(ctx, ori, dst, dst, imm); |
668 | } else { |
669 | move_imm(ctx, t1, imm, is32); |
670 | emit_insn(ctx, or, dst, dst, t1); |
671 | } |
672 | emit_zext_32(ctx, dst, is32); |
673 | break; |
674 | |
675 | /* dst = dst ^ src */ |
676 | case BPF_ALU | BPF_XOR | BPF_X: |
677 | case BPF_ALU64 | BPF_XOR | BPF_X: |
678 | emit_insn(ctx, xor, dst, dst, src); |
679 | emit_zext_32(ctx, dst, is32); |
680 | break; |
681 | |
682 | /* dst = dst ^ imm */ |
683 | case BPF_ALU | BPF_XOR | BPF_K: |
684 | case BPF_ALU64 | BPF_XOR | BPF_K: |
685 | if (is_unsigned_imm12(imm)) { |
686 | emit_insn(ctx, xori, dst, dst, imm); |
687 | } else { |
688 | move_imm(ctx, t1, imm, is32); |
689 | emit_insn(ctx, xor, dst, dst, t1); |
690 | } |
691 | emit_zext_32(ctx, dst, is32); |
692 | break; |
693 | |
694 | /* dst = dst << src (logical) */ |
695 | case BPF_ALU | BPF_LSH | BPF_X: |
696 | emit_insn(ctx, sllw, dst, dst, src); |
697 | emit_zext_32(ctx, dst, is32); |
698 | break; |
699 | |
700 | case BPF_ALU64 | BPF_LSH | BPF_X: |
701 | emit_insn(ctx, slld, dst, dst, src); |
702 | break; |
703 | |
704 | /* dst = dst << imm (logical) */ |
705 | case BPF_ALU | BPF_LSH | BPF_K: |
706 | emit_insn(ctx, slliw, dst, dst, imm); |
707 | emit_zext_32(ctx, dst, is32); |
708 | break; |
709 | |
710 | case BPF_ALU64 | BPF_LSH | BPF_K: |
711 | emit_insn(ctx, sllid, dst, dst, imm); |
712 | break; |
713 | |
714 | /* dst = dst >> src (logical) */ |
715 | case BPF_ALU | BPF_RSH | BPF_X: |
716 | emit_insn(ctx, srlw, dst, dst, src); |
717 | emit_zext_32(ctx, dst, is32); |
718 | break; |
719 | |
720 | case BPF_ALU64 | BPF_RSH | BPF_X: |
721 | emit_insn(ctx, srld, dst, dst, src); |
722 | break; |
723 | |
724 | /* dst = dst >> imm (logical) */ |
725 | case BPF_ALU | BPF_RSH | BPF_K: |
726 | emit_insn(ctx, srliw, dst, dst, imm); |
727 | emit_zext_32(ctx, dst, is32); |
728 | break; |
729 | |
730 | case BPF_ALU64 | BPF_RSH | BPF_K: |
731 | emit_insn(ctx, srlid, dst, dst, imm); |
732 | break; |
733 | |
734 | /* dst = dst >> src (arithmetic) */ |
735 | case BPF_ALU | BPF_ARSH | BPF_X: |
736 | emit_insn(ctx, sraw, dst, dst, src); |
737 | emit_zext_32(ctx, dst, is32); |
738 | break; |
739 | |
740 | case BPF_ALU64 | BPF_ARSH | BPF_X: |
741 | emit_insn(ctx, srad, dst, dst, src); |
742 | break; |
743 | |
744 | /* dst = dst >> imm (arithmetic) */ |
745 | case BPF_ALU | BPF_ARSH | BPF_K: |
746 | emit_insn(ctx, sraiw, dst, dst, imm); |
747 | emit_zext_32(ctx, dst, is32); |
748 | break; |
749 | |
750 | case BPF_ALU64 | BPF_ARSH | BPF_K: |
751 | emit_insn(ctx, sraid, dst, dst, imm); |
752 | break; |
753 | |
754 | /* dst = BSWAP##imm(dst) */ |
755 | case BPF_ALU | BPF_END | BPF_FROM_LE: |
756 | switch (imm) { |
757 | case 16: |
758 | /* zero-extend 16 bits into 64 bits */ |
759 | emit_insn(ctx, bstrpickd, dst, dst, 15, 0); |
760 | break; |
761 | case 32: |
762 | /* zero-extend 32 bits into 64 bits */ |
763 | emit_zext_32(ctx, dst, is32); |
764 | break; |
765 | case 64: |
766 | /* do nothing */ |
767 | break; |
768 | } |
769 | break; |
770 | |
771 | case BPF_ALU | BPF_END | BPF_FROM_BE: |
772 | case BPF_ALU64 | BPF_END | BPF_FROM_LE: |
773 | switch (imm) { |
774 | case 16: |
775 | emit_insn(ctx, revb2h, dst, dst); |
776 | /* zero-extend 16 bits into 64 bits */ |
777 | emit_insn(ctx, bstrpickd, dst, dst, 15, 0); |
778 | break; |
779 | case 32: |
780 | emit_insn(ctx, revb2w, dst, dst); |
781 | /* clear the upper 32 bits */ |
782 | emit_zext_32(ctx, dst, true); |
783 | break; |
784 | case 64: |
785 | emit_insn(ctx, revbd, dst, dst); |
786 | break; |
787 | } |
788 | break; |
789 | |
790 | /* PC += off if dst cond src */ |
791 | case BPF_JMP | BPF_JEQ | BPF_X: |
792 | case BPF_JMP | BPF_JNE | BPF_X: |
793 | case BPF_JMP | BPF_JGT | BPF_X: |
794 | case BPF_JMP | BPF_JGE | BPF_X: |
795 | case BPF_JMP | BPF_JLT | BPF_X: |
796 | case BPF_JMP | BPF_JLE | BPF_X: |
797 | case BPF_JMP | BPF_JSGT | BPF_X: |
798 | case BPF_JMP | BPF_JSGE | BPF_X: |
799 | case BPF_JMP | BPF_JSLT | BPF_X: |
800 | case BPF_JMP | BPF_JSLE | BPF_X: |
801 | case BPF_JMP32 | BPF_JEQ | BPF_X: |
802 | case BPF_JMP32 | BPF_JNE | BPF_X: |
803 | case BPF_JMP32 | BPF_JGT | BPF_X: |
804 | case BPF_JMP32 | BPF_JGE | BPF_X: |
805 | case BPF_JMP32 | BPF_JLT | BPF_X: |
806 | case BPF_JMP32 | BPF_JLE | BPF_X: |
807 | case BPF_JMP32 | BPF_JSGT | BPF_X: |
808 | case BPF_JMP32 | BPF_JSGE | BPF_X: |
809 | case BPF_JMP32 | BPF_JSLT | BPF_X: |
810 | case BPF_JMP32 | BPF_JSLE | BPF_X: |
811 | jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx); |
812 | move_reg(ctx, t1, dst); |
813 | move_reg(ctx, t2, src); |
814 | if (is_signed_bpf_cond(BPF_OP(code))) { |
815 | emit_sext_32(ctx, t1, is32); |
816 | emit_sext_32(ctx, t2, is32); |
817 | } else { |
818 | emit_zext_32(ctx, t1, is32); |
819 | emit_zext_32(ctx, t2, is32); |
820 | } |
821 | if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) |
822 | goto toofar; |
823 | break; |
824 | |
825 | /* PC += off if dst cond imm */ |
826 | case BPF_JMP | BPF_JEQ | BPF_K: |
827 | case BPF_JMP | BPF_JNE | BPF_K: |
828 | case BPF_JMP | BPF_JGT | BPF_K: |
829 | case BPF_JMP | BPF_JGE | BPF_K: |
830 | case BPF_JMP | BPF_JLT | BPF_K: |
831 | case BPF_JMP | BPF_JLE | BPF_K: |
832 | case BPF_JMP | BPF_JSGT | BPF_K: |
833 | case BPF_JMP | BPF_JSGE | BPF_K: |
834 | case BPF_JMP | BPF_JSLT | BPF_K: |
835 | case BPF_JMP | BPF_JSLE | BPF_K: |
836 | case BPF_JMP32 | BPF_JEQ | BPF_K: |
837 | case BPF_JMP32 | BPF_JNE | BPF_K: |
838 | case BPF_JMP32 | BPF_JGT | BPF_K: |
839 | case BPF_JMP32 | BPF_JGE | BPF_K: |
840 | case BPF_JMP32 | BPF_JLT | BPF_K: |
841 | case BPF_JMP32 | BPF_JLE | BPF_K: |
842 | case BPF_JMP32 | BPF_JSGT | BPF_K: |
843 | case BPF_JMP32 | BPF_JSGE | BPF_K: |
844 | case BPF_JMP32 | BPF_JSLT | BPF_K: |
845 | case BPF_JMP32 | BPF_JSLE | BPF_K: |
846 | jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx); |
847 | if (imm) { |
848 | move_imm(ctx, t1, imm, false); |
849 | tm = t1; |
850 | } else { |
851 | /* If imm is 0, simply use zero register. */ |
852 | tm = LOONGARCH_GPR_ZERO; |
853 | } |
854 | move_reg(ctx, t2, dst); |
855 | if (is_signed_bpf_cond(BPF_OP(code))) { |
856 | emit_sext_32(ctx, tm, is32); |
857 | emit_sext_32(ctx, t2, is32); |
858 | } else { |
859 | emit_zext_32(ctx, tm, is32); |
860 | emit_zext_32(ctx, t2, is32); |
861 | } |
862 | if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) |
863 | goto toofar; |
864 | break; |
865 | |
866 | /* PC += off if dst & src */ |
867 | case BPF_JMP | BPF_JSET | BPF_X: |
868 | case BPF_JMP32 | BPF_JSET | BPF_X: |
869 | jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx); |
870 | emit_insn(ctx, and, t1, dst, src); |
871 | emit_zext_32(ctx, t1, is32); |
872 | if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) |
873 | goto toofar; |
874 | break; |
875 | |
876 | /* PC += off if dst & imm */ |
877 | case BPF_JMP | BPF_JSET | BPF_K: |
878 | case BPF_JMP32 | BPF_JSET | BPF_K: |
879 | jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx); |
880 | move_imm(ctx, t1, imm, is32); |
881 | emit_insn(ctx, and, t1, dst, t1); |
882 | emit_zext_32(ctx, t1, is32); |
883 | if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) |
884 | goto toofar; |
885 | break; |
886 | |
887 | /* PC += off */ |
888 | case BPF_JMP | BPF_JA: |
889 | case BPF_JMP32 | BPF_JA: |
890 | if (BPF_CLASS(code) == BPF_JMP) |
891 | jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx); |
892 | else |
893 | jmp_offset = bpf2la_offset(bpf_insn: i, off: imm, ctx); |
894 | if (emit_uncond_jmp(ctx, jmp_offset) < 0) |
895 | goto toofar; |
896 | break; |
897 | |
898 | /* function call */ |
899 | case BPF_JMP | BPF_CALL: |
900 | mark_call(ctx); |
901 | ret = bpf_jit_get_func_addr(prog: ctx->prog, insn, extra_pass, |
902 | func_addr: &func_addr, func_addr_fixed: &func_addr_fixed); |
903 | if (ret < 0) |
904 | return ret; |
905 | |
906 | move_addr(ctx, t1, func_addr); |
907 | emit_insn(ctx, jirl, t1, LOONGARCH_GPR_RA, 0); |
908 | move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); |
909 | break; |
910 | |
911 | /* tail call */ |
912 | case BPF_JMP | BPF_TAIL_CALL: |
913 | mark_tail_call(ctx); |
914 | if (emit_bpf_tail_call(ctx) < 0) |
915 | return -EINVAL; |
916 | break; |
917 | |
918 | /* function return */ |
919 | case BPF_JMP | BPF_EXIT: |
920 | if (i == ctx->prog->len - 1) |
921 | break; |
922 | |
923 | jmp_offset = epilogue_offset(ctx); |
924 | if (emit_uncond_jmp(ctx, jmp_offset) < 0) |
925 | goto toofar; |
926 | break; |
927 | |
928 | /* dst = imm64 */ |
929 | case BPF_LD | BPF_IMM | BPF_DW: |
930 | { |
931 | const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; |
932 | |
933 | move_imm(ctx, dst, imm64, is32); |
934 | return 1; |
935 | } |
936 | |
937 | /* dst = *(size *)(src + off) */ |
938 | case BPF_LDX | BPF_MEM | BPF_B: |
939 | case BPF_LDX | BPF_MEM | BPF_H: |
940 | case BPF_LDX | BPF_MEM | BPF_W: |
941 | case BPF_LDX | BPF_MEM | BPF_DW: |
942 | case BPF_LDX | BPF_PROBE_MEM | BPF_DW: |
943 | case BPF_LDX | BPF_PROBE_MEM | BPF_W: |
944 | case BPF_LDX | BPF_PROBE_MEM | BPF_H: |
945 | case BPF_LDX | BPF_PROBE_MEM | BPF_B: |
946 | /* dst_reg = (s64)*(signed size *)(src_reg + off) */ |
947 | case BPF_LDX | BPF_MEMSX | BPF_B: |
948 | case BPF_LDX | BPF_MEMSX | BPF_H: |
949 | case BPF_LDX | BPF_MEMSX | BPF_W: |
950 | case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: |
951 | case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: |
952 | case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: |
953 | sign_extend = BPF_MODE(insn->code) == BPF_MEMSX || |
954 | BPF_MODE(insn->code) == BPF_PROBE_MEMSX; |
955 | switch (BPF_SIZE(code)) { |
956 | case BPF_B: |
957 | if (is_signed_imm12(off)) { |
958 | if (sign_extend) |
959 | emit_insn(ctx, ldb, dst, src, off); |
960 | else |
961 | emit_insn(ctx, ldbu, dst, src, off); |
962 | } else { |
963 | move_imm(ctx, t1, off, is32); |
964 | if (sign_extend) |
965 | emit_insn(ctx, ldxb, dst, src, t1); |
966 | else |
967 | emit_insn(ctx, ldxbu, dst, src, t1); |
968 | } |
969 | break; |
970 | case BPF_H: |
971 | if (is_signed_imm12(off)) { |
972 | if (sign_extend) |
973 | emit_insn(ctx, ldh, dst, src, off); |
974 | else |
975 | emit_insn(ctx, ldhu, dst, src, off); |
976 | } else { |
977 | move_imm(ctx, t1, off, is32); |
978 | if (sign_extend) |
979 | emit_insn(ctx, ldxh, dst, src, t1); |
980 | else |
981 | emit_insn(ctx, ldxhu, dst, src, t1); |
982 | } |
983 | break; |
984 | case BPF_W: |
985 | if (is_signed_imm12(off)) { |
986 | if (sign_extend) |
987 | emit_insn(ctx, ldw, dst, src, off); |
988 | else |
989 | emit_insn(ctx, ldwu, dst, src, off); |
990 | } else { |
991 | move_imm(ctx, t1, off, is32); |
992 | if (sign_extend) |
993 | emit_insn(ctx, ldxw, dst, src, t1); |
994 | else |
995 | emit_insn(ctx, ldxwu, dst, src, t1); |
996 | } |
997 | break; |
998 | case BPF_DW: |
999 | move_imm(ctx, t1, off, is32); |
1000 | emit_insn(ctx, ldxd, dst, src, t1); |
1001 | break; |
1002 | } |
1003 | |
1004 | ret = add_exception_handler(insn, ctx, dst_reg: dst); |
1005 | if (ret) |
1006 | return ret; |
1007 | break; |
1008 | |
1009 | /* *(size *)(dst + off) = imm */ |
1010 | case BPF_ST | BPF_MEM | BPF_B: |
1011 | case BPF_ST | BPF_MEM | BPF_H: |
1012 | case BPF_ST | BPF_MEM | BPF_W: |
1013 | case BPF_ST | BPF_MEM | BPF_DW: |
1014 | switch (BPF_SIZE(code)) { |
1015 | case BPF_B: |
1016 | move_imm(ctx, t1, imm, is32); |
1017 | if (is_signed_imm12(off)) { |
1018 | emit_insn(ctx, stb, t1, dst, off); |
1019 | } else { |
1020 | move_imm(ctx, t2, off, is32); |
1021 | emit_insn(ctx, stxb, t1, dst, t2); |
1022 | } |
1023 | break; |
1024 | case BPF_H: |
1025 | move_imm(ctx, t1, imm, is32); |
1026 | if (is_signed_imm12(off)) { |
1027 | emit_insn(ctx, sth, t1, dst, off); |
1028 | } else { |
1029 | move_imm(ctx, t2, off, is32); |
1030 | emit_insn(ctx, stxh, t1, dst, t2); |
1031 | } |
1032 | break; |
1033 | case BPF_W: |
1034 | move_imm(ctx, t1, imm, is32); |
1035 | if (is_signed_imm12(off)) { |
1036 | emit_insn(ctx, stw, t1, dst, off); |
1037 | } else if (is_signed_imm14(off)) { |
1038 | emit_insn(ctx, stptrw, t1, dst, off); |
1039 | } else { |
1040 | move_imm(ctx, t2, off, is32); |
1041 | emit_insn(ctx, stxw, t1, dst, t2); |
1042 | } |
1043 | break; |
1044 | case BPF_DW: |
1045 | move_imm(ctx, t1, imm, is32); |
1046 | if (is_signed_imm12(off)) { |
1047 | emit_insn(ctx, std, t1, dst, off); |
1048 | } else if (is_signed_imm14(off)) { |
1049 | emit_insn(ctx, stptrd, t1, dst, off); |
1050 | } else { |
1051 | move_imm(ctx, t2, off, is32); |
1052 | emit_insn(ctx, stxd, t1, dst, t2); |
1053 | } |
1054 | break; |
1055 | } |
1056 | break; |
1057 | |
1058 | /* *(size *)(dst + off) = src */ |
1059 | case BPF_STX | BPF_MEM | BPF_B: |
1060 | case BPF_STX | BPF_MEM | BPF_H: |
1061 | case BPF_STX | BPF_MEM | BPF_W: |
1062 | case BPF_STX | BPF_MEM | BPF_DW: |
1063 | switch (BPF_SIZE(code)) { |
1064 | case BPF_B: |
1065 | if (is_signed_imm12(off)) { |
1066 | emit_insn(ctx, stb, src, dst, off); |
1067 | } else { |
1068 | move_imm(ctx, t1, off, is32); |
1069 | emit_insn(ctx, stxb, src, dst, t1); |
1070 | } |
1071 | break; |
1072 | case BPF_H: |
1073 | if (is_signed_imm12(off)) { |
1074 | emit_insn(ctx, sth, src, dst, off); |
1075 | } else { |
1076 | move_imm(ctx, t1, off, is32); |
1077 | emit_insn(ctx, stxh, src, dst, t1); |
1078 | } |
1079 | break; |
1080 | case BPF_W: |
1081 | if (is_signed_imm12(off)) { |
1082 | emit_insn(ctx, stw, src, dst, off); |
1083 | } else if (is_signed_imm14(off)) { |
1084 | emit_insn(ctx, stptrw, src, dst, off); |
1085 | } else { |
1086 | move_imm(ctx, t1, off, is32); |
1087 | emit_insn(ctx, stxw, src, dst, t1); |
1088 | } |
1089 | break; |
1090 | case BPF_DW: |
1091 | if (is_signed_imm12(off)) { |
1092 | emit_insn(ctx, std, src, dst, off); |
1093 | } else if (is_signed_imm14(off)) { |
1094 | emit_insn(ctx, stptrd, src, dst, off); |
1095 | } else { |
1096 | move_imm(ctx, t1, off, is32); |
1097 | emit_insn(ctx, stxd, src, dst, t1); |
1098 | } |
1099 | break; |
1100 | } |
1101 | break; |
1102 | |
1103 | case BPF_STX | BPF_ATOMIC | BPF_W: |
1104 | case BPF_STX | BPF_ATOMIC | BPF_DW: |
1105 | emit_atomic(insn, ctx); |
1106 | break; |
1107 | |
1108 | /* Speculation barrier */ |
1109 | case BPF_ST | BPF_NOSPEC: |
1110 | break; |
1111 | |
1112 | default: |
1113 | pr_err("bpf_jit: unknown opcode %02x\n" , code); |
1114 | return -EINVAL; |
1115 | } |
1116 | |
1117 | return 0; |
1118 | |
1119 | toofar: |
1120 | pr_info_once("bpf_jit: opcode %02x, jump too far\n" , code); |
1121 | return -E2BIG; |
1122 | } |
1123 | |
1124 | static int build_body(struct jit_ctx *ctx, bool ) |
1125 | { |
1126 | int i; |
1127 | const struct bpf_prog *prog = ctx->prog; |
1128 | |
1129 | for (i = 0; i < prog->len; i++) { |
1130 | const struct bpf_insn *insn = &prog->insnsi[i]; |
1131 | int ret; |
1132 | |
1133 | if (ctx->image == NULL) |
1134 | ctx->offset[i] = ctx->idx; |
1135 | |
1136 | ret = build_insn(insn, ctx, extra_pass); |
1137 | if (ret > 0) { |
1138 | i++; |
1139 | if (ctx->image == NULL) |
1140 | ctx->offset[i] = ctx->idx; |
1141 | continue; |
1142 | } |
1143 | if (ret) |
1144 | return ret; |
1145 | } |
1146 | |
1147 | if (ctx->image == NULL) |
1148 | ctx->offset[i] = ctx->idx; |
1149 | |
1150 | return 0; |
1151 | } |
1152 | |
1153 | /* Fill space with break instructions */ |
1154 | static void jit_fill_hole(void *area, unsigned int size) |
1155 | { |
1156 | u32 *ptr; |
1157 | |
1158 | /* We are guaranteed to have aligned memory */ |
1159 | for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) |
1160 | *ptr++ = INSN_BREAK; |
1161 | } |
1162 | |
1163 | static int validate_code(struct jit_ctx *ctx) |
1164 | { |
1165 | int i; |
1166 | union loongarch_instruction insn; |
1167 | |
1168 | for (i = 0; i < ctx->idx; i++) { |
1169 | insn = ctx->image[i]; |
1170 | /* Check INSN_BREAK */ |
1171 | if (insn.word == INSN_BREAK) |
1172 | return -1; |
1173 | } |
1174 | |
1175 | if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) |
1176 | return -1; |
1177 | |
1178 | return 0; |
1179 | } |
1180 | |
1181 | struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) |
1182 | { |
1183 | bool tmp_blinded = false, = false; |
1184 | u8 *image_ptr; |
1185 | int image_size, prog_size, extable_size; |
1186 | struct jit_ctx ctx; |
1187 | struct jit_data *jit_data; |
1188 | struct bpf_binary_header *; |
1189 | struct bpf_prog *tmp, *orig_prog = prog; |
1190 | |
1191 | /* |
1192 | * If BPF JIT was not enabled then we must fall back to |
1193 | * the interpreter. |
1194 | */ |
1195 | if (!prog->jit_requested) |
1196 | return orig_prog; |
1197 | |
1198 | tmp = bpf_jit_blind_constants(fp: prog); |
1199 | /* |
1200 | * If blinding was requested and we failed during blinding, |
1201 | * we must fall back to the interpreter. Otherwise, we save |
1202 | * the new JITed code. |
1203 | */ |
1204 | if (IS_ERR(ptr: tmp)) |
1205 | return orig_prog; |
1206 | |
1207 | if (tmp != prog) { |
1208 | tmp_blinded = true; |
1209 | prog = tmp; |
1210 | } |
1211 | |
1212 | jit_data = prog->aux->jit_data; |
1213 | if (!jit_data) { |
1214 | jit_data = kzalloc(size: sizeof(*jit_data), GFP_KERNEL); |
1215 | if (!jit_data) { |
1216 | prog = orig_prog; |
1217 | goto out; |
1218 | } |
1219 | prog->aux->jit_data = jit_data; |
1220 | } |
1221 | if (jit_data->ctx.offset) { |
1222 | ctx = jit_data->ctx; |
1223 | image_ptr = jit_data->image; |
1224 | header = jit_data->header; |
1225 | extra_pass = true; |
1226 | prog_size = sizeof(u32) * ctx.idx; |
1227 | goto skip_init_ctx; |
1228 | } |
1229 | |
1230 | memset(&ctx, 0, sizeof(ctx)); |
1231 | ctx.prog = prog; |
1232 | |
1233 | ctx.offset = kvcalloc(n: prog->len + 1, size: sizeof(u32), GFP_KERNEL); |
1234 | if (ctx.offset == NULL) { |
1235 | prog = orig_prog; |
1236 | goto out_offset; |
1237 | } |
1238 | |
1239 | /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ |
1240 | build_prologue(ctx: &ctx); |
1241 | if (build_body(ctx: &ctx, extra_pass)) { |
1242 | prog = orig_prog; |
1243 | goto out_offset; |
1244 | } |
1245 | ctx.epilogue_offset = ctx.idx; |
1246 | build_epilogue(ctx: &ctx); |
1247 | |
1248 | extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); |
1249 | |
1250 | /* Now we know the actual image size. |
1251 | * As each LoongArch instruction is of length 32bit, |
1252 | * we are translating number of JITed intructions into |
1253 | * the size required to store these JITed code. |
1254 | */ |
1255 | prog_size = sizeof(u32) * ctx.idx; |
1256 | image_size = prog_size + extable_size; |
1257 | /* Now we know the size of the structure to make */ |
1258 | header = bpf_jit_binary_alloc(proglen: image_size, image_ptr: &image_ptr, |
1259 | alignment: sizeof(u32), bpf_fill_ill_insns: jit_fill_hole); |
1260 | if (header == NULL) { |
1261 | prog = orig_prog; |
1262 | goto out_offset; |
1263 | } |
1264 | |
1265 | /* 2. Now, the actual pass to generate final JIT code */ |
1266 | ctx.image = (union loongarch_instruction *)image_ptr; |
1267 | if (extable_size) |
1268 | prog->aux->extable = (void *)image_ptr + prog_size; |
1269 | |
1270 | skip_init_ctx: |
1271 | ctx.idx = 0; |
1272 | ctx.num_exentries = 0; |
1273 | |
1274 | build_prologue(ctx: &ctx); |
1275 | if (build_body(ctx: &ctx, extra_pass)) { |
1276 | bpf_jit_binary_free(hdr: header); |
1277 | prog = orig_prog; |
1278 | goto out_offset; |
1279 | } |
1280 | build_epilogue(ctx: &ctx); |
1281 | |
1282 | /* 3. Extra pass to validate JITed code */ |
1283 | if (validate_code(ctx: &ctx)) { |
1284 | bpf_jit_binary_free(hdr: header); |
1285 | prog = orig_prog; |
1286 | goto out_offset; |
1287 | } |
1288 | |
1289 | /* And we're done */ |
1290 | if (bpf_jit_enable > 1) |
1291 | bpf_jit_dump(flen: prog->len, proglen: prog_size, pass: 2, image: ctx.image); |
1292 | |
1293 | /* Update the icache */ |
1294 | flush_icache_range(start: (unsigned long)header, end: (unsigned long)(ctx.image + ctx.idx)); |
1295 | |
1296 | if (!prog->is_func || extra_pass) { |
1297 | if (extra_pass && ctx.idx != jit_data->ctx.idx) { |
1298 | pr_err_once("multi-func JIT bug %d != %d\n" , |
1299 | ctx.idx, jit_data->ctx.idx); |
1300 | bpf_jit_binary_free(hdr: header); |
1301 | prog->bpf_func = NULL; |
1302 | prog->jited = 0; |
1303 | prog->jited_len = 0; |
1304 | goto out_offset; |
1305 | } |
1306 | bpf_jit_binary_lock_ro(hdr: header); |
1307 | } else { |
1308 | jit_data->ctx = ctx; |
1309 | jit_data->image = image_ptr; |
1310 | jit_data->header = header; |
1311 | } |
1312 | prog->jited = 1; |
1313 | prog->jited_len = prog_size; |
1314 | prog->bpf_func = (void *)ctx.image; |
1315 | |
1316 | if (!prog->is_func || extra_pass) { |
1317 | int i; |
1318 | |
1319 | /* offset[prog->len] is the size of program */ |
1320 | for (i = 0; i <= prog->len; i++) |
1321 | ctx.offset[i] *= LOONGARCH_INSN_SIZE; |
1322 | bpf_prog_fill_jited_linfo(prog, insn_to_jit_off: ctx.offset + 1); |
1323 | |
1324 | out_offset: |
1325 | kvfree(addr: ctx.offset); |
1326 | kfree(objp: jit_data); |
1327 | prog->aux->jit_data = NULL; |
1328 | } |
1329 | |
1330 | out: |
1331 | if (tmp_blinded) |
1332 | bpf_jit_prog_release_other(fp: prog, fp_other: prog == orig_prog ? tmp : orig_prog); |
1333 | |
1334 | out_offset = -1; |
1335 | |
1336 | return prog; |
1337 | } |
1338 | |
1339 | /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ |
1340 | bool bpf_jit_supports_subprog_tailcalls(void) |
1341 | { |
1342 | return true; |
1343 | } |
1344 | |