1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* BPF JIT compiler for LoongArch
4	*
5	* Copyright (C) 2022 Loongson Technology Corporation Limited
6	*/
7	#include <linux/memory.h>
8	#include "bpf_jit.h"
9
10	#define LOONGARCH_MAX_REG_ARGS 8
11
12	#define LOONGARCH_LONG_JUMP_NINSNS 5
13	#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
14
15	#define LOONGARCH_FENTRY_NINSNS 2
16	#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
17	#define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
18
19	#define REG_TCC LOONGARCH_GPR_A6
20	#define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80)
21
22	static const int regmap[] = {
23	/* return value from in-kernel function, and exit value for eBPF program */
24	[BPF_REG_0] = LOONGARCH_GPR_A5,
25	/* arguments from eBPF program to in-kernel function */
26	[BPF_REG_1] = LOONGARCH_GPR_A0,
27	[BPF_REG_2] = LOONGARCH_GPR_A1,
28	[BPF_REG_3] = LOONGARCH_GPR_A2,
29	[BPF_REG_4] = LOONGARCH_GPR_A3,
30	[BPF_REG_5] = LOONGARCH_GPR_A4,
31	/* callee saved registers that in-kernel function will preserve */
32	[BPF_REG_6] = LOONGARCH_GPR_S0,
33	[BPF_REG_7] = LOONGARCH_GPR_S1,
34	[BPF_REG_8] = LOONGARCH_GPR_S2,
35	[BPF_REG_9] = LOONGARCH_GPR_S3,
36	/* read-only frame pointer to access stack */
37	[BPF_REG_FP] = LOONGARCH_GPR_S4,
38	/* temporary register for blinding constants */
39	[BPF_REG_AX] = LOONGARCH_GPR_T0,
40	};
41
42	static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
43	{
44	const struct bpf_prog *prog = ctx->prog;
45	const bool is_main_prog = !bpf_is_subprog(prog);
46
47	if (is_main_prog) {
48	/*
49	* LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT
50	* if (REG_TCC > T3 )
51	* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
52	* else
53	* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
54	* REG_TCC = LOONGARCH_GPR_SP + store_offset
55	*
56	* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
57	*
58	* The purpose of this code is to first push the TCC into stack,
59	* and then push the address of TCC into stack.
60	* In cases where bpf2bpf and tailcall are used in combination,
61	* the value in REG_TCC may be a count or an address,
62	* these two cases need to be judged and handled separately.
63	*/
64	emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
65	*store_offset -= sizeof(long);
66
67	emit_cond_jmp(ctx, BPF_JGT, REG_TCC, rd: LOONGARCH_GPR_T3, jmp_offset: 4);
68
69	/*
70	* If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count,
71	* push tcc into stack
72	*/
73	emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
74
75	/* Push the address of TCC into the REG_TCC */
76	emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
77
78	emit_uncond_jmp(ctx, jmp_offset: 2);
79
80	/*
81	* If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address,
82	* push tcc_ptr into stack
83	*/
84	emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
85	} else {
86	*store_offset -= sizeof(long);
87	emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
88	}
89
90	/* Push tcc_ptr into stack */
91	*store_offset -= sizeof(long);
92	emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
93	}
94
95	/*
96	* eBPF prog stack layout:
97	*
98	* high
99	* original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
100	* | $ra |
101	* +-------------------------+
102	* | $fp |
103	* +-------------------------+
104	* | $s0 |
105	* +-------------------------+
106	* | $s1 |
107	* +-------------------------+
108	* | $s2 |
109	* +-------------------------+
110	* | $s3 |
111	* +-------------------------+
112	* | $s4 |
113	* +-------------------------+
114	* | $s5 |
115	* +-------------------------+
116	* | tcc |
117	* +-------------------------+
118	* | tcc_ptr |
119	* +-------------------------+ <--BPF_REG_FP
120	* | prog->aux->stack_depth |
121	* | (optional) |
122	* current $sp -------------> +-------------------------+
123	* low
124	*/
125	static void build_prologue(struct jit_ctx *ctx)
126	{
127	int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
128	const struct bpf_prog *prog = ctx->prog;
129	const bool is_main_prog = !bpf_is_subprog(prog);
130
131	bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
132
133	/* To store ra, fp, s0, s1, s2, s3, s4, s5 */
134	stack_adjust += sizeof(long) * 8;
135
136	/* To store tcc and tcc_ptr */
137	stack_adjust += sizeof(long) * 2;
138
139	stack_adjust = round_up(stack_adjust, 16);
140	stack_adjust += bpf_stack_adjust;
141
142	move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA);
143	/* Reserve space for the move_imm + jirl instruction */
144	for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
145	emit_insn(ctx, nop);
146
147	/*
148	* First instruction initializes the tail call count (TCC)
149	* register to zero. On tail call we skip this instruction,
150	* and the TCC is passed in REG_TCC from the caller.
151	*/
152	if (is_main_prog)
153	emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0);
154
155	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
156
157	store_offset = stack_adjust - sizeof(long);
158	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
159
160	store_offset -= sizeof(long);
161	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
162
163	store_offset -= sizeof(long);
164	emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
165
166	store_offset -= sizeof(long);
167	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
168
169	store_offset -= sizeof(long);
170	emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
171
172	store_offset -= sizeof(long);
173	emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
174
175	store_offset -= sizeof(long);
176	emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
177
178	store_offset -= sizeof(long);
179	emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
180
181	prepare_bpf_tail_call_cnt(ctx, store_offset: &store_offset);
182
183	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
184
185	if (bpf_stack_adjust)
186	emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
187
188	ctx->stack_size = stack_adjust;
189	}
190
191	static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
192	{
193	int stack_adjust = ctx->stack_size;
194	int load_offset;
195
196	load_offset = stack_adjust - sizeof(long);
197	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
198
199	load_offset -= sizeof(long);
200	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
201
202	load_offset -= sizeof(long);
203	emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
204
205	load_offset -= sizeof(long);
206	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
207
208	load_offset -= sizeof(long);
209	emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
210
211	load_offset -= sizeof(long);
212	emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
213
214	load_offset -= sizeof(long);
215	emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
216
217	load_offset -= sizeof(long);
218	emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
219
220	/*
221	* When push into the stack, follow the order of tcc then tcc_ptr.
222	* When pop from the stack, first pop tcc_ptr then followed by tcc.
223	*/
224	load_offset -= 2 * sizeof(long);
225	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
226
227	load_offset += sizeof(long);
228	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
229
230	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
231
232	if (!is_tail_call) {
233	/* Set return value */
234	emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0);
235	/* Return to the caller */
236	emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
237	} else {
238	/*
239	* Call the next bpf prog and skip the first instruction
240	* of TCC initialization.
241	*/
242	emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 7);
243	}
244	}
245
246	static void build_epilogue(struct jit_ctx *ctx)
247	{
248	__build_epilogue(ctx, is_tail_call: false);
249	}
250
251	bool bpf_jit_supports_kfunc_call(void)
252	{
253	return true;
254	}
255
256	bool bpf_jit_supports_far_kfunc_call(void)
257	{
258	return true;
259	}
260
261	static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
262	{
263	int off, tc_ninsn = 0;
264	int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
265	u8 a1 = LOONGARCH_GPR_A1;
266	u8 a2 = LOONGARCH_GPR_A2;
267	u8 t1 = LOONGARCH_GPR_T1;
268	u8 t2 = LOONGARCH_GPR_T2;
269	u8 t3 = LOONGARCH_GPR_T3;
270	const int idx0 = ctx->idx;
271
272	#define cur_offset (ctx->idx - idx0)
273	#define jmp_offset (tc_ninsn - (cur_offset))
274
275	/*
276	* a0: &ctx
277	* a1: &array
278	* a2: index
279	*
280	* if (index >= array->map.max_entries)
281	* goto out;
282	*/
283	tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0];
284	emit_zext_32(ctx, a2, true);
285
286	off = offsetof(struct bpf_array, map.max_entries);
287	emit_insn(ctx, ldwu, t1, a1, off);
288	/* bgeu $a2, $t1, jmp_offset */
289	if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
290	goto toofar;
291
292	/*
293	* if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
294	* goto out;
295	*/
296	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
297	emit_insn(ctx, ldd, t3, REG_TCC, 0);
298	emit_insn(ctx, addid, t3, t3, 1);
299	emit_insn(ctx, std, t3, REG_TCC, 0);
300	emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
301	if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0)
302	goto toofar;
303
304	/*
305	* prog = array->ptrs[index];
306	* if (!prog)
307	* goto out;
308	*/
309	emit_insn(ctx, alsld, t2, a2, a1, 2);
310	off = offsetof(struct bpf_array, ptrs);
311	emit_insn(ctx, ldd, t2, t2, off);
312	/* beq $t2, $zero, jmp_offset */
313	if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
314	goto toofar;
315
316	/* goto *(prog->bpf_func + 4); */
317	off = offsetof(struct bpf_prog, bpf_func);
318	emit_insn(ctx, ldd, t3, t2, off);
319	__build_epilogue(ctx, is_tail_call: true);
320
321	return 0;
322
323	toofar:
324	pr_info_once("tail_call: jump too far\n");
325	return -1;
326	#undef cur_offset
327	#undef jmp_offset
328	}
329
330	static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
331	{
332	const u8 t1 = LOONGARCH_GPR_T1;
333	const u8 t2 = LOONGARCH_GPR_T2;
334	const u8 t3 = LOONGARCH_GPR_T3;
335	const u8 r0 = regmap[BPF_REG_0];
336	const u8 src = regmap[insn->src_reg];
337	const u8 dst = regmap[insn->dst_reg];
338	const s16 off = insn->off;
339	const s32 imm = insn->imm;
340	const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
341
342	move_imm(ctx, t1, off, false);
343	emit_insn(ctx, addd, t1, dst, t1);
344	move_reg(ctx, t3, src);
345
346	switch (imm) {
347	/* lock *(size *)(dst + off) <op>= src */
348	case BPF_ADD:
349	if (isdw)
350	emit_insn(ctx, amaddd, t2, t1, src);
351	else
352	emit_insn(ctx, amaddw, t2, t1, src);
353	break;
354	case BPF_AND:
355	if (isdw)
356	emit_insn(ctx, amandd, t2, t1, src);
357	else
358	emit_insn(ctx, amandw, t2, t1, src);
359	break;
360	case BPF_OR:
361	if (isdw)
362	emit_insn(ctx, amord, t2, t1, src);
363	else
364	emit_insn(ctx, amorw, t2, t1, src);
365	break;
366	case BPF_XOR:
367	if (isdw)
368	emit_insn(ctx, amxord, t2, t1, src);
369	else
370	emit_insn(ctx, amxorw, t2, t1, src);
371	break;
372	/* src = atomic_fetch_<op>(dst + off, src) */
373	case BPF_ADD | BPF_FETCH:
374	if (isdw) {
375	emit_insn(ctx, amaddd, src, t1, t3);
376	} else {
377	emit_insn(ctx, amaddw, src, t1, t3);
378	emit_zext_32(ctx, src, true);
379	}
380	break;
381	case BPF_AND | BPF_FETCH:
382	if (isdw) {
383	emit_insn(ctx, amandd, src, t1, t3);
384	} else {
385	emit_insn(ctx, amandw, src, t1, t3);
386	emit_zext_32(ctx, src, true);
387	}
388	break;
389	case BPF_OR | BPF_FETCH:
390	if (isdw) {
391	emit_insn(ctx, amord, src, t1, t3);
392	} else {
393	emit_insn(ctx, amorw, src, t1, t3);
394	emit_zext_32(ctx, src, true);
395	}
396	break;
397	case BPF_XOR | BPF_FETCH:
398	if (isdw) {
399	emit_insn(ctx, amxord, src, t1, t3);
400	} else {
401	emit_insn(ctx, amxorw, src, t1, t3);
402	emit_zext_32(ctx, src, true);
403	}
404	break;
405	/* src = atomic_xchg(dst + off, src); */
406	case BPF_XCHG:
407	if (isdw) {
408	emit_insn(ctx, amswapd, src, t1, t3);
409	} else {
410	emit_insn(ctx, amswapw, src, t1, t3);
411	emit_zext_32(ctx, src, true);
412	}
413	break;
414	/* r0 = atomic_cmpxchg(dst + off, r0, src); */
415	case BPF_CMPXCHG:
416	move_reg(ctx, t2, r0);
417	if (isdw) {
418	emit_insn(ctx, lld, r0, t1, 0);
419	emit_insn(ctx, bne, t2, r0, 4);
420	move_reg(ctx, t3, src);
421	emit_insn(ctx, scd, t3, t1, 0);
422	emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
423	} else {
424	emit_insn(ctx, llw, r0, t1, 0);
425	emit_zext_32(ctx, t2, true);
426	emit_zext_32(ctx, r0, true);
427	emit_insn(ctx, bne, t2, r0, 4);
428	move_reg(ctx, t3, src);
429	emit_insn(ctx, scw, t3, t1, 0);
430	emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
431	emit_zext_32(ctx, r0, true);
432	}
433	break;
434	}
435	}
436
437	static bool is_signed_bpf_cond(u8 cond)
438	{
439	return cond == BPF_JSGT || cond == BPF_JSLT ||
440	cond == BPF_JSGE || cond == BPF_JSLE;
441	}
442
443	#define BPF_FIXUP_REG_MASK GENMASK(31, 27)
444	#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
445
446	bool ex_handler_bpf(const struct exception_table_entry *ex,
447	struct pt_regs *regs)
448	{
449	int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
450	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
451
452	regs->regs[dst_reg] = 0;
453	regs->csr_era = (unsigned long)&ex->fixup - offset;
454
455	return true;
456	}
457
458	/* For accesses to BTF pointers, add an entry to the exception table */
459	static int add_exception_handler(const struct bpf_insn *insn,
460	struct jit_ctx *ctx,
461	int dst_reg)
462	{
463	unsigned long pc;
464	off_t offset;
465	struct exception_table_entry *ex;
466
467	if (!ctx->image || !ctx->prog->aux->extable)
468	return 0;
469
470	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
471	BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
472	return 0;
473
474	if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
475	return -EINVAL;
476
477	ex = &ctx->prog->aux->extable[ctx->num_exentries];
478	pc = (unsigned long)&ctx->image[ctx->idx - 1];
479
480	offset = pc - (long)&ex->insn;
481	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
482	return -ERANGE;
483
484	ex->insn = offset;
485
486	/*
487	* Since the extable follows the program, the fixup offset is always
488	* negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
489	* to keep things simple, and put the destination register in the upper
490	* bits. We don't need to worry about buildtime or runtime sort
491	* modifying the upper bits because the table is already sorted, and
492	* isn't part of the main exception table.
493	*/
494	offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
495	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
496	return -ERANGE;
497
498	ex->type = EX_TYPE_BPF;
499	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
500
501	ctx->num_exentries++;
502
503	return 0;
504	}
505
506	static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
507	{
508	u8 tm = -1;
509	u64 func_addr;
510	bool func_addr_fixed, sign_extend;
511	int i = insn - ctx->prog->insnsi;
512	int ret, jmp_offset, tcc_ptr_off;
513	const u8 code = insn->code;
514	const u8 cond = BPF_OP(code);
515	const u8 t1 = LOONGARCH_GPR_T1;
516	const u8 t2 = LOONGARCH_GPR_T2;
517	const u8 src = regmap[insn->src_reg];
518	const u8 dst = regmap[insn->dst_reg];
519	const s16 off = insn->off;
520	const s32 imm = insn->imm;
521	const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
522
523	switch (code) {
524	/* dst = src */
525	case BPF_ALU | BPF_MOV | BPF_X:
526	case BPF_ALU64 | BPF_MOV | BPF_X:
527	switch (off) {
528	case 0:
529	move_reg(ctx, dst, src);
530	emit_zext_32(ctx, dst, is32);
531	break;
532	case 8:
533	emit_insn(ctx, extwb, dst, src);
534	emit_zext_32(ctx, dst, is32);
535	break;
536	case 16:
537	emit_insn(ctx, extwh, dst, src);
538	emit_zext_32(ctx, dst, is32);
539	break;
540	case 32:
541	emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
542	break;
543	}
544	break;
545
546	/* dst = imm */
547	case BPF_ALU | BPF_MOV | BPF_K:
548	case BPF_ALU64 | BPF_MOV | BPF_K:
549	move_imm(ctx, dst, imm, is32);
550	break;
551
552	/* dst = dst + src */
553	case BPF_ALU | BPF_ADD | BPF_X:
554	case BPF_ALU64 | BPF_ADD | BPF_X:
555	emit_insn(ctx, addd, dst, dst, src);
556	emit_zext_32(ctx, dst, is32);
557	break;
558
559	/* dst = dst + imm */
560	case BPF_ALU | BPF_ADD | BPF_K:
561	case BPF_ALU64 | BPF_ADD | BPF_K:
562	if (is_signed_imm12(imm)) {
563	emit_insn(ctx, addid, dst, dst, imm);
564	} else {
565	move_imm(ctx, t1, imm, is32);
566	emit_insn(ctx, addd, dst, dst, t1);
567	}
568	emit_zext_32(ctx, dst, is32);
569	break;
570
571	/* dst = dst - src */
572	case BPF_ALU | BPF_SUB | BPF_X:
573	case BPF_ALU64 | BPF_SUB | BPF_X:
574	emit_insn(ctx, subd, dst, dst, src);
575	emit_zext_32(ctx, dst, is32);
576	break;
577
578	/* dst = dst - imm */
579	case BPF_ALU | BPF_SUB | BPF_K:
580	case BPF_ALU64 | BPF_SUB | BPF_K:
581	if (is_signed_imm12(-imm)) {
582	emit_insn(ctx, addid, dst, dst, -imm);
583	} else {
584	move_imm(ctx, t1, imm, is32);
585	emit_insn(ctx, subd, dst, dst, t1);
586	}
587	emit_zext_32(ctx, dst, is32);
588	break;
589
590	/* dst = dst * src */
591	case BPF_ALU | BPF_MUL | BPF_X:
592	case BPF_ALU64 | BPF_MUL | BPF_X:
593	emit_insn(ctx, muld, dst, dst, src);
594	emit_zext_32(ctx, dst, is32);
595	break;
596
597	/* dst = dst * imm */
598	case BPF_ALU | BPF_MUL | BPF_K:
599	case BPF_ALU64 | BPF_MUL | BPF_K:
600	move_imm(ctx, t1, imm, is32);
601	emit_insn(ctx, muld, dst, dst, t1);
602	emit_zext_32(ctx, dst, is32);
603	break;
604
605	/* dst = dst / src */
606	case BPF_ALU | BPF_DIV | BPF_X:
607	case BPF_ALU64 | BPF_DIV | BPF_X:
608	if (!off) {
609	emit_zext_32(ctx, dst, is32);
610	move_reg(ctx, t1, src);
611	emit_zext_32(ctx, t1, is32);
612	emit_insn(ctx, divdu, dst, dst, t1);
613	emit_zext_32(ctx, dst, is32);
614	} else {
615	emit_sext_32(ctx, dst, is32);
616	move_reg(ctx, t1, src);
617	emit_sext_32(ctx, t1, is32);
618	emit_insn(ctx, divd, dst, dst, t1);
619	emit_sext_32(ctx, dst, is32);
620	}
621	break;
622
623	/* dst = dst / imm */
624	case BPF_ALU | BPF_DIV | BPF_K:
625	case BPF_ALU64 | BPF_DIV | BPF_K:
626	if (!off) {
627	move_imm(ctx, t1, imm, is32);
628	emit_zext_32(ctx, dst, is32);
629	emit_insn(ctx, divdu, dst, dst, t1);
630	emit_zext_32(ctx, dst, is32);
631	} else {
632	move_imm(ctx, t1, imm, false);
633	emit_sext_32(ctx, t1, is32);
634	emit_sext_32(ctx, dst, is32);
635	emit_insn(ctx, divd, dst, dst, t1);
636	emit_sext_32(ctx, dst, is32);
637	}
638	break;
639
640	/* dst = dst % src */
641	case BPF_ALU | BPF_MOD | BPF_X:
642	case BPF_ALU64 | BPF_MOD | BPF_X:
643	if (!off) {
644	emit_zext_32(ctx, dst, is32);
645	move_reg(ctx, t1, src);
646	emit_zext_32(ctx, t1, is32);
647	emit_insn(ctx, moddu, dst, dst, t1);
648	emit_zext_32(ctx, dst, is32);
649	} else {
650	emit_sext_32(ctx, dst, is32);
651	move_reg(ctx, t1, src);
652	emit_sext_32(ctx, t1, is32);
653	emit_insn(ctx, modd, dst, dst, t1);
654	emit_sext_32(ctx, dst, is32);
655	}
656	break;
657
658	/* dst = dst % imm */
659	case BPF_ALU | BPF_MOD | BPF_K:
660	case BPF_ALU64 | BPF_MOD | BPF_K:
661	if (!off) {
662	move_imm(ctx, t1, imm, is32);
663	emit_zext_32(ctx, dst, is32);
664	emit_insn(ctx, moddu, dst, dst, t1);
665	emit_zext_32(ctx, dst, is32);
666	} else {
667	move_imm(ctx, t1, imm, false);
668	emit_sext_32(ctx, t1, is32);
669	emit_sext_32(ctx, dst, is32);
670	emit_insn(ctx, modd, dst, dst, t1);
671	emit_sext_32(ctx, dst, is32);
672	}
673	break;
674
675	/* dst = -dst */
676	case BPF_ALU | BPF_NEG:
677	case BPF_ALU64 | BPF_NEG:
678	move_imm(ctx, t1, imm, is32);
679	emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
680	emit_zext_32(ctx, dst, is32);
681	break;
682
683	/* dst = dst & src */
684	case BPF_ALU | BPF_AND | BPF_X:
685	case BPF_ALU64 | BPF_AND | BPF_X:
686	emit_insn(ctx, and, dst, dst, src);
687	emit_zext_32(ctx, dst, is32);
688	break;
689
690	/* dst = dst & imm */
691	case BPF_ALU | BPF_AND | BPF_K:
692	case BPF_ALU64 | BPF_AND | BPF_K:
693	if (is_unsigned_imm12(imm)) {
694	emit_insn(ctx, andi, dst, dst, imm);
695	} else {
696	move_imm(ctx, t1, imm, is32);
697	emit_insn(ctx, and, dst, dst, t1);
698	}
699	emit_zext_32(ctx, dst, is32);
700	break;
701
702	/* dst = dst | src */
703	case BPF_ALU | BPF_OR | BPF_X:
704	case BPF_ALU64 | BPF_OR | BPF_X:
705	emit_insn(ctx, or, dst, dst, src);
706	emit_zext_32(ctx, dst, is32);
707	break;
708
709	/* dst = dst | imm */
710	case BPF_ALU | BPF_OR | BPF_K:
711	case BPF_ALU64 | BPF_OR | BPF_K:
712	if (is_unsigned_imm12(imm)) {
713	emit_insn(ctx, ori, dst, dst, imm);
714	} else {
715	move_imm(ctx, t1, imm, is32);
716	emit_insn(ctx, or, dst, dst, t1);
717	}
718	emit_zext_32(ctx, dst, is32);
719	break;
720
721	/* dst = dst ^ src */
722	case BPF_ALU | BPF_XOR | BPF_X:
723	case BPF_ALU64 | BPF_XOR | BPF_X:
724	emit_insn(ctx, xor, dst, dst, src);
725	emit_zext_32(ctx, dst, is32);
726	break;
727
728	/* dst = dst ^ imm */
729	case BPF_ALU | BPF_XOR | BPF_K:
730	case BPF_ALU64 | BPF_XOR | BPF_K:
731	if (is_unsigned_imm12(imm)) {
732	emit_insn(ctx, xori, dst, dst, imm);
733	} else {
734	move_imm(ctx, t1, imm, is32);
735	emit_insn(ctx, xor, dst, dst, t1);
736	}
737	emit_zext_32(ctx, dst, is32);
738	break;
739
740	/* dst = dst << src (logical) */
741	case BPF_ALU | BPF_LSH | BPF_X:
742	emit_insn(ctx, sllw, dst, dst, src);
743	emit_zext_32(ctx, dst, is32);
744	break;
745
746	case BPF_ALU64 | BPF_LSH | BPF_X:
747	emit_insn(ctx, slld, dst, dst, src);
748	break;
749
750	/* dst = dst << imm (logical) */
751	case BPF_ALU | BPF_LSH | BPF_K:
752	emit_insn(ctx, slliw, dst, dst, imm);
753	emit_zext_32(ctx, dst, is32);
754	break;
755
756	case BPF_ALU64 | BPF_LSH | BPF_K:
757	emit_insn(ctx, sllid, dst, dst, imm);
758	break;
759
760	/* dst = dst >> src (logical) */
761	case BPF_ALU | BPF_RSH | BPF_X:
762	emit_insn(ctx, srlw, dst, dst, src);
763	emit_zext_32(ctx, dst, is32);
764	break;
765
766	case BPF_ALU64 | BPF_RSH | BPF_X:
767	emit_insn(ctx, srld, dst, dst, src);
768	break;
769
770	/* dst = dst >> imm (logical) */
771	case BPF_ALU | BPF_RSH | BPF_K:
772	emit_insn(ctx, srliw, dst, dst, imm);
773	emit_zext_32(ctx, dst, is32);
774	break;
775
776	case BPF_ALU64 | BPF_RSH | BPF_K:
777	emit_insn(ctx, srlid, dst, dst, imm);
778	break;
779
780	/* dst = dst >> src (arithmetic) */
781	case BPF_ALU | BPF_ARSH | BPF_X:
782	emit_insn(ctx, sraw, dst, dst, src);
783	emit_zext_32(ctx, dst, is32);
784	break;
785
786	case BPF_ALU64 | BPF_ARSH | BPF_X:
787	emit_insn(ctx, srad, dst, dst, src);
788	break;
789
790	/* dst = dst >> imm (arithmetic) */
791	case BPF_ALU | BPF_ARSH | BPF_K:
792	emit_insn(ctx, sraiw, dst, dst, imm);
793	emit_zext_32(ctx, dst, is32);
794	break;
795
796	case BPF_ALU64 | BPF_ARSH | BPF_K:
797	emit_insn(ctx, sraid, dst, dst, imm);
798	break;
799
800	/* dst = BSWAP##imm(dst) */
801	case BPF_ALU | BPF_END | BPF_FROM_LE:
802	switch (imm) {
803	case 16:
804	/* zero-extend 16 bits into 64 bits */
805	emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
806	break;
807	case 32:
808	/* zero-extend 32 bits into 64 bits */
809	emit_zext_32(ctx, dst, is32);
810	break;
811	case 64:
812	/* do nothing */
813	break;
814	}
815	break;
816
817	case BPF_ALU | BPF_END | BPF_FROM_BE:
818	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
819	switch (imm) {
820	case 16:
821	emit_insn(ctx, revb2h, dst, dst);
822	/* zero-extend 16 bits into 64 bits */
823	emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
824	break;
825	case 32:
826	emit_insn(ctx, revb2w, dst, dst);
827	/* clear the upper 32 bits */
828	emit_zext_32(ctx, dst, true);
829	break;
830	case 64:
831	emit_insn(ctx, revbd, dst, dst);
832	break;
833	}
834	break;
835
836	/* PC += off if dst cond src */
837	case BPF_JMP | BPF_JEQ | BPF_X:
838	case BPF_JMP | BPF_JNE | BPF_X:
839	case BPF_JMP | BPF_JGT | BPF_X:
840	case BPF_JMP | BPF_JGE | BPF_X:
841	case BPF_JMP | BPF_JLT | BPF_X:
842	case BPF_JMP | BPF_JLE | BPF_X:
843	case BPF_JMP | BPF_JSGT | BPF_X:
844	case BPF_JMP | BPF_JSGE | BPF_X:
845	case BPF_JMP | BPF_JSLT | BPF_X:
846	case BPF_JMP | BPF_JSLE | BPF_X:
847	case BPF_JMP32 | BPF_JEQ | BPF_X:
848	case BPF_JMP32 | BPF_JNE | BPF_X:
849	case BPF_JMP32 | BPF_JGT | BPF_X:
850	case BPF_JMP32 | BPF_JGE | BPF_X:
851	case BPF_JMP32 | BPF_JLT | BPF_X:
852	case BPF_JMP32 | BPF_JLE | BPF_X:
853	case BPF_JMP32 | BPF_JSGT | BPF_X:
854	case BPF_JMP32 | BPF_JSGE | BPF_X:
855	case BPF_JMP32 | BPF_JSLT | BPF_X:
856	case BPF_JMP32 | BPF_JSLE | BPF_X:
857	jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx);
858	move_reg(ctx, t1, dst);
859	move_reg(ctx, t2, src);
860	if (is_signed_bpf_cond(BPF_OP(code))) {
861	emit_sext_32(ctx, t1, is32);
862	emit_sext_32(ctx, t2, is32);
863	} else {
864	emit_zext_32(ctx, t1, is32);
865	emit_zext_32(ctx, t2, is32);
866	}
867	if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
868	goto toofar;
869	break;
870
871	/* PC += off if dst cond imm */
872	case BPF_JMP | BPF_JEQ | BPF_K:
873	case BPF_JMP | BPF_JNE | BPF_K:
874	case BPF_JMP | BPF_JGT | BPF_K:
875	case BPF_JMP | BPF_JGE | BPF_K:
876	case BPF_JMP | BPF_JLT | BPF_K:
877	case BPF_JMP | BPF_JLE | BPF_K:
878	case BPF_JMP | BPF_JSGT | BPF_K:
879	case BPF_JMP | BPF_JSGE | BPF_K:
880	case BPF_JMP | BPF_JSLT | BPF_K:
881	case BPF_JMP | BPF_JSLE | BPF_K:
882	case BPF_JMP32 | BPF_JEQ | BPF_K:
883	case BPF_JMP32 | BPF_JNE | BPF_K:
884	case BPF_JMP32 | BPF_JGT | BPF_K:
885	case BPF_JMP32 | BPF_JGE | BPF_K:
886	case BPF_JMP32 | BPF_JLT | BPF_K:
887	case BPF_JMP32 | BPF_JLE | BPF_K:
888	case BPF_JMP32 | BPF_JSGT | BPF_K:
889	case BPF_JMP32 | BPF_JSGE | BPF_K:
890	case BPF_JMP32 | BPF_JSLT | BPF_K:
891	case BPF_JMP32 | BPF_JSLE | BPF_K:
892	jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx);
893	if (imm) {
894	move_imm(ctx, t1, imm, false);
895	tm = t1;
896	} else {
897	/* If imm is 0, simply use zero register. */
898	tm = LOONGARCH_GPR_ZERO;
899	}
900	move_reg(ctx, t2, dst);
901	if (is_signed_bpf_cond(BPF_OP(code))) {
902	emit_sext_32(ctx, tm, is32);
903	emit_sext_32(ctx, t2, is32);
904	} else {
905	emit_zext_32(ctx, tm, is32);
906	emit_zext_32(ctx, t2, is32);
907	}
908	if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
909	goto toofar;
910	break;
911
912	/* PC += off if dst & src */
913	case BPF_JMP | BPF_JSET | BPF_X:
914	case BPF_JMP32 | BPF_JSET | BPF_X:
915	jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx);
916	emit_insn(ctx, and, t1, dst, src);
917	emit_zext_32(ctx, t1, is32);
918	if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
919	goto toofar;
920	break;
921
922	/* PC += off if dst & imm */
923	case BPF_JMP | BPF_JSET | BPF_K:
924	case BPF_JMP32 | BPF_JSET | BPF_K:
925	jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx);
926	move_imm(ctx, t1, imm, is32);
927	emit_insn(ctx, and, t1, dst, t1);
928	emit_zext_32(ctx, t1, is32);
929	if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
930	goto toofar;
931	break;
932
933	/* PC += off */
934	case BPF_JMP | BPF_JA:
935	case BPF_JMP32 | BPF_JA:
936	if (BPF_CLASS(code) == BPF_JMP)
937	jmp_offset = bpf2la_offset(bpf_insn: i, off, ctx);
938	else
939	jmp_offset = bpf2la_offset(bpf_insn: i, off: imm, ctx);
940	if (emit_uncond_jmp(ctx, jmp_offset) < 0)
941	goto toofar;
942	break;
943
944	/* function call */
945	case BPF_JMP | BPF_CALL:
946	ret = bpf_jit_get_func_addr(prog: ctx->prog, insn, extra_pass,
947	func_addr: &func_addr, func_addr_fixed: &func_addr_fixed);
948	if (ret < 0)
949	return ret;
950
951	if (insn->src_reg == BPF_PSEUDO_CALL) {
952	tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
953	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
954	}
955
956	if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
957	const struct btf_func_model *m;
958	int i;
959
960	m = bpf_jit_find_kfunc_model(prog: ctx->prog, insn);
961	if (!m)
962	return -EINVAL;
963
964	for (i = 0; i < m->nr_args; i++) {
965	u8 reg = regmap[BPF_REG_1 + i];
966	bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG;
967
968	emit_abi_ext(ctx, reg, size: m->arg_size[i], sign);
969	}
970	}
971
972	move_addr(ctx, t1, func_addr);
973	emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
974
975	if (insn->src_reg != BPF_PSEUDO_CALL)
976	move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
977
978	break;
979
980	/* tail call */
981	case BPF_JMP | BPF_TAIL_CALL:
982	if (emit_bpf_tail_call(ctx, insn: i) < 0)
983	return -EINVAL;
984	break;
985
986	/* function return */
987	case BPF_JMP | BPF_EXIT:
988	if (i == ctx->prog->len - 1)
989	break;
990
991	jmp_offset = epilogue_offset(ctx);
992	if (emit_uncond_jmp(ctx, jmp_offset) < 0)
993	goto toofar;
994	break;
995
996	/* dst = imm64 */
997	case BPF_LD | BPF_IMM | BPF_DW:
998	{
999	const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
1000
1001	if (bpf_pseudo_func(insn))
1002	move_addr(ctx, dst, imm64);
1003	else
1004	move_imm(ctx, dst, imm64, is32);
1005	return 1;
1006	}
1007
1008	/* dst = *(size *)(src + off) */
1009	case BPF_LDX | BPF_MEM | BPF_B:
1010	case BPF_LDX | BPF_MEM | BPF_H:
1011	case BPF_LDX | BPF_MEM | BPF_W:
1012	case BPF_LDX | BPF_MEM | BPF_DW:
1013	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1014	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1015	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1016	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1017	/* dst_reg = (s64)*(signed size *)(src_reg + off) */
1018	case BPF_LDX | BPF_MEMSX | BPF_B:
1019	case BPF_LDX | BPF_MEMSX | BPF_H:
1020	case BPF_LDX | BPF_MEMSX | BPF_W:
1021	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1022	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1023	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1024	sign_extend = BPF_MODE(insn->code) == BPF_MEMSX ||
1025	BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
1026	switch (BPF_SIZE(code)) {
1027	case BPF_B:
1028	if (is_signed_imm12(off)) {
1029	if (sign_extend)
1030	emit_insn(ctx, ldb, dst, src, off);
1031	else
1032	emit_insn(ctx, ldbu, dst, src, off);
1033	} else {
1034	move_imm(ctx, t1, off, is32);
1035	if (sign_extend)
1036	emit_insn(ctx, ldxb, dst, src, t1);
1037	else
1038	emit_insn(ctx, ldxbu, dst, src, t1);
1039	}
1040	break;
1041	case BPF_H:
1042	if (is_signed_imm12(off)) {
1043	if (sign_extend)
1044	emit_insn(ctx, ldh, dst, src, off);
1045	else
1046	emit_insn(ctx, ldhu, dst, src, off);
1047	} else {
1048	move_imm(ctx, t1, off, is32);
1049	if (sign_extend)
1050	emit_insn(ctx, ldxh, dst, src, t1);
1051	else
1052	emit_insn(ctx, ldxhu, dst, src, t1);
1053	}
1054	break;
1055	case BPF_W:
1056	if (is_signed_imm12(off)) {
1057	if (sign_extend)
1058	emit_insn(ctx, ldw, dst, src, off);
1059	else
1060	emit_insn(ctx, ldwu, dst, src, off);
1061	} else {
1062	move_imm(ctx, t1, off, is32);
1063	if (sign_extend)
1064	emit_insn(ctx, ldxw, dst, src, t1);
1065	else
1066	emit_insn(ctx, ldxwu, dst, src, t1);
1067	}
1068	break;
1069	case BPF_DW:
1070	move_imm(ctx, t1, off, is32);
1071	emit_insn(ctx, ldxd, dst, src, t1);
1072	break;
1073	}
1074
1075	ret = add_exception_handler(insn, ctx, dst_reg: dst);
1076	if (ret)
1077	return ret;
1078	break;
1079
1080	/* *(size *)(dst + off) = imm */
1081	case BPF_ST | BPF_MEM | BPF_B:
1082	case BPF_ST | BPF_MEM | BPF_H:
1083	case BPF_ST | BPF_MEM | BPF_W:
1084	case BPF_ST | BPF_MEM | BPF_DW:
1085	switch (BPF_SIZE(code)) {
1086	case BPF_B:
1087	move_imm(ctx, t1, imm, is32);
1088	if (is_signed_imm12(off)) {
1089	emit_insn(ctx, stb, t1, dst, off);
1090	} else {
1091	move_imm(ctx, t2, off, is32);
1092	emit_insn(ctx, stxb, t1, dst, t2);
1093	}
1094	break;
1095	case BPF_H:
1096	move_imm(ctx, t1, imm, is32);
1097	if (is_signed_imm12(off)) {
1098	emit_insn(ctx, sth, t1, dst, off);
1099	} else {
1100	move_imm(ctx, t2, off, is32);
1101	emit_insn(ctx, stxh, t1, dst, t2);
1102	}
1103	break;
1104	case BPF_W:
1105	move_imm(ctx, t1, imm, is32);
1106	if (is_signed_imm12(off)) {
1107	emit_insn(ctx, stw, t1, dst, off);
1108	} else if (is_signed_imm14(off)) {
1109	emit_insn(ctx, stptrw, t1, dst, off);
1110	} else {
1111	move_imm(ctx, t2, off, is32);
1112	emit_insn(ctx, stxw, t1, dst, t2);
1113	}
1114	break;
1115	case BPF_DW:
1116	move_imm(ctx, t1, imm, is32);
1117	if (is_signed_imm12(off)) {
1118	emit_insn(ctx, std, t1, dst, off);
1119	} else if (is_signed_imm14(off)) {
1120	emit_insn(ctx, stptrd, t1, dst, off);
1121	} else {
1122	move_imm(ctx, t2, off, is32);
1123	emit_insn(ctx, stxd, t1, dst, t2);
1124	}
1125	break;
1126	}
1127	break;
1128
1129	/* *(size *)(dst + off) = src */
1130	case BPF_STX | BPF_MEM | BPF_B:
1131	case BPF_STX | BPF_MEM | BPF_H:
1132	case BPF_STX | BPF_MEM | BPF_W:
1133	case BPF_STX | BPF_MEM | BPF_DW:
1134	switch (BPF_SIZE(code)) {
1135	case BPF_B:
1136	if (is_signed_imm12(off)) {
1137	emit_insn(ctx, stb, src, dst, off);
1138	} else {
1139	move_imm(ctx, t1, off, is32);
1140	emit_insn(ctx, stxb, src, dst, t1);
1141	}
1142	break;
1143	case BPF_H:
1144	if (is_signed_imm12(off)) {
1145	emit_insn(ctx, sth, src, dst, off);
1146	} else {
1147	move_imm(ctx, t1, off, is32);
1148	emit_insn(ctx, stxh, src, dst, t1);
1149	}
1150	break;
1151	case BPF_W:
1152	if (is_signed_imm12(off)) {
1153	emit_insn(ctx, stw, src, dst, off);
1154	} else if (is_signed_imm14(off)) {
1155	emit_insn(ctx, stptrw, src, dst, off);
1156	} else {
1157	move_imm(ctx, t1, off, is32);
1158	emit_insn(ctx, stxw, src, dst, t1);
1159	}
1160	break;
1161	case BPF_DW:
1162	if (is_signed_imm12(off)) {
1163	emit_insn(ctx, std, src, dst, off);
1164	} else if (is_signed_imm14(off)) {
1165	emit_insn(ctx, stptrd, src, dst, off);
1166	} else {
1167	move_imm(ctx, t1, off, is32);
1168	emit_insn(ctx, stxd, src, dst, t1);
1169	}
1170	break;
1171	}
1172	break;
1173
1174	case BPF_STX | BPF_ATOMIC | BPF_W:
1175	case BPF_STX | BPF_ATOMIC | BPF_DW:
1176	emit_atomic(insn, ctx);
1177	break;
1178
1179	/* Speculation barrier */
1180	case BPF_ST | BPF_NOSPEC:
1181	break;
1182
1183	default:
1184	pr_err("bpf_jit: unknown opcode %02x\n", code);
1185	return -EINVAL;
1186	}
1187
1188	return 0;
1189
1190	toofar:
1191	pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
1192	return -E2BIG;
1193	}
1194
1195	static int build_body(struct jit_ctx *ctx, bool extra_pass)
1196	{
1197	int i;
1198	const struct bpf_prog *prog = ctx->prog;
1199
1200	for (i = 0; i < prog->len; i++) {
1201	const struct bpf_insn *insn = &prog->insnsi[i];
1202	int ret;
1203
1204	if (ctx->image == NULL)
1205	ctx->offset[i] = ctx->idx;
1206
1207	ret = build_insn(insn, ctx, extra_pass);
1208	if (ret > 0) {
1209	i++;
1210	if (ctx->image == NULL)
1211	ctx->offset[i] = ctx->idx;
1212	continue;
1213	}
1214	if (ret)
1215	return ret;
1216	}
1217
1218	if (ctx->image == NULL)
1219	ctx->offset[i] = ctx->idx;
1220
1221	return 0;
1222	}
1223
1224	/* Fill space with break instructions */
1225	static void jit_fill_hole(void *area, unsigned int size)
1226	{
1227	u32 *ptr;
1228
1229	/* We are guaranteed to have aligned memory */
1230	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1231	*ptr++ = INSN_BREAK;
1232	}
1233
1234	static int validate_code(struct jit_ctx *ctx)
1235	{
1236	int i;
1237	union loongarch_instruction insn;
1238
1239	for (i = 0; i < ctx->idx; i++) {
1240	insn = ctx->image[i];
1241	/* Check INSN_BREAK */
1242	if (insn.word == INSN_BREAK)
1243	return -1;
1244	}
1245
1246	return 0;
1247	}
1248
1249	static int validate_ctx(struct jit_ctx *ctx)
1250	{
1251	if (validate_code(ctx))
1252	return -1;
1253
1254	if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1255	return -1;
1256
1257	return 0;
1258	}
1259
1260	static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target)
1261	{
1262	if (!target) {
1263	pr_err("bpf_jit: jump target address is error\n");
1264	return -EFAULT;
1265	}
1266
1267	move_imm(ctx, LOONGARCH_GPR_T1, target, false);
1268	emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
1269
1270	return 0;
1271	}
1272
1273	static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
1274	{
1275	int i;
1276	struct jit_ctx ctx;
1277
1278	ctx.idx = 0;
1279	ctx.image = (union loongarch_instruction *)insns;
1280
1281	if (!target) {
1282	for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
1283	emit_insn((&ctx), nop);
1284	return 0;
1285	}
1286
1287	return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target);
1288	}
1289
1290	static int emit_call(struct jit_ctx *ctx, u64 addr)
1291	{
1292	return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr);
1293	}
1294
1295	void *bpf_arch_text_copy(void *dst, void *src, size_t len)
1296	{
1297	int ret;
1298
1299	mutex_lock(&text_mutex);
1300	ret = larch_insn_text_copy(dst, src, len);
1301	mutex_unlock(lock: &text_mutex);
1302
1303	return ret ? ERR_PTR(error: -EINVAL) : dst;
1304	}
1305
1306	int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
1307	enum bpf_text_poke_type new_t, void *old_addr,
1308	void *new_addr)
1309	{
1310	int ret;
1311	bool is_call;
1312	unsigned long size = 0;
1313	unsigned long offset = 0;
1314	void *image = NULL;
1315	char namebuf[KSYM_NAME_LEN];
1316	u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1317	u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1318
1319	/* Only poking bpf text is supported. Since kernel function entry
1320	* is set up by ftrace, we rely on ftrace to poke kernel functions.
1321	*/
1322	if (!__bpf_address_lookup(addr: (unsigned long)ip, size: &size, off: &offset, sym: namebuf))
1323	return -ENOTSUPP;
1324
1325	image = ip - offset;
1326
1327	/* zero offset means we're poking bpf prog entry */
1328	if (offset == 0) {
1329	/* skip to the nop instruction in bpf prog entry:
1330	* move t0, ra
1331	* nop
1332	*/
1333	ip = image + LOONGARCH_INSN_SIZE;
1334	}
1335
1336	is_call = old_t == BPF_MOD_CALL;
1337	ret = emit_jump_or_nops(target: old_addr, ip, insns: old_insns, is_call);
1338	if (ret)
1339	return ret;
1340
1341	if (memcmp(p: ip, q: old_insns, LOONGARCH_LONG_JUMP_NBYTES))
1342	return -EFAULT;
1343
1344	is_call = new_t == BPF_MOD_CALL;
1345	ret = emit_jump_or_nops(target: new_addr, ip, insns: new_insns, is_call);
1346	if (ret)
1347	return ret;
1348
1349	mutex_lock(&text_mutex);
1350	if (memcmp(p: ip, q: new_insns, LOONGARCH_LONG_JUMP_NBYTES))
1351	ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES);
1352	mutex_unlock(lock: &text_mutex);
1353
1354	return ret;
1355	}
1356
1357	int bpf_arch_text_invalidate(void *dst, size_t len)
1358	{
1359	int i;
1360	int ret = 0;
1361	u32 *inst;
1362
1363	inst = kvmalloc(len, GFP_KERNEL);
1364	if (!inst)
1365	return -ENOMEM;
1366
1367	for (i = 0; i < (len / sizeof(u32)); i++)
1368	inst[i] = INSN_BREAK;
1369
1370	mutex_lock(&text_mutex);
1371	if (larch_insn_text_copy(dst, inst, len))
1372	ret = -EINVAL;
1373	mutex_unlock(lock: &text_mutex);
1374
1375	kvfree(addr: inst);
1376
1377	return ret;
1378	}
1379
1380	static void store_args(struct jit_ctx *ctx, int nargs, int args_off)
1381	{
1382	int i;
1383
1384	for (i = 0; i < nargs; i++) {
1385	emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1386	args_off -= 8;
1387	}
1388	}
1389
1390	static void restore_args(struct jit_ctx *ctx, int nargs, int args_off)
1391	{
1392	int i;
1393
1394	for (i = 0; i < nargs; i++) {
1395	emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1396	args_off -= 8;
1397	}
1398	}
1399
1400	static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
1401	int args_off, int retval_off, int run_ctx_off, bool save_ret)
1402	{
1403	int ret;
1404	u32 *branch;
1405	struct bpf_prog *p = l->link.prog;
1406	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
1407
1408	if (l->cookie) {
1409	move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false);
1410	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
1411	} else {
1412	emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
1413	}
1414
1415	/* arg1: prog */
1416	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1417	/* arg2: &run_ctx */
1418	emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
1419	ret = emit_call(ctx, addr: (const u64)bpf_trampoline_enter(prog: p));
1420	if (ret)
1421	return ret;
1422
1423	/* store prog start time */
1424	move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
1425
1426	/*
1427	* if (__bpf_prog_enter(prog) == 0)
1428	* goto skip_exec_of_prog;
1429	*/
1430	branch = (u32 *)ctx->image + ctx->idx;
1431	/* nop reserved for conditional jump */
1432	emit_insn(ctx, nop);
1433
1434	/* arg1: &args_off */
1435	emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
1436	if (!p->jited)
1437	move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
1438	ret = emit_call(ctx, addr: (const u64)p->bpf_func);
1439	if (ret)
1440	return ret;
1441
1442	if (save_ret) {
1443	emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1444	emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1445	}
1446
1447	/* update branch with beqz */
1448	if (ctx->image) {
1449	int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch;
1450	*branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
1451	}
1452
1453	/* arg1: prog */
1454	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1455	/* arg2: prog start time */
1456	move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
1457	/* arg3: &run_ctx */
1458	emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
1459	ret = emit_call(ctx, addr: (const u64)bpf_trampoline_exit(prog: p));
1460
1461	return ret;
1462	}
1463
1464	static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
1465	int args_off, int retval_off, int run_ctx_off, u32 **branches)
1466	{
1467	int i;
1468
1469	emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
1470	for (i = 0; i < tl->nr_links; i++) {
1471	invoke_bpf_prog(ctx, l: tl->links[i], args_off, retval_off, run_ctx_off, save_ret: true);
1472	emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
1473	branches[i] = (u32 *)ctx->image + ctx->idx;
1474	emit_insn(ctx, nop);
1475	}
1476	}
1477
1478	void *arch_alloc_bpf_trampoline(unsigned int size)
1479	{
1480	return bpf_prog_pack_alloc(size, bpf_fill_ill_insns: jit_fill_hole);
1481	}
1482
1483	void arch_free_bpf_trampoline(void *image, unsigned int size)
1484	{
1485	bpf_prog_pack_free(ptr: image, size);
1486	}
1487
1488	/*
1489	* Sign-extend the register if necessary
1490	*/
1491	static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign)
1492	{
1493	/* ABI requires unsigned char/short to be zero-extended */
1494	if (!sign && (size == 1 || size == 2)) {
1495	if (rd != rj)
1496	move_reg(ctx, rd, rj);
1497	return;
1498	}
1499
1500	switch (size) {
1501	case 1:
1502	emit_insn(ctx, extwb, rd, rj);
1503	break;
1504	case 2:
1505	emit_insn(ctx, extwh, rd, rj);
1506	break;
1507	case 4:
1508	emit_insn(ctx, addiw, rd, rj, 0);
1509	break;
1510	case 8:
1511	if (rd != rj)
1512	move_reg(ctx, rd, rj);
1513	break;
1514	default:
1515	pr_warn("bpf_jit: invalid size %d for sign_extend\n", size);
1516	}
1517	}
1518
1519	static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
1520	const struct btf_func_model *m, struct bpf_tramp_links *tlinks,
1521	void *func_addr, u32 flags)
1522	{
1523	int i, ret, save_ret;
1524	int stack_size, nargs;
1525	int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;
1526	bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
1527	void *orig_call = func_addr;
1528	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
1529	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
1530	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
1531	u32 **branches = NULL;
1532
1533	/*
1534	* FP + 8 [ RA to parent func ] return address to parent
1535	* function
1536	* FP + 0 [ FP of parent func ] frame pointer of parent
1537	* function
1538	* FP - 8 [ T0 to traced func ] return address of traced
1539	* function
1540	* FP - 16 [ FP of traced func ] frame pointer of traced
1541	* function
1542	*
1543	* FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
1544	* BPF_TRAMP_F_RET_FENTRY_RET
1545	* [ argN ]
1546	* [ ... ]
1547	* FP - args_off [ arg1 ]
1548	*
1549	* FP - nargs_off [ regs count ]
1550	*
1551	* FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG
1552	*
1553	* FP - run_ctx_off [ bpf_tramp_run_ctx ]
1554	*
1555	* FP - sreg_off [ callee saved reg ]
1556	*
1557	* FP - tcc_ptr_off [ tail_call_cnt_ptr ]
1558	*/
1559
1560	if (m->nr_args > LOONGARCH_MAX_REG_ARGS)
1561	return -ENOTSUPP;
1562
1563	/* FIXME: No support of struct argument */
1564	for (i = 0; i < m->nr_args; i++) {
1565	if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
1566	return -ENOTSUPP;
1567	}
1568
1569	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
1570	return -ENOTSUPP;
1571
1572	/* Room of trampoline frame to store return address and frame pointer */
1573	stack_size = 16;
1574
1575	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
1576	if (save_ret)
1577	stack_size += 16; /* Save BPF R0 and A0 */
1578
1579	retval_off = stack_size;
1580
1581	/* Room of trampoline frame to store args */
1582	nargs = m->nr_args;
1583	stack_size += nargs * 8;
1584	args_off = stack_size;
1585
1586	/* Room of trampoline frame to store args number */
1587	stack_size += 8;
1588	nargs_off = stack_size;
1589
1590	/* Room of trampoline frame to store ip address */
1591	if (flags & BPF_TRAMP_F_IP_ARG) {
1592	stack_size += 8;
1593	ip_off = stack_size;
1594	}
1595
1596	/* Room of trampoline frame to store struct bpf_tramp_run_ctx */
1597	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
1598	run_ctx_off = stack_size;
1599
1600	stack_size += 8;
1601	sreg_off = stack_size;
1602
1603	/* Room of trampoline frame to store tail_call_cnt_ptr */
1604	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
1605	stack_size += 8;
1606	tcc_ptr_off = stack_size;
1607	}
1608
1609	stack_size = round_up(stack_size, 16);
1610
1611	if (is_struct_ops) {
1612	/*
1613	* For the trampoline called directly, just handle
1614	* the frame of trampoline.
1615	*/
1616	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1617	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
1618	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1619	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1620	} else {
1621	/*
1622	* For the trampoline called from function entry,
1623	* the frame of traced function and the frame of
1624	* trampoline need to be considered.
1625	*/
1626	/* RA and FP for parent function */
1627	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
1628	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
1629	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
1630	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
1631
1632	/* RA and FP for traced function */
1633	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1634	emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
1635	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1636	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1637	}
1638
1639	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1640	emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1641
1642	/* callee saved register S1 to pass start time */
1643	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
1644
1645	/* store ip address of the traced function */
1646	if (flags & BPF_TRAMP_F_IP_ARG) {
1647	move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false);
1648	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off);
1649	}
1650
1651	/* store nargs number */
1652	move_imm(ctx, LOONGARCH_GPR_T1, nargs, false);
1653	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off);
1654
1655	store_args(ctx, nargs, args_off);
1656
1657	/* To traced function */
1658	/* Ftrace jump skips 2 NOP instructions */
1659	if (is_kernel_text(addr: (unsigned long)orig_call) ||
1660	is_module_text_address(addr: (unsigned long)orig_call))
1661	orig_call += LOONGARCH_FENTRY_NBYTES;
1662	/* Direct jump skips 5 NOP instructions */
1663	else if (is_bpf_text_address(addr: (unsigned long)orig_call))
1664	orig_call += LOONGARCH_BPF_FENTRY_NBYTES;
1665
1666	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1667	move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
1668	ret = emit_call(ctx, addr: (const u64)__bpf_tramp_enter);
1669	if (ret)
1670	return ret;
1671	}
1672
1673	for (i = 0; i < fentry->nr_links; i++) {
1674	ret = invoke_bpf_prog(ctx, l: fentry->links[i], args_off, retval_off,
1675	run_ctx_off, save_ret: flags & BPF_TRAMP_F_RET_FENTRY_RET);
1676	if (ret)
1677	return ret;
1678	}
1679	if (fmod_ret->nr_links) {
1680	branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL);
1681	if (!branches)
1682	return -ENOMEM;
1683
1684	invoke_bpf_mod_ret(ctx, tl: fmod_ret, args_off, retval_off, run_ctx_off, branches);
1685	}
1686
1687	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1688	restore_args(ctx, nargs: m->nr_args, args_off);
1689
1690	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1691	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1692
1693	ret = emit_call(ctx, addr: (const u64)orig_call);
1694	if (ret)
1695	goto out;
1696	emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1697	emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1698	im->ip_after_call = ctx->ro_image + ctx->idx;
1699	/* Reserve space for the move_imm + jirl instruction */
1700	for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
1701	emit_insn(ctx, nop);
1702	}
1703
1704	for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) {
1705	int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i];
1706	*branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
1707	}
1708
1709	for (i = 0; i < fexit->nr_links; i++) {
1710	ret = invoke_bpf_prog(ctx, l: fexit->links[i], args_off, retval_off, run_ctx_off, save_ret: false);
1711	if (ret)
1712	goto out;
1713	}
1714
1715	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1716	im->ip_epilogue = ctx->ro_image + ctx->idx;
1717	move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
1718	ret = emit_call(ctx, addr: (const u64)__bpf_tramp_exit);
1719	if (ret)
1720	goto out;
1721	}
1722
1723	if (flags & BPF_TRAMP_F_RESTORE_REGS)
1724	restore_args(ctx, nargs: m->nr_args, args_off);
1725
1726	if (save_ret) {
1727	emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1728	if (is_struct_ops)
1729	sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0],
1730	m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG);
1731	else
1732	emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1733	}
1734
1735	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
1736
1737	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1738	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1739
1740	if (is_struct_ops) {
1741	/* trampoline called directly */
1742	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
1743	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1744	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
1745
1746	emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
1747	} else {
1748	/* trampoline called from function entry */
1749	emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
1750	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1751	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
1752
1753	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
1754	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
1755	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
1756
1757	if (flags & BPF_TRAMP_F_SKIP_FRAME) {
1758	/* return to parent function */
1759	move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
1760	emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
1761	} else {
1762	/* return to traced function */
1763	move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA);
1764	move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
1765	emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0);
1766	}
1767	}
1768
1769	ret = ctx->idx;
1770	out:
1771	kfree(objp: branches);
1772
1773	return ret;
1774	}
1775
1776	int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
1777	void *ro_image_end, const struct btf_func_model *m,
1778	u32 flags, struct bpf_tramp_links *tlinks, void *func_addr)
1779	{
1780	int ret, size;
1781	void *image, *tmp;
1782	struct jit_ctx ctx;
1783
1784	size = ro_image_end - ro_image;
1785	image = kvmalloc(size, GFP_KERNEL);
1786	if (!image)
1787	return -ENOMEM;
1788
1789	ctx.image = (union loongarch_instruction *)image;
1790	ctx.ro_image = (union loongarch_instruction *)ro_image;
1791	ctx.idx = 0;
1792
1793	jit_fill_hole(area: image, size: (unsigned int)(ro_image_end - ro_image));
1794	ret = __arch_prepare_bpf_trampoline(ctx: &ctx, im, m, tlinks, func_addr, flags);
1795	if (ret < 0)
1796	goto out;
1797
1798	if (validate_code(ctx: &ctx) < 0) {
1799	ret = -EINVAL;
1800	goto out;
1801	}
1802
1803	tmp = bpf_arch_text_copy(dst: ro_image, src: image, len: size);
1804	if (IS_ERR(ptr: tmp)) {
1805	ret = PTR_ERR(ptr: tmp);
1806	goto out;
1807	}
1808
1809	out:
1810	kvfree(addr: image);
1811	return ret < 0 ? ret : size;
1812	}
1813
1814	int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1815	struct bpf_tramp_links *tlinks, void *func_addr)
1816	{
1817	int ret;
1818	struct jit_ctx ctx;
1819	struct bpf_tramp_image im;
1820
1821	ctx.image = NULL;
1822	ctx.idx = 0;
1823
1824	ret = __arch_prepare_bpf_trampoline(ctx: &ctx, im: &im, m, tlinks, func_addr, flags);
1825
1826	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
1827	}
1828
1829	struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1830	{
1831	bool tmp_blinded = false, extra_pass = false;
1832	u8 *image_ptr;
1833	int image_size, prog_size, extable_size;
1834	struct jit_ctx ctx;
1835	struct jit_data *jit_data;
1836	struct bpf_binary_header *header;
1837	struct bpf_prog *tmp, *orig_prog = prog;
1838
1839	/*
1840	* If BPF JIT was not enabled then we must fall back to
1841	* the interpreter.
1842	*/
1843	if (!prog->jit_requested)
1844	return orig_prog;
1845
1846	tmp = bpf_jit_blind_constants(fp: prog);
1847	/*
1848	* If blinding was requested and we failed during blinding,
1849	* we must fall back to the interpreter. Otherwise, we save
1850	* the new JITed code.
1851	*/
1852	if (IS_ERR(ptr: tmp))
1853	return orig_prog;
1854
1855	if (tmp != prog) {
1856	tmp_blinded = true;
1857	prog = tmp;
1858	}
1859
1860	jit_data = prog->aux->jit_data;
1861	if (!jit_data) {
1862	jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1863	if (!jit_data) {
1864	prog = orig_prog;
1865	goto out;
1866	}
1867	prog->aux->jit_data = jit_data;
1868	}
1869	if (jit_data->ctx.offset) {
1870	ctx = jit_data->ctx;
1871	image_ptr = jit_data->image;
1872	header = jit_data->header;
1873	extra_pass = true;
1874	prog_size = sizeof(u32) * ctx.idx;
1875	goto skip_init_ctx;
1876	}
1877
1878	memset(&ctx, 0, sizeof(ctx));
1879	ctx.prog = prog;
1880
1881	ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1882	if (ctx.offset == NULL) {
1883	prog = orig_prog;
1884	goto out_offset;
1885	}
1886
1887	/* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1888	build_prologue(ctx: &ctx);
1889	if (build_body(ctx: &ctx, extra_pass)) {
1890	prog = orig_prog;
1891	goto out_offset;
1892	}
1893	ctx.epilogue_offset = ctx.idx;
1894	build_epilogue(ctx: &ctx);
1895
1896	extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
1897
1898	/* Now we know the actual image size.
1899	* As each LoongArch instruction is of length 32bit,
1900	* we are translating number of JITed intructions into
1901	* the size required to store these JITed code.
1902	*/
1903	prog_size = sizeof(u32) * ctx.idx;
1904	image_size = prog_size + extable_size;
1905	/* Now we know the size of the structure to make */
1906	header = bpf_jit_binary_alloc(proglen: image_size, image_ptr: &image_ptr,
1907	alignment: sizeof(u32), bpf_fill_ill_insns: jit_fill_hole);
1908	if (header == NULL) {
1909	prog = orig_prog;
1910	goto out_offset;
1911	}
1912
1913	/* 2. Now, the actual pass to generate final JIT code */
1914	ctx.image = (union loongarch_instruction *)image_ptr;
1915	if (extable_size)
1916	prog->aux->extable = (void *)image_ptr + prog_size;
1917
1918	skip_init_ctx:
1919	ctx.idx = 0;
1920	ctx.num_exentries = 0;
1921
1922	build_prologue(ctx: &ctx);
1923	if (build_body(ctx: &ctx, extra_pass)) {
1924	bpf_jit_binary_free(hdr: header);
1925	prog = orig_prog;
1926	goto out_offset;
1927	}
1928	build_epilogue(ctx: &ctx);
1929
1930	/* 3. Extra pass to validate JITed code */
1931	if (validate_ctx(ctx: &ctx)) {
1932	bpf_jit_binary_free(hdr: header);
1933	prog = orig_prog;
1934	goto out_offset;
1935	}
1936
1937	/* And we're done */
1938	if (bpf_jit_enable > 1)
1939	bpf_jit_dump(flen: prog->len, proglen: prog_size, pass: 2, image: ctx.image);
1940
1941	/* Update the icache */
1942	flush_icache_range(start: (unsigned long)header, end: (unsigned long)(ctx.image + ctx.idx));
1943
1944	if (!prog->is_func || extra_pass) {
1945	int err;
1946
1947	if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1948	pr_err_once("multi-func JIT bug %d != %d\n",
1949	ctx.idx, jit_data->ctx.idx);
1950	goto out_free;
1951	}
1952	err = bpf_jit_binary_lock_ro(hdr: header);
1953	if (err) {
1954	pr_err_once("bpf_jit_binary_lock_ro() returned %d\n",
1955	err);
1956	goto out_free;
1957	}
1958	} else {
1959	jit_data->ctx = ctx;
1960	jit_data->image = image_ptr;
1961	jit_data->header = header;
1962	}
1963	prog->jited = 1;
1964	prog->jited_len = prog_size;
1965	prog->bpf_func = (void *)ctx.image;
1966
1967	if (!prog->is_func || extra_pass) {
1968	int i;
1969
1970	/* offset[prog->len] is the size of program */
1971	for (i = 0; i <= prog->len; i++)
1972	ctx.offset[i] *= LOONGARCH_INSN_SIZE;
1973	bpf_prog_fill_jited_linfo(prog, insn_to_jit_off: ctx.offset + 1);
1974
1975	out_offset:
1976	kvfree(addr: ctx.offset);
1977	kfree(objp: jit_data);
1978	prog->aux->jit_data = NULL;
1979	}
1980
1981	out:
1982	if (tmp_blinded)
1983	bpf_jit_prog_release_other(fp: prog, fp_other: prog == orig_prog ? tmp : orig_prog);
1984
1985
1986	return prog;
1987
1988	out_free:
1989	bpf_jit_binary_free(hdr: header);
1990	prog->bpf_func = NULL;
1991	prog->jited = 0;
1992	prog->jited_len = 0;
1993	goto out_offset;
1994	}
1995
1996	bool bpf_jit_bypass_spec_v1(void)
1997	{
1998	return true;
1999	}
2000
2001	bool bpf_jit_bypass_spec_v4(void)
2002	{
2003	return true;
2004	}
2005
2006	/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
2007	bool bpf_jit_supports_subprog_tailcalls(void)
2008	{
2009	return true;
2010	}
2011