1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* BPF JIT compiler for ARM64
4	*
5	* Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
6	*/
7
8	#define pr_fmt(fmt) "bpf_jit: " fmt
9
10	#include <linux/arm-smccc.h>
11	#include <linux/bitfield.h>
12	#include <linux/bpf.h>
13	#include <linux/cfi.h>
14	#include <linux/filter.h>
15	#include <linux/memory.h>
16	#include <linux/printk.h>
17	#include <linux/slab.h>
18
19	#include <asm/asm-extable.h>
20	#include <asm/byteorder.h>
21	#include <asm/cacheflush.h>
22	#include <asm/cpufeature.h>
23	#include <asm/debug-monitors.h>
24	#include <asm/insn.h>
25	#include <asm/text-patching.h>
26	#include <asm/set_memory.h>
27
28	#include "bpf_jit.h"
29
30	#define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
31	#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
32	#define TCCNT_PTR (MAX_BPF_JIT_REG + 2)
33	#define TMP_REG_3 (MAX_BPF_JIT_REG + 3)
34	#define PRIVATE_SP (MAX_BPF_JIT_REG + 4)
35	#define ARENA_VM_START (MAX_BPF_JIT_REG + 5)
36
37	#define check_imm(bits, imm) do { \
38	if ((((imm) > 0) && ((imm) >> (bits))) || \
39	(((imm) < 0) && (~(imm) >> (bits)))) { \
40	pr_info("[%2d] imm=%d(0x%x) out of range\n", \
41	i, imm, imm); \
42	return -EINVAL; \
43	} \
44	} while (0)
45	#define check_imm19(imm) check_imm(19, imm)
46	#define check_imm26(imm) check_imm(26, imm)
47
48	/* Map BPF registers to A64 registers */
49	static const int bpf2a64[] = {
50	/* return value from in-kernel function, and exit value from eBPF */
51	[BPF_REG_0] = A64_R(7),
52	/* arguments from eBPF program to in-kernel function */
53	[BPF_REG_1] = A64_R(0),
54	[BPF_REG_2] = A64_R(1),
55	[BPF_REG_3] = A64_R(2),
56	[BPF_REG_4] = A64_R(3),
57	[BPF_REG_5] = A64_R(4),
58	/* callee saved registers that in-kernel function will preserve */
59	[BPF_REG_6] = A64_R(19),
60	[BPF_REG_7] = A64_R(20),
61	[BPF_REG_8] = A64_R(21),
62	[BPF_REG_9] = A64_R(22),
63	/* read-only frame pointer to access stack */
64	[BPF_REG_FP] = A64_R(25),
65	/* temporary registers for BPF JIT */
66	[TMP_REG_1] = A64_R(10),
67	[TMP_REG_2] = A64_R(11),
68	[TMP_REG_3] = A64_R(12),
69	/* tail_call_cnt_ptr */
70	[TCCNT_PTR] = A64_R(26),
71	/* temporary register for blinding constants */
72	[BPF_REG_AX] = A64_R(9),
73	/* callee saved register for private stack pointer */
74	[PRIVATE_SP] = A64_R(27),
75	/* callee saved register for kern_vm_start address */
76	[ARENA_VM_START] = A64_R(28),
77	};
78
79	struct jit_ctx {
80	const struct bpf_prog *prog;
81	int idx;
82	int epilogue_offset;
83	int *offset;
84	int exentry_idx;
85	int nr_used_callee_reg;
86	u8 used_callee_reg[8]; /* r6~r9, fp, arena_vm_start */
87	__le32 *image;
88	__le32 *ro_image;
89	u32 stack_size;
90	u64 user_vm_start;
91	u64 arena_vm_start;
92	bool fp_used;
93	bool priv_sp_used;
94	bool write;
95	};
96
97	struct bpf_plt {
98	u32 insn_ldr; /* load target */
99	u32 insn_br; /* branch to target */
100	u64 target; /* target value */
101	};
102
103	#define PLT_TARGET_SIZE sizeof_field(struct bpf_plt, target)
104	#define PLT_TARGET_OFFSET offsetof(struct bpf_plt, target)
105
106	/* Memory size/value to protect private stack overflow/underflow */
107	#define PRIV_STACK_GUARD_SZ 16
108	#define PRIV_STACK_GUARD_VAL 0xEB9F12345678eb9fULL
109
110	static inline void emit(const u32 insn, struct jit_ctx *ctx)
111	{
112	if (ctx->image != NULL && ctx->write)
113	ctx->image[ctx->idx] = cpu_to_le32(insn);
114
115	ctx->idx++;
116	}
117
118	static inline void emit_u32_data(const u32 data, struct jit_ctx *ctx)
119	{
120	if (ctx->image != NULL && ctx->write)
121	ctx->image[ctx->idx] = data;
122
123	ctx->idx++;
124	}
125
126	static inline void emit_a64_mov_i(const int is64, const int reg,
127	const s32 val, struct jit_ctx *ctx)
128	{
129	u16 hi = val >> 16;
130	u16 lo = val & 0xffff;
131
132	if (hi & 0x8000) {
133	if (hi == 0xffff) {
134	emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx);
135	} else {
136	emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx);
137	if (lo != 0xffff)
138	emit(A64_MOVK(is64, reg, lo, 0), ctx);
139	}
140	} else {
141	emit(A64_MOVZ(is64, reg, lo, 0), ctx);
142	if (hi)
143	emit(A64_MOVK(is64, reg, hi, 16), ctx);
144	}
145	}
146
147	static int i64_i16_blocks(const u64 val, bool inverse)
148	{
149	return (((val >> 0) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
150	(((val >> 16) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
151	(((val >> 32) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
152	(((val >> 48) & 0xffff) != (inverse ? 0xffff : 0x0000));
153	}
154
155	static inline void emit_a64_mov_i64(const int reg, const u64 val,
156	struct jit_ctx *ctx)
157	{
158	u64 nrm_tmp = val, rev_tmp = ~val;
159	bool inverse;
160	int shift;
161
162	if (!(nrm_tmp >> 32))
163	return emit_a64_mov_i(is64: 0, reg, val: (u32)val, ctx);
164
165	inverse = i64_i16_blocks(val: nrm_tmp, inverse: true) < i64_i16_blocks(val: nrm_tmp, inverse: false);
166	shift = max(round_down((inverse ? (fls64(rev_tmp) - 1) :
167	(fls64(nrm_tmp) - 1)), 16), 0);
168	if (inverse)
169	emit(A64_MOVN(1, reg, (rev_tmp >> shift) & 0xffff, shift), ctx);
170	else
171	emit(A64_MOVZ(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx);
172	shift -= 16;
173	while (shift >= 0) {
174	if (((nrm_tmp >> shift) & 0xffff) != (inverse ? 0xffff : 0x0000))
175	emit(A64_MOVK(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx);
176	shift -= 16;
177	}
178	}
179
180	static inline void emit_bti(u32 insn, struct jit_ctx *ctx)
181	{
182	if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
183	emit(insn, ctx);
184	}
185
186	static inline void emit_kcfi(u32 hash, struct jit_ctx *ctx)
187	{
188	if (IS_ENABLED(CONFIG_CFI))
189	emit_u32_data(data: hash, ctx);
190	}
191
192	/*
193	* Kernel addresses in the vmalloc space use at most 48 bits, and the
194	* remaining bits are guaranteed to be 0x1. So we can compose the address
195	* with a fixed length movn/movk/movk sequence.
196	*/
197	static inline void emit_addr_mov_i64(const int reg, const u64 val,
198	struct jit_ctx *ctx)
199	{
200	u64 tmp = val;
201	int shift = 0;
202
203	emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx);
204	while (shift < 32) {
205	tmp >>= 16;
206	shift += 16;
207	emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx);
208	}
209	}
210
211	static bool should_emit_indirect_call(long target, const struct jit_ctx *ctx)
212	{
213	long offset;
214
215	/* when ctx->ro_image is not allocated or the target is unknown,
216	* emit indirect call
217	*/
218	if (!ctx->ro_image || !target)
219	return true;
220
221	offset = target - (long)&ctx->ro_image[ctx->idx];
222	return offset < -SZ_128M || offset >= SZ_128M;
223	}
224
225	static void emit_direct_call(u64 target, struct jit_ctx *ctx)
226	{
227	u32 insn;
228	unsigned long pc;
229
230	pc = (unsigned long)&ctx->ro_image[ctx->idx];
231	insn = aarch64_insn_gen_branch_imm(pc, target, AARCH64_INSN_BRANCH_LINK);
232	emit(insn, ctx);
233	}
234
235	static void emit_indirect_call(u64 target, struct jit_ctx *ctx)
236	{
237	u8 tmp;
238
239	tmp = bpf2a64[TMP_REG_1];
240	emit_addr_mov_i64(reg: tmp, val: target, ctx);
241	emit(A64_BLR(tmp), ctx);
242	}
243
244	static void emit_call(u64 target, struct jit_ctx *ctx)
245	{
246	if (should_emit_indirect_call(target: (long)target, ctx))
247	emit_indirect_call(target, ctx);
248	else
249	emit_direct_call(target, ctx);
250	}
251
252	static inline int bpf2a64_offset(int bpf_insn, int off,
253	const struct jit_ctx *ctx)
254	{
255	/* BPF JMP offset is relative to the next instruction */
256	bpf_insn++;
257	/*
258	* Whereas arm64 branch instructions encode the offset
259	* from the branch itself, so we must subtract 1 from the
260	* instruction offset.
261	*/
262	return ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1);
263	}
264
265	static void jit_fill_hole(void *area, unsigned int size)
266	{
267	__le32 *ptr;
268	/* We are guaranteed to have aligned memory. */
269	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
270	*ptr++ = cpu_to_le32(AARCH64_BREAK_FAULT);
271	}
272
273	int bpf_arch_text_invalidate(void *dst, size_t len)
274	{
275	if (!aarch64_insn_set(dst, AARCH64_BREAK_FAULT, len))
276	return -EINVAL;
277
278	return 0;
279	}
280
281	static inline int epilogue_offset(const struct jit_ctx *ctx)
282	{
283	int to = ctx->epilogue_offset;
284	int from = ctx->idx;
285
286	return to - from;
287	}
288
289	static bool is_addsub_imm(u32 imm)
290	{
291	/* Either imm12 or shifted imm12. */
292	return !(imm & ~0xfff) || !(imm & ~0xfff000);
293	}
294
295	static inline void emit_a64_add_i(const bool is64, const int dst, const int src,
296	const int tmp, const s32 imm, struct jit_ctx *ctx)
297	{
298	if (is_addsub_imm(imm)) {
299	emit(A64_ADD_I(is64, dst, src, imm), ctx);
300	} else if (is_addsub_imm(imm: -(u32)imm)) {
301	emit(A64_SUB_I(is64, dst, src, -imm), ctx);
302	} else {
303	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
304	emit(A64_ADD(is64, dst, src, tmp), ctx);
305	}
306	}
307
308	/*
309	* There are 3 types of AArch64 LDR/STR (immediate) instruction:
310	* Post-index, Pre-index, Unsigned offset.
311	*
312	* For BPF ldr/str, the "unsigned offset" type is sufficient.
313	*
314	* "Unsigned offset" type LDR(immediate) format:
315	*
316	* 3 2 1 0
317	* 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
318	* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
319	* |x x|1 1 1 0 0 1 0 1| imm12 | Rn | Rt |
320	* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
321	* scale
322	*
323	* "Unsigned offset" type STR(immediate) format:
324	* 3 2 1 0
325	* 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
326	* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
327	* |x x|1 1 1 0 0 1 0 0| imm12 | Rn | Rt |
328	* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
329	* scale
330	*
331	* The offset is calculated from imm12 and scale in the following way:
332	*
333	* offset = (u64)imm12 << scale
334	*/
335	static bool is_lsi_offset(int offset, int scale)
336	{
337	if (offset < 0)
338	return false;
339
340	if (offset > (0xFFF << scale))
341	return false;
342
343	if (offset & ((1 << scale) - 1))
344	return false;
345
346	return true;
347	}
348
349	/* generated main prog prologue:
350	* bti c // if CONFIG_ARM64_BTI_KERNEL
351	* mov x9, lr
352	* nop // POKE_OFFSET
353	* paciasp // if CONFIG_ARM64_PTR_AUTH_KERNEL
354	* stp x29, lr, [sp, #-16]!
355	* mov x29, sp
356	* stp xzr, x26, [sp, #-16]!
357	* mov x26, sp
358	* // PROLOGUE_OFFSET
359	* // save callee-saved registers
360	*/
361	static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx)
362	{
363	const bool is_main_prog = !bpf_is_subprog(prog: ctx->prog);
364	const u8 ptr = bpf2a64[TCCNT_PTR];
365
366	if (is_main_prog) {
367	/* Initialize tail_call_cnt. */
368	emit(A64_PUSH(A64_ZR, ptr, A64_SP), ctx);
369	emit(A64_MOV(1, ptr, A64_SP), ctx);
370	} else
371	emit(A64_PUSH(ptr, ptr, A64_SP), ctx);
372	}
373
374	static void find_used_callee_regs(struct jit_ctx *ctx)
375	{
376	int i;
377	const struct bpf_prog *prog = ctx->prog;
378	const struct bpf_insn *insn = &prog->insnsi[0];
379	int reg_used = 0;
380
381	for (i = 0; i < prog->len; i++, insn++) {
382	if (insn->dst_reg == BPF_REG_6 || insn->src_reg == BPF_REG_6)
383	reg_used |= 1;
384
385	if (insn->dst_reg == BPF_REG_7 || insn->src_reg == BPF_REG_7)
386	reg_used |= 2;
387
388	if (insn->dst_reg == BPF_REG_8 || insn->src_reg == BPF_REG_8)
389	reg_used |= 4;
390
391	if (insn->dst_reg == BPF_REG_9 || insn->src_reg == BPF_REG_9)
392	reg_used |= 8;
393
394	if (insn->dst_reg == BPF_REG_FP || insn->src_reg == BPF_REG_FP) {
395	ctx->fp_used = true;
396	reg_used |= 16;
397	}
398	}
399
400	i = 0;
401	if (reg_used & 1)
402	ctx->used_callee_reg[i++] = bpf2a64[BPF_REG_6];
403
404	if (reg_used & 2)
405	ctx->used_callee_reg[i++] = bpf2a64[BPF_REG_7];
406
407	if (reg_used & 4)
408	ctx->used_callee_reg[i++] = bpf2a64[BPF_REG_8];
409
410	if (reg_used & 8)
411	ctx->used_callee_reg[i++] = bpf2a64[BPF_REG_9];
412
413	if (reg_used & 16) {
414	ctx->used_callee_reg[i++] = bpf2a64[BPF_REG_FP];
415	if (ctx->priv_sp_used)
416	ctx->used_callee_reg[i++] = bpf2a64[PRIVATE_SP];
417	}
418
419	if (ctx->arena_vm_start)
420	ctx->used_callee_reg[i++] = bpf2a64[ARENA_VM_START];
421
422	ctx->nr_used_callee_reg = i;
423	}
424
425	/* Save callee-saved registers */
426	static void push_callee_regs(struct jit_ctx *ctx)
427	{
428	int reg1, reg2, i;
429
430	/*
431	* Program acting as exception boundary should save all ARM64
432	* Callee-saved registers as the exception callback needs to recover
433	* all ARM64 Callee-saved registers in its epilogue.
434	*/
435	if (ctx->prog->aux->exception_boundary) {
436	emit(A64_PUSH(A64_R(19), A64_R(20), A64_SP), ctx);
437	emit(A64_PUSH(A64_R(21), A64_R(22), A64_SP), ctx);
438	emit(A64_PUSH(A64_R(23), A64_R(24), A64_SP), ctx);
439	emit(A64_PUSH(A64_R(25), A64_R(26), A64_SP), ctx);
440	emit(A64_PUSH(A64_R(27), A64_R(28), A64_SP), ctx);
441	ctx->fp_used = true;
442	} else {
443	find_used_callee_regs(ctx);
444	for (i = 0; i + 1 < ctx->nr_used_callee_reg; i += 2) {
445	reg1 = ctx->used_callee_reg[i];
446	reg2 = ctx->used_callee_reg[i + 1];
447	emit(A64_PUSH(reg1, reg2, A64_SP), ctx);
448	}
449	if (i < ctx->nr_used_callee_reg) {
450	reg1 = ctx->used_callee_reg[i];
451	/* keep SP 16-byte aligned */
452	emit(A64_PUSH(reg1, A64_ZR, A64_SP), ctx);
453	}
454	}
455	}
456
457	/* Restore callee-saved registers */
458	static void pop_callee_regs(struct jit_ctx *ctx)
459	{
460	struct bpf_prog_aux *aux = ctx->prog->aux;
461	int reg1, reg2, i;
462
463	/*
464	* Program acting as exception boundary pushes R23 and R24 in addition
465	* to BPF callee-saved registers. Exception callback uses the boundary
466	* program's stack frame, so recover these extra registers in the above
467	* two cases.
468	*/
469	if (aux->exception_boundary || aux->exception_cb) {
470	emit(A64_POP(A64_R(27), A64_R(28), A64_SP), ctx);
471	emit(A64_POP(A64_R(25), A64_R(26), A64_SP), ctx);
472	emit(A64_POP(A64_R(23), A64_R(24), A64_SP), ctx);
473	emit(A64_POP(A64_R(21), A64_R(22), A64_SP), ctx);
474	emit(A64_POP(A64_R(19), A64_R(20), A64_SP), ctx);
475	} else {
476	i = ctx->nr_used_callee_reg - 1;
477	if (ctx->nr_used_callee_reg % 2 != 0) {
478	reg1 = ctx->used_callee_reg[i];
479	emit(A64_POP(reg1, A64_ZR, A64_SP), ctx);
480	i--;
481	}
482	while (i > 0) {
483	reg1 = ctx->used_callee_reg[i - 1];
484	reg2 = ctx->used_callee_reg[i];
485	emit(A64_POP(reg1, reg2, A64_SP), ctx);
486	i -= 2;
487	}
488	}
489	}
490
491	static void emit_percpu_ptr(const u8 dst_reg, void __percpu *ptr,
492	struct jit_ctx *ctx)
493	{
494	const u8 tmp = bpf2a64[TMP_REG_1];
495
496	emit_a64_mov_i64(reg: dst_reg, val: (__force const u64)ptr, ctx);
497	if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
498	emit(A64_MRS_TPIDR_EL2(tmp), ctx);
499	else
500	emit(A64_MRS_TPIDR_EL1(tmp), ctx);
501	emit(A64_ADD(1, dst_reg, dst_reg, tmp), ctx);
502	}
503
504	#define BTI_INSNS (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) ? 1 : 0)
505	#define PAC_INSNS (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) ? 1 : 0)
506
507	/* Offset of nop instruction in bpf prog entry to be poked */
508	#define POKE_OFFSET (BTI_INSNS + 1)
509
510	/* Tail call offset to jump into */
511	#define PROLOGUE_OFFSET (BTI_INSNS + 2 + PAC_INSNS + 4)
512
513	static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
514	{
515	const struct bpf_prog *prog = ctx->prog;
516	const bool is_main_prog = !bpf_is_subprog(prog);
517	const u8 fp = bpf2a64[BPF_REG_FP];
518	const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
519	const u8 priv_sp = bpf2a64[PRIVATE_SP];
520	void __percpu *priv_stack_ptr;
521	int cur_offset;
522
523	/*
524	* BPF prog stack layout
525	*
526	* high
527	* original A64_SP => 0:+-----+ BPF prologue
528	* |FP/LR|
529	* current A64_FP => -16:+-----+
530	* | ... | callee saved registers
531	* BPF fp register => -64:+-----+ <= (BPF_FP)
532	* | |
533	* | ... | BPF prog stack
534	* | |
535	* +-----+ <= (BPF_FP - prog->aux->stack_depth)
536	* |RSVD | padding
537	* current A64_SP => +-----+ <= (BPF_FP - ctx->stack_size)
538	* | |
539	* | ... | Function call stack
540	* | |
541	* +-----+
542	* low
543	*
544	*/
545
546	emit_kcfi(hash: is_main_prog ? cfi_bpf_hash : cfi_bpf_subprog_hash, ctx);
547	const int idx0 = ctx->idx;
548
549	/* bpf function may be invoked by 3 instruction types:
550	* 1. bl, attached via freplace to bpf prog via short jump
551	* 2. br, attached via freplace to bpf prog via long jump
552	* 3. blr, working as a function pointer, used by emit_call.
553	* So BTI_JC should used here to support both br and blr.
554	*/
555	emit_bti(A64_BTI_JC, ctx);
556
557	emit(A64_MOV(1, A64_R(9), A64_LR), ctx);
558	emit(A64_NOP, ctx);
559
560	if (!prog->aux->exception_cb) {
561	/* Sign lr */
562	if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
563	emit(A64_PACIASP, ctx);
564
565	/* Save FP and LR registers to stay align with ARM64 AAPCS */
566	emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
567	emit(A64_MOV(1, A64_FP, A64_SP), ctx);
568
569	prepare_bpf_tail_call_cnt(ctx);
570
571	if (!ebpf_from_cbpf && is_main_prog) {
572	cur_offset = ctx->idx - idx0;
573	if (cur_offset != PROLOGUE_OFFSET) {
574	pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n",
575	cur_offset, PROLOGUE_OFFSET);
576	return -1;
577	}
578	/* BTI landing pad for the tail call, done with a BR */
579	emit_bti(A64_BTI_J, ctx);
580	}
581	push_callee_regs(ctx);
582	} else {
583	/*
584	* Exception callback receives FP of Main Program as third
585	* parameter
586	*/
587	emit(A64_MOV(1, A64_FP, A64_R(2)), ctx);
588	/*
589	* Main Program already pushed the frame record and the
590	* callee-saved registers. The exception callback will not push
591	* anything and re-use the main program's stack.
592	*
593	* 12 registers are on the stack
594	*/
595	emit(A64_SUB_I(1, A64_SP, A64_FP, 96), ctx);
596	}
597
598	/* Stack must be multiples of 16B */
599	ctx->stack_size = round_up(prog->aux->stack_depth, 16);
600
601	if (ctx->fp_used) {
602	if (ctx->priv_sp_used) {
603	/* Set up private stack pointer */
604	priv_stack_ptr = prog->aux->priv_stack_ptr + PRIV_STACK_GUARD_SZ;
605	emit_percpu_ptr(dst_reg: priv_sp, ptr: priv_stack_ptr, ctx);
606	emit(A64_ADD_I(1, fp, priv_sp, ctx->stack_size), ctx);
607	} else {
608	/* Set up BPF prog stack base register */
609	emit(A64_MOV(1, fp, A64_SP), ctx);
610	}
611	}
612
613	/* Set up function call stack */
614	if (ctx->stack_size && !ctx->priv_sp_used)
615	emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
616
617	if (ctx->arena_vm_start)
618	emit_a64_mov_i64(reg: arena_vm_base, val: ctx->arena_vm_start, ctx);
619
620	return 0;
621	}
622
623	static int emit_bpf_tail_call(struct jit_ctx *ctx)
624	{
625	/* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */
626	const u8 r2 = bpf2a64[BPF_REG_2];
627	const u8 r3 = bpf2a64[BPF_REG_3];
628
629	const u8 tmp = bpf2a64[TMP_REG_1];
630	const u8 prg = bpf2a64[TMP_REG_2];
631	const u8 tcc = bpf2a64[TMP_REG_3];
632	const u8 ptr = bpf2a64[TCCNT_PTR];
633	size_t off;
634	__le32 *branch1 = NULL;
635	__le32 *branch2 = NULL;
636	__le32 *branch3 = NULL;
637
638	/* if (index >= array->map.max_entries)
639	* goto out;
640	*/
641	off = offsetof(struct bpf_array, map.max_entries);
642	emit_a64_mov_i64(reg: tmp, val: off, ctx);
643	emit(A64_LDR32(tmp, r2, tmp), ctx);
644	emit(A64_MOV(0, r3, r3), ctx);
645	emit(A64_CMP(0, r3, tmp), ctx);
646	branch1 = ctx->image + ctx->idx;
647	emit(A64_NOP, ctx);
648
649	/*
650	* if ((*tail_call_cnt_ptr) >= MAX_TAIL_CALL_CNT)
651	* goto out;
652	*/
653	emit_a64_mov_i64(reg: tmp, MAX_TAIL_CALL_CNT, ctx);
654	emit(A64_LDR64I(tcc, ptr, 0), ctx);
655	emit(A64_CMP(1, tcc, tmp), ctx);
656	branch2 = ctx->image + ctx->idx;
657	emit(A64_NOP, ctx);
658
659	/* (*tail_call_cnt_ptr)++; */
660	emit(A64_ADD_I(1, tcc, tcc, 1), ctx);
661
662	/* prog = array->ptrs[index];
663	* if (prog == NULL)
664	* goto out;
665	*/
666	off = offsetof(struct bpf_array, ptrs);
667	emit_a64_mov_i64(reg: tmp, val: off, ctx);
668	emit(A64_ADD(1, tmp, r2, tmp), ctx);
669	emit(A64_LSL(1, prg, r3, 3), ctx);
670	emit(A64_LDR64(prg, tmp, prg), ctx);
671	branch3 = ctx->image + ctx->idx;
672	emit(A64_NOP, ctx);
673
674	/* Update tail_call_cnt if the slot is populated. */
675	emit(A64_STR64I(tcc, ptr, 0), ctx);
676
677	/* restore SP */
678	if (ctx->stack_size && !ctx->priv_sp_used)
679	emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
680
681	pop_callee_regs(ctx);
682
683	/* goto *(prog->bpf_func + prologue_offset); */
684	off = offsetof(struct bpf_prog, bpf_func);
685	emit_a64_mov_i64(reg: tmp, val: off, ctx);
686	emit(A64_LDR64(tmp, prg, tmp), ctx);
687	emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx);
688	emit(A64_BR(tmp), ctx);
689
690	if (ctx->image) {
691	off = &ctx->image[ctx->idx] - branch1;
692	*branch1 = cpu_to_le32(A64_B_(A64_COND_CS, off));
693
694	off = &ctx->image[ctx->idx] - branch2;
695	*branch2 = cpu_to_le32(A64_B_(A64_COND_CS, off));
696
697	off = &ctx->image[ctx->idx] - branch3;
698	*branch3 = cpu_to_le32(A64_CBZ(1, prg, off));
699	}
700
701	return 0;
702	}
703
704	static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx)
705	{
706	const s32 imm = insn->imm;
707	const s16 off = insn->off;
708	const u8 code = insn->code;
709	const bool arena = BPF_MODE(code) == BPF_PROBE_ATOMIC;
710	const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
711	const u8 dst = bpf2a64[insn->dst_reg];
712	const u8 src = bpf2a64[insn->src_reg];
713	const u8 tmp = bpf2a64[TMP_REG_1];
714	u8 reg;
715
716	switch (imm) {
717	case BPF_LOAD_ACQ:
718	reg = src;
719	break;
720	case BPF_STORE_REL:
721	reg = dst;
722	break;
723	default:
724	pr_err_once("unknown atomic load/store op code %02x\n", imm);
725	return -EINVAL;
726	}
727
728	if (off) {
729	emit_a64_add_i(is64: 1, dst: tmp, src: reg, tmp, imm: off, ctx);
730	reg = tmp;
731	}
732	if (arena) {
733	emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx);
734	reg = tmp;
735	}
736
737	switch (imm) {
738	case BPF_LOAD_ACQ:
739	switch (BPF_SIZE(code)) {
740	case BPF_B:
741	emit(A64_LDARB(dst, reg), ctx);
742	break;
743	case BPF_H:
744	emit(A64_LDARH(dst, reg), ctx);
745	break;
746	case BPF_W:
747	emit(A64_LDAR32(dst, reg), ctx);
748	break;
749	case BPF_DW:
750	emit(A64_LDAR64(dst, reg), ctx);
751	break;
752	}
753	break;
754	case BPF_STORE_REL:
755	switch (BPF_SIZE(code)) {
756	case BPF_B:
757	emit(A64_STLRB(src, reg), ctx);
758	break;
759	case BPF_H:
760	emit(A64_STLRH(src, reg), ctx);
761	break;
762	case BPF_W:
763	emit(A64_STLR32(src, reg), ctx);
764	break;
765	case BPF_DW:
766	emit(A64_STLR64(src, reg), ctx);
767	break;
768	}
769	break;
770	default:
771	pr_err_once("unexpected atomic load/store op code %02x\n",
772	imm);
773	return -EINVAL;
774	}
775
776	return 0;
777	}
778
779	#ifdef CONFIG_ARM64_LSE_ATOMICS
780	static int emit_lse_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
781	{
782	const u8 code = insn->code;
783	const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
784	const u8 dst = bpf2a64[insn->dst_reg];
785	const u8 src = bpf2a64[insn->src_reg];
786	const u8 tmp = bpf2a64[TMP_REG_1];
787	const u8 tmp2 = bpf2a64[TMP_REG_2];
788	const bool isdw = BPF_SIZE(code) == BPF_DW;
789	const bool arena = BPF_MODE(code) == BPF_PROBE_ATOMIC;
790	const s16 off = insn->off;
791	u8 reg = dst;
792
793	if (off) {
794	emit_a64_add_i(1, tmp, reg, tmp, off, ctx);
795	reg = tmp;
796	}
797	if (arena) {
798	emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx);
799	reg = tmp;
800	}
801
802	switch (insn->imm) {
803	/* lock *(u32/u64 *)(dst_reg + off) <op>= src_reg */
804	case BPF_ADD:
805	emit(A64_STADD(isdw, reg, src), ctx);
806	break;
807	case BPF_AND:
808	emit(A64_MVN(isdw, tmp2, src), ctx);
809	emit(A64_STCLR(isdw, reg, tmp2), ctx);
810	break;
811	case BPF_OR:
812	emit(A64_STSET(isdw, reg, src), ctx);
813	break;
814	case BPF_XOR:
815	emit(A64_STEOR(isdw, reg, src), ctx);
816	break;
817	/* src_reg = atomic_fetch_<op>(dst_reg + off, src_reg) */
818	case BPF_ADD | BPF_FETCH:
819	emit(A64_LDADDAL(isdw, src, reg, src), ctx);
820	break;
821	case BPF_AND | BPF_FETCH:
822	emit(A64_MVN(isdw, tmp2, src), ctx);
823	emit(A64_LDCLRAL(isdw, src, reg, tmp2), ctx);
824	break;
825	case BPF_OR | BPF_FETCH:
826	emit(A64_LDSETAL(isdw, src, reg, src), ctx);
827	break;
828	case BPF_XOR | BPF_FETCH:
829	emit(A64_LDEORAL(isdw, src, reg, src), ctx);
830	break;
831	/* src_reg = atomic_xchg(dst_reg + off, src_reg); */
832	case BPF_XCHG:
833	emit(A64_SWPAL(isdw, src, reg, src), ctx);
834	break;
835	/* r0 = atomic_cmpxchg(dst_reg + off, r0, src_reg); */
836	case BPF_CMPXCHG:
837	emit(A64_CASAL(isdw, src, reg, bpf2a64[BPF_REG_0]), ctx);
838	break;
839	default:
840	pr_err_once("unknown atomic op code %02x\n", insn->imm);
841	return -EINVAL;
842	}
843
844	return 0;
845	}
846	#else
847	static inline int emit_lse_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
848	{
849	return -EINVAL;
850	}
851	#endif
852
853	static int emit_ll_sc_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
854	{
855	const u8 code = insn->code;
856	const u8 dst = bpf2a64[insn->dst_reg];
857	const u8 src = bpf2a64[insn->src_reg];
858	const u8 tmp = bpf2a64[TMP_REG_1];
859	const u8 tmp2 = bpf2a64[TMP_REG_2];
860	const u8 tmp3 = bpf2a64[TMP_REG_3];
861	const int i = insn - ctx->prog->insnsi;
862	const s32 imm = insn->imm;
863	const s16 off = insn->off;
864	const bool isdw = BPF_SIZE(code) == BPF_DW;
865	u8 reg = dst;
866	s32 jmp_offset;
867
868	if (BPF_MODE(code) == BPF_PROBE_ATOMIC) {
869	/* ll_sc based atomics don't support unsafe pointers yet. */
870	pr_err_once("unknown atomic opcode %02x\n", code);
871	return -EINVAL;
872	}
873
874	if (off) {
875	emit_a64_add_i(is64: 1, dst: tmp, src: reg, tmp, imm: off, ctx);
876	reg = tmp;
877	}
878
879	if (imm == BPF_ADD || imm == BPF_AND ||
880	imm == BPF_OR || imm == BPF_XOR) {
881	/* lock *(u32/u64 *)(dst_reg + off) <op>= src_reg */
882	emit(A64_LDXR(isdw, tmp2, reg), ctx);
883	if (imm == BPF_ADD)
884	emit(A64_ADD(isdw, tmp2, tmp2, src), ctx);
885	else if (imm == BPF_AND)
886	emit(A64_AND(isdw, tmp2, tmp2, src), ctx);
887	else if (imm == BPF_OR)
888	emit(A64_ORR(isdw, tmp2, tmp2, src), ctx);
889	else
890	emit(A64_EOR(isdw, tmp2, tmp2, src), ctx);
891	emit(A64_STXR(isdw, tmp2, reg, tmp3), ctx);
892	jmp_offset = -3;
893	check_imm19(jmp_offset);
894	emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
895	} else if (imm == (BPF_ADD | BPF_FETCH) ||
896	imm == (BPF_AND | BPF_FETCH) ||
897	imm == (BPF_OR | BPF_FETCH) ||
898	imm == (BPF_XOR | BPF_FETCH)) {
899	/* src_reg = atomic_fetch_<op>(dst_reg + off, src_reg) */
900	const u8 ax = bpf2a64[BPF_REG_AX];
901
902	emit(A64_MOV(isdw, ax, src), ctx);
903	emit(A64_LDXR(isdw, src, reg), ctx);
904	if (imm == (BPF_ADD | BPF_FETCH))
905	emit(A64_ADD(isdw, tmp2, src, ax), ctx);
906	else if (imm == (BPF_AND | BPF_FETCH))
907	emit(A64_AND(isdw, tmp2, src, ax), ctx);
908	else if (imm == (BPF_OR | BPF_FETCH))
909	emit(A64_ORR(isdw, tmp2, src, ax), ctx);
910	else
911	emit(A64_EOR(isdw, tmp2, src, ax), ctx);
912	emit(A64_STLXR(isdw, tmp2, reg, tmp3), ctx);
913	jmp_offset = -3;
914	check_imm19(jmp_offset);
915	emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
916	emit(A64_DMB_ISH, ctx);
917	} else if (imm == BPF_XCHG) {
918	/* src_reg = atomic_xchg(dst_reg + off, src_reg); */
919	emit(A64_MOV(isdw, tmp2, src), ctx);
920	emit(A64_LDXR(isdw, src, reg), ctx);
921	emit(A64_STLXR(isdw, tmp2, reg, tmp3), ctx);
922	jmp_offset = -2;
923	check_imm19(jmp_offset);
924	emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
925	emit(A64_DMB_ISH, ctx);
926	} else if (imm == BPF_CMPXCHG) {
927	/* r0 = atomic_cmpxchg(dst_reg + off, r0, src_reg); */
928	const u8 r0 = bpf2a64[BPF_REG_0];
929
930	emit(A64_MOV(isdw, tmp2, r0), ctx);
931	emit(A64_LDXR(isdw, r0, reg), ctx);
932	emit(A64_EOR(isdw, tmp3, r0, tmp2), ctx);
933	jmp_offset = 4;
934	check_imm19(jmp_offset);
935	emit(A64_CBNZ(isdw, tmp3, jmp_offset), ctx);
936	emit(A64_STLXR(isdw, src, reg, tmp3), ctx);
937	jmp_offset = -4;
938	check_imm19(jmp_offset);
939	emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
940	emit(A64_DMB_ISH, ctx);
941	} else {
942	pr_err_once("unknown atomic op code %02x\n", imm);
943	return -EINVAL;
944	}
945
946	return 0;
947	}
948
949	void dummy_tramp(void);
950
951	asm (
952	" .pushsection .text, \"ax\", @progbits\n"
953	" .global dummy_tramp\n"
954	" .type dummy_tramp, %function\n"
955	"dummy_tramp:"
956	#if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)
957	" bti j\n" /* dummy_tramp is called via "br x10" */
958	#endif
959	" mov x10, x30\n"
960	" mov x30, x9\n"
961	" ret x10\n"
962	" .size dummy_tramp, .-dummy_tramp\n"
963	" .popsection\n"
964	);
965
966	/* build a plt initialized like this:
967	*
968	* plt:
969	* ldr tmp, target
970	* br tmp
971	* target:
972	* .quad dummy_tramp
973	*
974	* when a long jump trampoline is attached, target is filled with the
975	* trampoline address, and when the trampoline is removed, target is
976	* restored to dummy_tramp address.
977	*/
978	static void build_plt(struct jit_ctx *ctx)
979	{
980	const u8 tmp = bpf2a64[TMP_REG_1];
981	struct bpf_plt *plt = NULL;
982
983	/* make sure target is 64-bit aligned */
984	if ((ctx->idx + PLT_TARGET_OFFSET / AARCH64_INSN_SIZE) % 2)
985	emit(A64_NOP, ctx);
986
987	plt = (struct bpf_plt *)(ctx->image + ctx->idx);
988	/* plt is called via bl, no BTI needed here */
989	emit(A64_LDR64LIT(tmp, 2 * AARCH64_INSN_SIZE), ctx);
990	emit(A64_BR(tmp), ctx);
991
992	if (ctx->image)
993	plt->target = (u64)&dummy_tramp;
994	}
995
996	/* Clobbers BPF registers 1-4, aka x0-x3 */
997	static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx)
998	{
999	const u8 r1 = bpf2a64[BPF_REG_1]; /* aka x0 */
1000	u8 k = get_spectre_bhb_loop_value();
1001
1002	if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
1003	cpu_mitigations_off() || __nospectre_bhb ||
1004	arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE)
1005	return;
1006
1007	if (ns_capable_noaudit(ns: &init_user_ns, CAP_SYS_ADMIN))
1008	return;
1009
1010	if (supports_clearbhb(SCOPE_SYSTEM)) {
1011	emit(aarch64_insn_gen_hint(AARCH64_INSN_HINT_CLEARBHB), ctx);
1012	return;
1013	}
1014
1015	if (k) {
1016	emit_a64_mov_i64(reg: r1, val: k, ctx);
1017	emit(A64_B(1), ctx);
1018	emit(A64_SUBS_I(true, r1, r1, 1), ctx);
1019	emit(A64_B_(A64_COND_NE, -2), ctx);
1020	emit(aarch64_insn_gen_dsb(AARCH64_INSN_MB_ISH), ctx);
1021	emit(insn: aarch64_insn_get_isb_value(), ctx);
1022	}
1023
1024	if (is_spectre_bhb_fw_mitigated()) {
1025	emit(A64_ORR_I(false, r1, AARCH64_INSN_REG_ZR,
1026	ARM_SMCCC_ARCH_WORKAROUND_3), ctx);
1027	switch (arm_smccc_1_1_get_conduit()) {
1028	case SMCCC_CONDUIT_HVC:
1029	emit(insn: aarch64_insn_get_hvc_value(), ctx);
1030	break;
1031	case SMCCC_CONDUIT_SMC:
1032	emit(insn: aarch64_insn_get_smc_value(), ctx);
1033	break;
1034	default:
1035	pr_err_once("Firmware mitigation enabled with unknown conduit\n");
1036	}
1037	}
1038	}
1039
1040	static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
1041	{
1042	const u8 r0 = bpf2a64[BPF_REG_0];
1043	const u8 ptr = bpf2a64[TCCNT_PTR];
1044
1045	/* We're done with BPF stack */
1046	if (ctx->stack_size && !ctx->priv_sp_used)
1047	emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
1048
1049	pop_callee_regs(ctx);
1050
1051	emit(A64_POP(A64_ZR, ptr, A64_SP), ctx);
1052
1053	if (was_classic)
1054	build_bhb_mitigation(ctx);
1055
1056	/* Restore FP/LR registers */
1057	emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
1058
1059	/* Move the return value from bpf:r0 (aka x7) to x0 */
1060	emit(A64_MOV(1, A64_R(0), r0), ctx);
1061
1062	/* Authenticate lr */
1063	if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
1064	emit(A64_AUTIASP, ctx);
1065
1066	emit(A64_RET(A64_LR), ctx);
1067	}
1068
1069	/*
1070	* Metadata encoding for exception handling in JITed code.
1071	*
1072	* Format of `fixup` field in `struct exception_table_entry`:
1073	*
1074	* Bit layout of `fixup` (32-bit):
1075	*
1076	* +-----------+--------+-----------+-----------+----------+
1077	* | 31-27 | 26-22 | 21 | 20-16 | 15-0 |
1078	* | | | | | |
1079	* | FIXUP_REG | Unused | ARENA_ACC | ARENA_REG | OFFSET |
1080	* +-----------+--------+-----------+-----------+----------+
1081	*
1082	* - OFFSET (16 bits): Offset used to compute address for Load/Store instruction.
1083	* - ARENA_REG (5 bits): Register that is used to calculate the address for load/store when
1084	* accessing the arena region.
1085	* - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region.
1086	* - FIXUP_REG (5 bits): Destination register for the load instruction (cleared on fault) or set to
1087	* DONT_CLEAR if it is a store instruction.
1088	*/
1089
1090	#define BPF_FIXUP_OFFSET_MASK GENMASK(15, 0)
1091	#define BPF_FIXUP_ARENA_REG_MASK GENMASK(20, 16)
1092	#define BPF_ARENA_ACCESS BIT(21)
1093	#define BPF_FIXUP_REG_MASK GENMASK(31, 27)
1094	#define DONT_CLEAR 5 /* Unused ARM64 register from BPF's POV */
1095
1096	bool ex_handler_bpf(const struct exception_table_entry *ex,
1097	struct pt_regs *regs)
1098	{
1099	int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
1100	s16 off = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
1101	int arena_reg = FIELD_GET(BPF_FIXUP_ARENA_REG_MASK, ex->fixup);
1102	bool is_arena = !!(ex->fixup & BPF_ARENA_ACCESS);
1103	bool is_write = (dst_reg == DONT_CLEAR);
1104	unsigned long addr;
1105
1106	if (is_arena) {
1107	addr = regs->regs[arena_reg] + off;
1108	bpf_prog_report_arena_violation(write: is_write, addr, fault_ip: regs->pc);
1109	}
1110
1111	if (dst_reg != DONT_CLEAR)
1112	regs->regs[dst_reg] = 0;
1113	/* Skip the faulting instruction */
1114	regs->pc += AARCH64_INSN_SIZE;
1115
1116	return true;
1117	}
1118
1119	/* For accesses to BTF pointers, add an entry to the exception table */
1120	static int add_exception_handler(const struct bpf_insn *insn,
1121	struct jit_ctx *ctx,
1122	int dst_reg)
1123	{
1124	off_t ins_offset;
1125	s16 off = insn->off;
1126	bool is_arena;
1127	int arena_reg;
1128	unsigned long pc;
1129	struct exception_table_entry *ex;
1130
1131	if (!ctx->image)
1132	/* First pass */
1133	return 0;
1134
1135	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
1136	BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
1137	BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
1138	BPF_MODE(insn->code) != BPF_PROBE_MEM32SX &&
1139	BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
1140	return 0;
1141
1142	is_arena = (BPF_MODE(insn->code) == BPF_PROBE_MEM32) ||
1143	(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) ||
1144	(BPF_MODE(insn->code) == BPF_PROBE_ATOMIC);
1145
1146	if (!ctx->prog->aux->extable ||
1147	WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries))
1148	return -EINVAL;
1149
1150	ex = &ctx->prog->aux->extable[ctx->exentry_idx];
1151	pc = (unsigned long)&ctx->ro_image[ctx->idx - 1];
1152
1153	/*
1154	* This is the relative offset of the instruction that may fault from
1155	* the exception table itself. This will be written to the exception
1156	* table and if this instruction faults, the destination register will
1157	* be set to '0' and the execution will jump to the next instruction.
1158	*/
1159	ins_offset = pc - (long)&ex->insn;
1160	if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
1161	return -ERANGE;
1162
1163	/*
1164	* The offsets above have been calculated using the RO buffer but we
1165	* need to use the R/W buffer for writes.
1166	* switch ex to rw buffer for writing.
1167	*/
1168	ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image);
1169
1170	ex->insn = ins_offset;
1171
1172	if (BPF_CLASS(insn->code) != BPF_LDX)
1173	dst_reg = DONT_CLEAR;
1174
1175	ex->fixup = FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
1176
1177	if (is_arena) {
1178	ex->fixup |= BPF_ARENA_ACCESS;
1179	/*
1180	* insn->src_reg/dst_reg holds the address in the arena region with upper 32-bits
1181	* being zero because of a preceding addr_space_cast(r<n>, 0x0, 0x1) instruction.
1182	* This address is adjusted with the addition of arena_vm_start (see the
1183	* implementation of BPF_PROBE_MEM32 and BPF_PROBE_ATOMIC) before being used for the
1184	* memory access. Pass the reg holding the unmodified 32-bit address to
1185	* ex_handler_bpf.
1186	*/
1187	if (BPF_CLASS(insn->code) == BPF_LDX)
1188	arena_reg = bpf2a64[insn->src_reg];
1189	else
1190	arena_reg = bpf2a64[insn->dst_reg];
1191
1192	ex->fixup |= FIELD_PREP(BPF_FIXUP_OFFSET_MASK, off) |
1193	FIELD_PREP(BPF_FIXUP_ARENA_REG_MASK, arena_reg);
1194	}
1195
1196	ex->type = EX_TYPE_BPF;
1197
1198	ctx->exentry_idx++;
1199	return 0;
1200	}
1201
1202	/* JITs an eBPF instruction.
1203	* Returns:
1204	* 0 - successfully JITed an 8-byte eBPF instruction.
1205	* >0 - successfully JITed a 16-byte eBPF instruction.
1206	* <0 - failed to JIT.
1207	*/
1208	static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
1209	bool extra_pass)
1210	{
1211	const u8 code = insn->code;
1212	u8 dst = bpf2a64[insn->dst_reg];
1213	u8 src = bpf2a64[insn->src_reg];
1214	const u8 tmp = bpf2a64[TMP_REG_1];
1215	const u8 tmp2 = bpf2a64[TMP_REG_2];
1216	const u8 tmp3 = bpf2a64[TMP_REG_3];
1217	const u8 fp = bpf2a64[BPF_REG_FP];
1218	const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
1219	const u8 priv_sp = bpf2a64[PRIVATE_SP];
1220	const s16 off = insn->off;
1221	const s32 imm = insn->imm;
1222	const int i = insn - ctx->prog->insnsi;
1223	const bool is64 = BPF_CLASS(code) == BPF_ALU64 ||
1224	BPF_CLASS(code) == BPF_JMP;
1225	u8 jmp_cond;
1226	s32 jmp_offset;
1227	u32 a64_insn;
1228	u8 src_adj;
1229	u8 dst_adj;
1230	int off_adj;
1231	int ret;
1232	bool sign_extend;
1233
1234	switch (code) {
1235	/* dst = src */
1236	case BPF_ALU | BPF_MOV | BPF_X:
1237	case BPF_ALU64 | BPF_MOV | BPF_X:
1238	if (insn_is_cast_user(insn)) {
1239	emit(A64_MOV(0, tmp, src), ctx); // 32-bit mov clears the upper 32 bits
1240	emit_a64_mov_i(is64: 0, reg: dst, val: ctx->user_vm_start >> 32, ctx);
1241	emit(A64_LSL(1, dst, dst, 32), ctx);
1242	emit(A64_CBZ(1, tmp, 2), ctx);
1243	emit(A64_ORR(1, tmp, dst, tmp), ctx);
1244	emit(A64_MOV(1, dst, tmp), ctx);
1245	break;
1246	} else if (insn_is_mov_percpu_addr(insn)) {
1247	if (dst != src)
1248	emit(A64_MOV(1, dst, src), ctx);
1249	if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
1250	emit(A64_MRS_TPIDR_EL2(tmp), ctx);
1251	else
1252	emit(A64_MRS_TPIDR_EL1(tmp), ctx);
1253	emit(A64_ADD(1, dst, dst, tmp), ctx);
1254	break;
1255	}
1256	switch (insn->off) {
1257	case 0:
1258	emit(A64_MOV(is64, dst, src), ctx);
1259	break;
1260	case 8:
1261	emit(A64_SXTB(is64, dst, src), ctx);
1262	break;
1263	case 16:
1264	emit(A64_SXTH(is64, dst, src), ctx);
1265	break;
1266	case 32:
1267	emit(A64_SXTW(is64, dst, src), ctx);
1268	break;
1269	}
1270	break;
1271	/* dst = dst OP src */
1272	case BPF_ALU | BPF_ADD | BPF_X:
1273	case BPF_ALU64 | BPF_ADD | BPF_X:
1274	emit(A64_ADD(is64, dst, dst, src), ctx);
1275	break;
1276	case BPF_ALU | BPF_SUB | BPF_X:
1277	case BPF_ALU64 | BPF_SUB | BPF_X:
1278	emit(A64_SUB(is64, dst, dst, src), ctx);
1279	break;
1280	case BPF_ALU | BPF_AND | BPF_X:
1281	case BPF_ALU64 | BPF_AND | BPF_X:
1282	emit(A64_AND(is64, dst, dst, src), ctx);
1283	break;
1284	case BPF_ALU | BPF_OR | BPF_X:
1285	case BPF_ALU64 | BPF_OR | BPF_X:
1286	emit(A64_ORR(is64, dst, dst, src), ctx);
1287	break;
1288	case BPF_ALU | BPF_XOR | BPF_X:
1289	case BPF_ALU64 | BPF_XOR | BPF_X:
1290	emit(A64_EOR(is64, dst, dst, src), ctx);
1291	break;
1292	case BPF_ALU | BPF_MUL | BPF_X:
1293	case BPF_ALU64 | BPF_MUL | BPF_X:
1294	emit(A64_MUL(is64, dst, dst, src), ctx);
1295	break;
1296	case BPF_ALU | BPF_DIV | BPF_X:
1297	case BPF_ALU64 | BPF_DIV | BPF_X:
1298	if (!off)
1299	emit(A64_UDIV(is64, dst, dst, src), ctx);
1300	else
1301	emit(A64_SDIV(is64, dst, dst, src), ctx);
1302	break;
1303	case BPF_ALU | BPF_MOD | BPF_X:
1304	case BPF_ALU64 | BPF_MOD | BPF_X:
1305	if (!off)
1306	emit(A64_UDIV(is64, tmp, dst, src), ctx);
1307	else
1308	emit(A64_SDIV(is64, tmp, dst, src), ctx);
1309	emit(A64_MSUB(is64, dst, dst, tmp, src), ctx);
1310	break;
1311	case BPF_ALU | BPF_LSH | BPF_X:
1312	case BPF_ALU64 | BPF_LSH | BPF_X:
1313	emit(A64_LSLV(is64, dst, dst, src), ctx);
1314	break;
1315	case BPF_ALU | BPF_RSH | BPF_X:
1316	case BPF_ALU64 | BPF_RSH | BPF_X:
1317	emit(A64_LSRV(is64, dst, dst, src), ctx);
1318	break;
1319	case BPF_ALU | BPF_ARSH | BPF_X:
1320	case BPF_ALU64 | BPF_ARSH | BPF_X:
1321	emit(A64_ASRV(is64, dst, dst, src), ctx);
1322	break;
1323	/* dst = -dst */
1324	case BPF_ALU | BPF_NEG:
1325	case BPF_ALU64 | BPF_NEG:
1326	emit(A64_NEG(is64, dst, dst), ctx);
1327	break;
1328	/* dst = BSWAP##imm(dst) */
1329	case BPF_ALU | BPF_END | BPF_FROM_LE:
1330	case BPF_ALU | BPF_END | BPF_FROM_BE:
1331	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
1332	#ifdef CONFIG_CPU_BIG_ENDIAN
1333	if (BPF_CLASS(code) == BPF_ALU && BPF_SRC(code) == BPF_FROM_BE)
1334	goto emit_bswap_uxt;
1335	#else /* !CONFIG_CPU_BIG_ENDIAN */
1336	if (BPF_CLASS(code) == BPF_ALU && BPF_SRC(code) == BPF_FROM_LE)
1337	goto emit_bswap_uxt;
1338	#endif
1339	switch (imm) {
1340	case 16:
1341	emit(A64_REV16(is64, dst, dst), ctx);
1342	/* zero-extend 16 bits into 64 bits */
1343	emit(A64_UXTH(is64, dst, dst), ctx);
1344	break;
1345	case 32:
1346	emit(A64_REV32(0, dst, dst), ctx);
1347	/* upper 32 bits already cleared */
1348	break;
1349	case 64:
1350	emit(A64_REV64(dst, dst), ctx);
1351	break;
1352	}
1353	break;
1354	emit_bswap_uxt:
1355	switch (imm) {
1356	case 16:
1357	/* zero-extend 16 bits into 64 bits */
1358	emit(A64_UXTH(is64, dst, dst), ctx);
1359	break;
1360	case 32:
1361	/* zero-extend 32 bits into 64 bits */
1362	emit(A64_UXTW(is64, dst, dst), ctx);
1363	break;
1364	case 64:
1365	/* nop */
1366	break;
1367	}
1368	break;
1369	/* dst = imm */
1370	case BPF_ALU | BPF_MOV | BPF_K:
1371	case BPF_ALU64 | BPF_MOV | BPF_K:
1372	emit_a64_mov_i(is64, reg: dst, val: imm, ctx);
1373	break;
1374	/* dst = dst OP imm */
1375	case BPF_ALU | BPF_ADD | BPF_K:
1376	case BPF_ALU64 | BPF_ADD | BPF_K:
1377	emit_a64_add_i(is64, dst, src: dst, tmp, imm, ctx);
1378	break;
1379	case BPF_ALU | BPF_SUB | BPF_K:
1380	case BPF_ALU64 | BPF_SUB | BPF_K:
1381	if (is_addsub_imm(imm)) {
1382	emit(A64_SUB_I(is64, dst, dst, imm), ctx);
1383	} else if (is_addsub_imm(imm: -(u32)imm)) {
1384	emit(A64_ADD_I(is64, dst, dst, -imm), ctx);
1385	} else {
1386	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1387	emit(A64_SUB(is64, dst, dst, tmp), ctx);
1388	}
1389	break;
1390	case BPF_ALU | BPF_AND | BPF_K:
1391	case BPF_ALU64 | BPF_AND | BPF_K:
1392	a64_insn = A64_AND_I(is64, dst, dst, imm);
1393	if (a64_insn != AARCH64_BREAK_FAULT) {
1394	emit(insn: a64_insn, ctx);
1395	} else {
1396	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1397	emit(A64_AND(is64, dst, dst, tmp), ctx);
1398	}
1399	break;
1400	case BPF_ALU | BPF_OR | BPF_K:
1401	case BPF_ALU64 | BPF_OR | BPF_K:
1402	a64_insn = A64_ORR_I(is64, dst, dst, imm);
1403	if (a64_insn != AARCH64_BREAK_FAULT) {
1404	emit(insn: a64_insn, ctx);
1405	} else {
1406	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1407	emit(A64_ORR(is64, dst, dst, tmp), ctx);
1408	}
1409	break;
1410	case BPF_ALU | BPF_XOR | BPF_K:
1411	case BPF_ALU64 | BPF_XOR | BPF_K:
1412	a64_insn = A64_EOR_I(is64, dst, dst, imm);
1413	if (a64_insn != AARCH64_BREAK_FAULT) {
1414	emit(insn: a64_insn, ctx);
1415	} else {
1416	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1417	emit(A64_EOR(is64, dst, dst, tmp), ctx);
1418	}
1419	break;
1420	case BPF_ALU | BPF_MUL | BPF_K:
1421	case BPF_ALU64 | BPF_MUL | BPF_K:
1422	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1423	emit(A64_MUL(is64, dst, dst, tmp), ctx);
1424	break;
1425	case BPF_ALU | BPF_DIV | BPF_K:
1426	case BPF_ALU64 | BPF_DIV | BPF_K:
1427	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1428	if (!off)
1429	emit(A64_UDIV(is64, dst, dst, tmp), ctx);
1430	else
1431	emit(A64_SDIV(is64, dst, dst, tmp), ctx);
1432	break;
1433	case BPF_ALU | BPF_MOD | BPF_K:
1434	case BPF_ALU64 | BPF_MOD | BPF_K:
1435	emit_a64_mov_i(is64, reg: tmp2, val: imm, ctx);
1436	if (!off)
1437	emit(A64_UDIV(is64, tmp, dst, tmp2), ctx);
1438	else
1439	emit(A64_SDIV(is64, tmp, dst, tmp2), ctx);
1440	emit(A64_MSUB(is64, dst, dst, tmp, tmp2), ctx);
1441	break;
1442	case BPF_ALU | BPF_LSH | BPF_K:
1443	case BPF_ALU64 | BPF_LSH | BPF_K:
1444	emit(A64_LSL(is64, dst, dst, imm), ctx);
1445	break;
1446	case BPF_ALU | BPF_RSH | BPF_K:
1447	case BPF_ALU64 | BPF_RSH | BPF_K:
1448	emit(A64_LSR(is64, dst, dst, imm), ctx);
1449	break;
1450	case BPF_ALU | BPF_ARSH | BPF_K:
1451	case BPF_ALU64 | BPF_ARSH | BPF_K:
1452	emit(A64_ASR(is64, dst, dst, imm), ctx);
1453	break;
1454
1455	/* JUMP reg */
1456	case BPF_JMP | BPF_JA | BPF_X:
1457	emit(A64_BR(dst), ctx);
1458	break;
1459	/* JUMP off */
1460	case BPF_JMP | BPF_JA:
1461	case BPF_JMP32 | BPF_JA:
1462	if (BPF_CLASS(code) == BPF_JMP)
1463	jmp_offset = bpf2a64_offset(bpf_insn: i, off, ctx);
1464	else
1465	jmp_offset = bpf2a64_offset(bpf_insn: i, off: imm, ctx);
1466	check_imm26(jmp_offset);
1467	emit(A64_B(jmp_offset), ctx);
1468	break;
1469	/* IF (dst COND src) JUMP off */
1470	case BPF_JMP | BPF_JEQ | BPF_X:
1471	case BPF_JMP | BPF_JGT | BPF_X:
1472	case BPF_JMP | BPF_JLT | BPF_X:
1473	case BPF_JMP | BPF_JGE | BPF_X:
1474	case BPF_JMP | BPF_JLE | BPF_X:
1475	case BPF_JMP | BPF_JNE | BPF_X:
1476	case BPF_JMP | BPF_JSGT | BPF_X:
1477	case BPF_JMP | BPF_JSLT | BPF_X:
1478	case BPF_JMP | BPF_JSGE | BPF_X:
1479	case BPF_JMP | BPF_JSLE | BPF_X:
1480	case BPF_JMP32 | BPF_JEQ | BPF_X:
1481	case BPF_JMP32 | BPF_JGT | BPF_X:
1482	case BPF_JMP32 | BPF_JLT | BPF_X:
1483	case BPF_JMP32 | BPF_JGE | BPF_X:
1484	case BPF_JMP32 | BPF_JLE | BPF_X:
1485	case BPF_JMP32 | BPF_JNE | BPF_X:
1486	case BPF_JMP32 | BPF_JSGT | BPF_X:
1487	case BPF_JMP32 | BPF_JSLT | BPF_X:
1488	case BPF_JMP32 | BPF_JSGE | BPF_X:
1489	case BPF_JMP32 | BPF_JSLE | BPF_X:
1490	emit(A64_CMP(is64, dst, src), ctx);
1491	emit_cond_jmp:
1492	jmp_offset = bpf2a64_offset(bpf_insn: i, off, ctx);
1493	check_imm19(jmp_offset);
1494	switch (BPF_OP(code)) {
1495	case BPF_JEQ:
1496	jmp_cond = A64_COND_EQ;
1497	break;
1498	case BPF_JGT:
1499	jmp_cond = A64_COND_HI;
1500	break;
1501	case BPF_JLT:
1502	jmp_cond = A64_COND_CC;
1503	break;
1504	case BPF_JGE:
1505	jmp_cond = A64_COND_CS;
1506	break;
1507	case BPF_JLE:
1508	jmp_cond = A64_COND_LS;
1509	break;
1510	case BPF_JSET:
1511	case BPF_JNE:
1512	jmp_cond = A64_COND_NE;
1513	break;
1514	case BPF_JSGT:
1515	jmp_cond = A64_COND_GT;
1516	break;
1517	case BPF_JSLT:
1518	jmp_cond = A64_COND_LT;
1519	break;
1520	case BPF_JSGE:
1521	jmp_cond = A64_COND_GE;
1522	break;
1523	case BPF_JSLE:
1524	jmp_cond = A64_COND_LE;
1525	break;
1526	default:
1527	return -EFAULT;
1528	}
1529	emit(A64_B_(jmp_cond, jmp_offset), ctx);
1530	break;
1531	case BPF_JMP | BPF_JSET | BPF_X:
1532	case BPF_JMP32 | BPF_JSET | BPF_X:
1533	emit(A64_TST(is64, dst, src), ctx);
1534	goto emit_cond_jmp;
1535	/* IF (dst COND imm) JUMP off */
1536	case BPF_JMP | BPF_JEQ | BPF_K:
1537	case BPF_JMP | BPF_JGT | BPF_K:
1538	case BPF_JMP | BPF_JLT | BPF_K:
1539	case BPF_JMP | BPF_JGE | BPF_K:
1540	case BPF_JMP | BPF_JLE | BPF_K:
1541	case BPF_JMP | BPF_JNE | BPF_K:
1542	case BPF_JMP | BPF_JSGT | BPF_K:
1543	case BPF_JMP | BPF_JSLT | BPF_K:
1544	case BPF_JMP | BPF_JSGE | BPF_K:
1545	case BPF_JMP | BPF_JSLE | BPF_K:
1546	case BPF_JMP32 | BPF_JEQ | BPF_K:
1547	case BPF_JMP32 | BPF_JGT | BPF_K:
1548	case BPF_JMP32 | BPF_JLT | BPF_K:
1549	case BPF_JMP32 | BPF_JGE | BPF_K:
1550	case BPF_JMP32 | BPF_JLE | BPF_K:
1551	case BPF_JMP32 | BPF_JNE | BPF_K:
1552	case BPF_JMP32 | BPF_JSGT | BPF_K:
1553	case BPF_JMP32 | BPF_JSLT | BPF_K:
1554	case BPF_JMP32 | BPF_JSGE | BPF_K:
1555	case BPF_JMP32 | BPF_JSLE | BPF_K:
1556	if (is_addsub_imm(imm)) {
1557	emit(A64_CMP_I(is64, dst, imm), ctx);
1558	} else if (is_addsub_imm(imm: -(u32)imm)) {
1559	emit(A64_CMN_I(is64, dst, -imm), ctx);
1560	} else {
1561	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1562	emit(A64_CMP(is64, dst, tmp), ctx);
1563	}
1564	goto emit_cond_jmp;
1565	case BPF_JMP | BPF_JSET | BPF_K:
1566	case BPF_JMP32 | BPF_JSET | BPF_K:
1567	a64_insn = A64_TST_I(is64, dst, imm);
1568	if (a64_insn != AARCH64_BREAK_FAULT) {
1569	emit(insn: a64_insn, ctx);
1570	} else {
1571	emit_a64_mov_i(is64, reg: tmp, val: imm, ctx);
1572	emit(A64_TST(is64, dst, tmp), ctx);
1573	}
1574	goto emit_cond_jmp;
1575	/* function call */
1576	case BPF_JMP | BPF_CALL:
1577	{
1578	const u8 r0 = bpf2a64[BPF_REG_0];
1579	bool func_addr_fixed;
1580	u64 func_addr;
1581	u32 cpu_offset;
1582
1583	/* Implement helper call to bpf_get_smp_processor_id() inline */
1584	if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) {
1585	cpu_offset = offsetof(struct thread_info, cpu);
1586
1587	emit(A64_MRS_SP_EL0(tmp), ctx);
1588	if (is_lsi_offset(offset: cpu_offset, scale: 2)) {
1589	emit(A64_LDR32I(r0, tmp, cpu_offset), ctx);
1590	} else {
1591	emit_a64_mov_i(is64: 1, reg: tmp2, val: cpu_offset, ctx);
1592	emit(A64_LDR32(r0, tmp, tmp2), ctx);
1593	}
1594	break;
1595	}
1596
1597	/* Implement helper call to bpf_get_current_task/_btf() inline */
1598	if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task ||
1599	insn->imm == BPF_FUNC_get_current_task_btf)) {
1600	emit(A64_MRS_SP_EL0(r0), ctx);
1601	break;
1602	}
1603
1604	ret = bpf_jit_get_func_addr(prog: ctx->prog, insn, extra_pass,
1605	func_addr: &func_addr, func_addr_fixed: &func_addr_fixed);
1606	if (ret < 0)
1607	return ret;
1608	emit_call(target: func_addr, ctx);
1609	/*
1610	* Call to arch_bpf_timed_may_goto() is emitted by the
1611	* verifier and called with custom calling convention with
1612	* first argument and return value in BPF_REG_AX (x9).
1613	*/
1614	if (func_addr != (u64)arch_bpf_timed_may_goto)
1615	emit(A64_MOV(1, r0, A64_R(0)), ctx);
1616	break;
1617	}
1618	/* tail call */
1619	case BPF_JMP | BPF_TAIL_CALL:
1620	if (emit_bpf_tail_call(ctx))
1621	return -EFAULT;
1622	break;
1623	/* function return */
1624	case BPF_JMP | BPF_EXIT:
1625	/* Optimization: when last instruction is EXIT,
1626	simply fallthrough to epilogue. */
1627	if (i == ctx->prog->len - 1)
1628	break;
1629	jmp_offset = epilogue_offset(ctx);
1630	check_imm26(jmp_offset);
1631	emit(A64_B(jmp_offset), ctx);
1632	break;
1633
1634	/* dst = imm64 */
1635	case BPF_LD | BPF_IMM | BPF_DW:
1636	{
1637	const struct bpf_insn insn1 = insn[1];
1638	u64 imm64;
1639
1640	imm64 = (u64)insn1.imm << 32 | (u32)imm;
1641	if (bpf_pseudo_func(insn))
1642	emit_addr_mov_i64(reg: dst, val: imm64, ctx);
1643	else
1644	emit_a64_mov_i64(reg: dst, val: imm64, ctx);
1645
1646	return 1;
1647	}
1648
1649	/* LDX: dst = (u64)*(unsigned size *)(src + off) */
1650	case BPF_LDX | BPF_MEM | BPF_W:
1651	case BPF_LDX | BPF_MEM | BPF_H:
1652	case BPF_LDX | BPF_MEM | BPF_B:
1653	case BPF_LDX | BPF_MEM | BPF_DW:
1654	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1655	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1656	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1657	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1658	/* LDXS: dst_reg = (s64)*(signed size *)(src_reg + off) */
1659	case BPF_LDX | BPF_MEMSX | BPF_B:
1660	case BPF_LDX | BPF_MEMSX | BPF_H:
1661	case BPF_LDX | BPF_MEMSX | BPF_W:
1662	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1663	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1664	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1665	case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:
1666	case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
1667	case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
1668	case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
1669	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
1670	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
1671	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
1672	if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
1673	BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) {
1674	emit(A64_ADD(1, tmp2, src, arena_vm_base), ctx);
1675	src = tmp2;
1676	}
1677	if (src == fp) {
1678	src_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
1679	off_adj = off + ctx->stack_size;
1680	} else {
1681	src_adj = src;
1682	off_adj = off;
1683	}
1684	sign_extend = (BPF_MODE(insn->code) == BPF_MEMSX ||
1685	BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
1686	BPF_MODE(insn->code) == BPF_PROBE_MEM32SX);
1687	switch (BPF_SIZE(code)) {
1688	case BPF_W:
1689	if (is_lsi_offset(offset: off_adj, scale: 2)) {
1690	if (sign_extend)
1691	emit(A64_LDRSWI(dst, src_adj, off_adj), ctx);
1692	else
1693	emit(A64_LDR32I(dst, src_adj, off_adj), ctx);
1694	} else {
1695	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1696	if (sign_extend)
1697	emit(A64_LDRSW(dst, src, tmp), ctx);
1698	else
1699	emit(A64_LDR32(dst, src, tmp), ctx);
1700	}
1701	break;
1702	case BPF_H:
1703	if (is_lsi_offset(offset: off_adj, scale: 1)) {
1704	if (sign_extend)
1705	emit(A64_LDRSHI(dst, src_adj, off_adj), ctx);
1706	else
1707	emit(A64_LDRHI(dst, src_adj, off_adj), ctx);
1708	} else {
1709	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1710	if (sign_extend)
1711	emit(A64_LDRSH(dst, src, tmp), ctx);
1712	else
1713	emit(A64_LDRH(dst, src, tmp), ctx);
1714	}
1715	break;
1716	case BPF_B:
1717	if (is_lsi_offset(offset: off_adj, scale: 0)) {
1718	if (sign_extend)
1719	emit(A64_LDRSBI(dst, src_adj, off_adj), ctx);
1720	else
1721	emit(A64_LDRBI(dst, src_adj, off_adj), ctx);
1722	} else {
1723	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1724	if (sign_extend)
1725	emit(A64_LDRSB(dst, src, tmp), ctx);
1726	else
1727	emit(A64_LDRB(dst, src, tmp), ctx);
1728	}
1729	break;
1730	case BPF_DW:
1731	if (is_lsi_offset(offset: off_adj, scale: 3)) {
1732	emit(A64_LDR64I(dst, src_adj, off_adj), ctx);
1733	} else {
1734	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1735	emit(A64_LDR64(dst, src, tmp), ctx);
1736	}
1737	break;
1738	}
1739
1740	ret = add_exception_handler(insn, ctx, dst_reg: dst);
1741	if (ret)
1742	return ret;
1743	break;
1744
1745	/* speculation barrier against v1 and v4 */
1746	case BPF_ST | BPF_NOSPEC:
1747	if (alternative_has_cap_likely(ARM64_HAS_SB)) {
1748	emit(A64_SB, ctx);
1749	} else {
1750	emit(A64_DSB_NSH, ctx);
1751	emit(A64_ISB, ctx);
1752	}
1753	break;
1754
1755	/* ST: *(size *)(dst + off) = imm */
1756	case BPF_ST | BPF_MEM | BPF_W:
1757	case BPF_ST | BPF_MEM | BPF_H:
1758	case BPF_ST | BPF_MEM | BPF_B:
1759	case BPF_ST | BPF_MEM | BPF_DW:
1760	case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
1761	case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
1762	case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
1763	case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
1764	if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
1765	emit(A64_ADD(1, tmp3, dst, arena_vm_base), ctx);
1766	dst = tmp3;
1767	}
1768	if (dst == fp) {
1769	dst_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
1770	off_adj = off + ctx->stack_size;
1771	} else {
1772	dst_adj = dst;
1773	off_adj = off;
1774	}
1775	/* Load imm to a register then store it */
1776	emit_a64_mov_i(is64: 1, reg: tmp, val: imm, ctx);
1777	switch (BPF_SIZE(code)) {
1778	case BPF_W:
1779	if (is_lsi_offset(offset: off_adj, scale: 2)) {
1780	emit(A64_STR32I(tmp, dst_adj, off_adj), ctx);
1781	} else {
1782	emit_a64_mov_i(is64: 1, reg: tmp2, val: off, ctx);
1783	emit(A64_STR32(tmp, dst, tmp2), ctx);
1784	}
1785	break;
1786	case BPF_H:
1787	if (is_lsi_offset(offset: off_adj, scale: 1)) {
1788	emit(A64_STRHI(tmp, dst_adj, off_adj), ctx);
1789	} else {
1790	emit_a64_mov_i(is64: 1, reg: tmp2, val: off, ctx);
1791	emit(A64_STRH(tmp, dst, tmp2), ctx);
1792	}
1793	break;
1794	case BPF_B:
1795	if (is_lsi_offset(offset: off_adj, scale: 0)) {
1796	emit(A64_STRBI(tmp, dst_adj, off_adj), ctx);
1797	} else {
1798	emit_a64_mov_i(is64: 1, reg: tmp2, val: off, ctx);
1799	emit(A64_STRB(tmp, dst, tmp2), ctx);
1800	}
1801	break;
1802	case BPF_DW:
1803	if (is_lsi_offset(offset: off_adj, scale: 3)) {
1804	emit(A64_STR64I(tmp, dst_adj, off_adj), ctx);
1805	} else {
1806	emit_a64_mov_i(is64: 1, reg: tmp2, val: off, ctx);
1807	emit(A64_STR64(tmp, dst, tmp2), ctx);
1808	}
1809	break;
1810	}
1811
1812	ret = add_exception_handler(insn, ctx, dst_reg: dst);
1813	if (ret)
1814	return ret;
1815	break;
1816
1817	/* STX: *(size *)(dst + off) = src */
1818	case BPF_STX | BPF_MEM | BPF_W:
1819	case BPF_STX | BPF_MEM | BPF_H:
1820	case BPF_STX | BPF_MEM | BPF_B:
1821	case BPF_STX | BPF_MEM | BPF_DW:
1822	case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
1823	case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
1824	case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
1825	case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
1826	if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
1827	emit(A64_ADD(1, tmp2, dst, arena_vm_base), ctx);
1828	dst = tmp2;
1829	}
1830	if (dst == fp) {
1831	dst_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
1832	off_adj = off + ctx->stack_size;
1833	} else {
1834	dst_adj = dst;
1835	off_adj = off;
1836	}
1837	switch (BPF_SIZE(code)) {
1838	case BPF_W:
1839	if (is_lsi_offset(offset: off_adj, scale: 2)) {
1840	emit(A64_STR32I(src, dst_adj, off_adj), ctx);
1841	} else {
1842	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1843	emit(A64_STR32(src, dst, tmp), ctx);
1844	}
1845	break;
1846	case BPF_H:
1847	if (is_lsi_offset(offset: off_adj, scale: 1)) {
1848	emit(A64_STRHI(src, dst_adj, off_adj), ctx);
1849	} else {
1850	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1851	emit(A64_STRH(src, dst, tmp), ctx);
1852	}
1853	break;
1854	case BPF_B:
1855	if (is_lsi_offset(offset: off_adj, scale: 0)) {
1856	emit(A64_STRBI(src, dst_adj, off_adj), ctx);
1857	} else {
1858	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1859	emit(A64_STRB(src, dst, tmp), ctx);
1860	}
1861	break;
1862	case BPF_DW:
1863	if (is_lsi_offset(offset: off_adj, scale: 3)) {
1864	emit(A64_STR64I(src, dst_adj, off_adj), ctx);
1865	} else {
1866	emit_a64_mov_i(is64: 1, reg: tmp, val: off, ctx);
1867	emit(A64_STR64(src, dst, tmp), ctx);
1868	}
1869	break;
1870	}
1871
1872	ret = add_exception_handler(insn, ctx, dst_reg: dst);
1873	if (ret)
1874	return ret;
1875	break;
1876
1877	case BPF_STX | BPF_ATOMIC | BPF_B:
1878	case BPF_STX | BPF_ATOMIC | BPF_H:
1879	case BPF_STX | BPF_ATOMIC | BPF_W:
1880	case BPF_STX | BPF_ATOMIC | BPF_DW:
1881	case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
1882	case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
1883	case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
1884	case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
1885	if (bpf_atomic_is_load_store(atomic_insn: insn))
1886	ret = emit_atomic_ld_st(insn, ctx);
1887	else if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
1888	ret = emit_lse_atomic(insn, ctx);
1889	else
1890	ret = emit_ll_sc_atomic(insn, ctx);
1891	if (ret)
1892	return ret;
1893
1894	if (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
1895	ret = add_exception_handler(insn, ctx, dst_reg: dst);
1896	if (ret)
1897	return ret;
1898	}
1899	break;
1900
1901	default:
1902	pr_err_once("unknown opcode %02x\n", code);
1903	return -EINVAL;
1904	}
1905
1906	return 0;
1907	}
1908
1909	static int build_body(struct jit_ctx *ctx, bool extra_pass)
1910	{
1911	const struct bpf_prog *prog = ctx->prog;
1912	int i;
1913
1914	/*
1915	* - offset[0] offset of the end of prologue,
1916	* start of the 1st instruction.
1917	* - offset[1] - offset of the end of 1st instruction,
1918	* start of the 2nd instruction
1919	* [....]
1920	* - offset[3] - offset of the end of 3rd instruction,
1921	* start of 4th instruction
1922	*/
1923	for (i = 0; i < prog->len; i++) {
1924	const struct bpf_insn *insn = &prog->insnsi[i];
1925	int ret;
1926
1927	ctx->offset[i] = ctx->idx;
1928	ret = build_insn(insn, ctx, extra_pass);
1929	if (ret > 0) {
1930	i++;
1931	ctx->offset[i] = ctx->idx;
1932	continue;
1933	}
1934	if (ret)
1935	return ret;
1936	}
1937	/*
1938	* offset is allocated with prog->len + 1 so fill in
1939	* the last element with the offset after the last
1940	* instruction (end of program)
1941	*/
1942	ctx->offset[i] = ctx->idx;
1943
1944	return 0;
1945	}
1946
1947	static int validate_code(struct jit_ctx *ctx)
1948	{
1949	int i;
1950
1951	for (i = 0; i < ctx->idx; i++) {
1952	u32 a64_insn = le32_to_cpu(ctx->image[i]);
1953
1954	if (a64_insn == AARCH64_BREAK_FAULT)
1955	return -1;
1956	}
1957	return 0;
1958	}
1959
1960	static int validate_ctx(struct jit_ctx *ctx)
1961	{
1962	if (validate_code(ctx))
1963	return -1;
1964
1965	if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries))
1966	return -1;
1967
1968	return 0;
1969	}
1970
1971	static inline void bpf_flush_icache(void *start, void *end)
1972	{
1973	flush_icache_range(start: (unsigned long)start, end: (unsigned long)end);
1974	}
1975
1976	static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
1977	{
1978	int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
1979	u64 *stack_ptr;
1980
1981	for_each_possible_cpu(cpu) {
1982	stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
1983	stack_ptr[0] = PRIV_STACK_GUARD_VAL;
1984	stack_ptr[1] = PRIV_STACK_GUARD_VAL;
1985	stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL;
1986	stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL;
1987	}
1988	}
1989
1990	static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size,
1991	struct bpf_prog *prog)
1992	{
1993	int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
1994	u64 *stack_ptr;
1995
1996	for_each_possible_cpu(cpu) {
1997	stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
1998	if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
1999	stack_ptr[1] != PRIV_STACK_GUARD_VAL ||
2000	stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL ||
2001	stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) {
2002	pr_err("BPF private stack overflow/underflow detected for prog %sx\n",
2003	bpf_jit_get_prog_name(prog));
2004	break;
2005	}
2006	}
2007	}
2008
2009	struct arm64_jit_data {
2010	struct bpf_binary_header *header;
2011	u8 *ro_image;
2012	struct bpf_binary_header *ro_header;
2013	struct jit_ctx ctx;
2014	};
2015
2016	struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
2017	{
2018	int image_size, prog_size, extable_size, extable_align, extable_offset;
2019	struct bpf_prog *tmp, *orig_prog = prog;
2020	struct bpf_binary_header *header;
2021	struct bpf_binary_header *ro_header = NULL;
2022	struct arm64_jit_data *jit_data;
2023	void __percpu *priv_stack_ptr = NULL;
2024	bool was_classic = bpf_prog_was_classic(prog);
2025	int priv_stack_alloc_sz;
2026	bool tmp_blinded = false;
2027	bool extra_pass = false;
2028	struct jit_ctx ctx;
2029	u8 *image_ptr;
2030	u8 *ro_image_ptr;
2031	int body_idx;
2032	int exentry_idx;
2033
2034	if (!prog->jit_requested)
2035	return orig_prog;
2036
2037	tmp = bpf_jit_blind_constants(fp: prog);
2038	/* If blinding was requested and we failed during blinding,
2039	* we must fall back to the interpreter.
2040	*/
2041	if (IS_ERR(ptr: tmp))
2042	return orig_prog;
2043	if (tmp != prog) {
2044	tmp_blinded = true;
2045	prog = tmp;
2046	}
2047
2048	jit_data = prog->aux->jit_data;
2049	if (!jit_data) {
2050	jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
2051	if (!jit_data) {
2052	prog = orig_prog;
2053	goto out;
2054	}
2055	prog->aux->jit_data = jit_data;
2056	}
2057	priv_stack_ptr = prog->aux->priv_stack_ptr;
2058	if (!priv_stack_ptr && prog->aux->jits_use_priv_stack) {
2059	/* Allocate actual private stack size with verifier-calculated
2060	* stack size plus two memory guards to protect overflow and
2061	* underflow.
2062	*/
2063	priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +
2064	2 * PRIV_STACK_GUARD_SZ;
2065	priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL);
2066	if (!priv_stack_ptr) {
2067	prog = orig_prog;
2068	goto out_priv_stack;
2069	}
2070
2071	priv_stack_init_guard(priv_stack_ptr, alloc_size: priv_stack_alloc_sz);
2072	prog->aux->priv_stack_ptr = priv_stack_ptr;
2073	}
2074	if (jit_data->ctx.offset) {
2075	ctx = jit_data->ctx;
2076	ro_image_ptr = jit_data->ro_image;
2077	ro_header = jit_data->ro_header;
2078	header = jit_data->header;
2079	image_ptr = (void *)header + ((void *)ro_image_ptr
2080	- (void *)ro_header);
2081	extra_pass = true;
2082	prog_size = sizeof(u32) * ctx.idx;
2083	goto skip_init_ctx;
2084	}
2085	memset(&ctx, 0, sizeof(ctx));
2086	ctx.prog = prog;
2087
2088	ctx.offset = kvcalloc(prog->len + 1, sizeof(int), GFP_KERNEL);
2089	if (ctx.offset == NULL) {
2090	prog = orig_prog;
2091	goto out_off;
2092	}
2093
2094	ctx.user_vm_start = bpf_arena_get_user_vm_start(arena: prog->aux->arena);
2095	ctx.arena_vm_start = bpf_arena_get_kern_vm_start(arena: prog->aux->arena);
2096
2097	if (priv_stack_ptr)
2098	ctx.priv_sp_used = true;
2099
2100	/* Pass 1: Estimate the maximum image size.
2101	*
2102	* BPF line info needs ctx->offset[i] to be the offset of
2103	* instruction[i] in jited image, so build prologue first.
2104	*/
2105	if (build_prologue(ctx: &ctx, ebpf_from_cbpf: was_classic)) {
2106	prog = orig_prog;
2107	goto out_off;
2108	}
2109
2110	if (build_body(ctx: &ctx, extra_pass)) {
2111	prog = orig_prog;
2112	goto out_off;
2113	}
2114
2115	ctx.epilogue_offset = ctx.idx;
2116	build_epilogue(ctx: &ctx, was_classic);
2117	build_plt(ctx: &ctx);
2118
2119	extable_align = __alignof__(struct exception_table_entry);
2120	extable_size = prog->aux->num_exentries *
2121	sizeof(struct exception_table_entry);
2122
2123	/* Now we know the maximum image size. */
2124	prog_size = sizeof(u32) * ctx.idx;
2125	/* also allocate space for plt target */
2126	extable_offset = round_up(prog_size + PLT_TARGET_SIZE, extable_align);
2127	image_size = extable_offset + extable_size;
2128	ro_header = bpf_jit_binary_pack_alloc(proglen: image_size, ro_image: &ro_image_ptr,
2129	alignment: sizeof(u32), rw_hdr: &header, rw_image: &image_ptr,
2130	bpf_fill_ill_insns: jit_fill_hole);
2131	if (!ro_header) {
2132	prog = orig_prog;
2133	goto out_off;
2134	}
2135
2136	/* Pass 2: Determine jited position and result for each instruction */
2137
2138	/*
2139	* Use the image(RW) for writing the JITed instructions. But also save
2140	* the ro_image(RX) for calculating the offsets in the image. The RW
2141	* image will be later copied to the RX image from where the program
2142	* will run. The bpf_jit_binary_pack_finalize() will do this copy in the
2143	* final step.
2144	*/
2145	ctx.image = (__le32 *)image_ptr;
2146	ctx.ro_image = (__le32 *)ro_image_ptr;
2147	if (extable_size)
2148	prog->aux->extable = (void *)ro_image_ptr + extable_offset;
2149	skip_init_ctx:
2150	ctx.idx = 0;
2151	ctx.exentry_idx = 0;
2152	ctx.write = true;
2153
2154	build_prologue(ctx: &ctx, ebpf_from_cbpf: was_classic);
2155
2156	/* Record exentry_idx and body_idx before first build_body */
2157	exentry_idx = ctx.exentry_idx;
2158	body_idx = ctx.idx;
2159	/* Dont write body instructions to memory for now */
2160	ctx.write = false;
2161
2162	if (build_body(ctx: &ctx, extra_pass)) {
2163	prog = orig_prog;
2164	goto out_free_hdr;
2165	}
2166
2167	ctx.epilogue_offset = ctx.idx;
2168	ctx.exentry_idx = exentry_idx;
2169	ctx.idx = body_idx;
2170	ctx.write = true;
2171
2172	/* Pass 3: Adjust jump offset and write final image */
2173	if (build_body(ctx: &ctx, extra_pass) ||
2174	WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) {
2175	prog = orig_prog;
2176	goto out_free_hdr;
2177	}
2178
2179	build_epilogue(ctx: &ctx, was_classic);
2180	build_plt(ctx: &ctx);
2181
2182	/* Extra pass to validate JITed code. */
2183	if (validate_ctx(ctx: &ctx)) {
2184	prog = orig_prog;
2185	goto out_free_hdr;
2186	}
2187
2188	/* update the real prog size */
2189	prog_size = sizeof(u32) * ctx.idx;
2190
2191	/* And we're done. */
2192	if (bpf_jit_enable > 1)
2193	bpf_jit_dump(flen: prog->len, proglen: prog_size, pass: 2, image: ctx.image);
2194
2195	if (!prog->is_func || extra_pass) {
2196	/* The jited image may shrink since the jited result for
2197	* BPF_CALL to subprog may be changed from indirect call
2198	* to direct call.
2199	*/
2200	if (extra_pass && ctx.idx > jit_data->ctx.idx) {
2201	pr_err_once("multi-func JIT bug %d > %d\n",
2202	ctx.idx, jit_data->ctx.idx);
2203	prog->bpf_func = NULL;
2204	prog->jited = 0;
2205	prog->jited_len = 0;
2206	goto out_free_hdr;
2207	}
2208	if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
2209	/* ro_header has been freed */
2210	ro_header = NULL;
2211	prog = orig_prog;
2212	goto out_off;
2213	}
2214	/*
2215	* The instructions have now been copied to the ROX region from
2216	* where they will execute. Now the data cache has to be cleaned to
2217	* the PoU and the I-cache has to be invalidated for the VAs.
2218	*/
2219	bpf_flush_icache(start: ro_header, end: ctx.ro_image + ctx.idx);
2220	} else {
2221	jit_data->ctx = ctx;
2222	jit_data->ro_image = ro_image_ptr;
2223	jit_data->header = header;
2224	jit_data->ro_header = ro_header;
2225	}
2226
2227	prog->bpf_func = (void *)ctx.ro_image + cfi_get_offset();
2228	prog->jited = 1;
2229	prog->jited_len = prog_size - cfi_get_offset();
2230
2231	if (!prog->is_func || extra_pass) {
2232	int i;
2233
2234	/* offset[prog->len] is the size of program */
2235	for (i = 0; i <= prog->len; i++)
2236	ctx.offset[i] *= AARCH64_INSN_SIZE;
2237	bpf_prog_fill_jited_linfo(prog, insn_to_jit_off: ctx.offset + 1);
2238	/*
2239	* The bpf_prog_update_insn_ptrs function expects offsets to
2240	* point to the first byte of the jitted instruction (unlike
2241	* the bpf_prog_fill_jited_linfo above, which, for historical
2242	* reasons, expects to point to the next instruction)
2243	*/
2244	bpf_prog_update_insn_ptrs(prog, offsets: ctx.offset, image: ctx.ro_image);
2245	out_off:
2246	if (!ro_header && priv_stack_ptr) {
2247	free_percpu(pdata: priv_stack_ptr);
2248	prog->aux->priv_stack_ptr = NULL;
2249	}
2250	kvfree(addr: ctx.offset);
2251	out_priv_stack:
2252	kfree(objp: jit_data);
2253	prog->aux->jit_data = NULL;
2254	}
2255	out:
2256	if (tmp_blinded)
2257	bpf_jit_prog_release_other(fp: prog, fp_other: prog == orig_prog ?
2258	tmp : orig_prog);
2259	return prog;
2260
2261	out_free_hdr:
2262	if (header) {
2263	bpf_arch_text_copy(dst: &ro_header->size, src: &header->size,
2264	len: sizeof(header->size));
2265	bpf_jit_binary_pack_free(ro_header, rw_header: header);
2266	}
2267	goto out_off;
2268	}
2269
2270	bool bpf_jit_supports_private_stack(void)
2271	{
2272	return true;
2273	}
2274
2275	bool bpf_jit_supports_kfunc_call(void)
2276	{
2277	return true;
2278	}
2279
2280	void *bpf_arch_text_copy(void *dst, void *src, size_t len)
2281	{
2282	if (!aarch64_insn_copy(dst, src, len))
2283	return ERR_PTR(error: -EINVAL);
2284	return dst;
2285	}
2286
2287	u64 bpf_jit_alloc_exec_limit(void)
2288	{
2289	return VMALLOC_END - VMALLOC_START;
2290	}
2291
2292	/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
2293	bool bpf_jit_supports_subprog_tailcalls(void)
2294	{
2295	return true;
2296	}
2297
2298	static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
2299	int bargs_off, int retval_off, int run_ctx_off,
2300	bool save_ret)
2301	{
2302	__le32 *branch;
2303	u64 enter_prog;
2304	u64 exit_prog;
2305	struct bpf_prog *p = l->link.prog;
2306	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
2307
2308	enter_prog = (u64)bpf_trampoline_enter(prog: p);
2309	exit_prog = (u64)bpf_trampoline_exit(prog: p);
2310
2311	if (l->cookie == 0) {
2312	/* if cookie is zero, one instruction is enough to store it */
2313	emit(A64_STR64I(A64_ZR, A64_SP, run_ctx_off + cookie_off), ctx);
2314	} else {
2315	emit_a64_mov_i64(A64_R(10), l->cookie, ctx);
2316	emit(A64_STR64I(A64_R(10), A64_SP, run_ctx_off + cookie_off),
2317	ctx);
2318	}
2319
2320	/* save p to callee saved register x19 to avoid loading p with mov_i64
2321	* each time.
2322	*/
2323	emit_addr_mov_i64(A64_R(19), (const u64)p, ctx);
2324
2325	/* arg1: prog */
2326	emit(A64_MOV(1, A64_R(0), A64_R(19)), ctx);
2327	/* arg2: &run_ctx */
2328	emit(A64_ADD_I(1, A64_R(1), A64_SP, run_ctx_off), ctx);
2329
2330	emit_call(target: enter_prog, ctx);
2331
2332	/* save return value to callee saved register x20 */
2333	emit(A64_MOV(1, A64_R(20), A64_R(0)), ctx);
2334
2335	/* if (__bpf_prog_enter(prog) == 0)
2336	* goto skip_exec_of_prog;
2337	*/
2338	branch = ctx->image + ctx->idx;
2339	emit(A64_NOP, ctx);
2340
2341	emit(A64_ADD_I(1, A64_R(0), A64_SP, bargs_off), ctx);
2342	if (!p->jited)
2343	emit_addr_mov_i64(A64_R(1), (const u64)p->insnsi, ctx);
2344
2345	emit_call(target: (const u64)p->bpf_func, ctx);
2346
2347	if (save_ret)
2348	emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx);
2349
2350	if (ctx->image) {
2351	int offset = &ctx->image[ctx->idx] - branch;
2352	*branch = cpu_to_le32(A64_CBZ(1, A64_R(0), offset));
2353	}
2354
2355	/* arg1: prog */
2356	emit(A64_MOV(1, A64_R(0), A64_R(19)), ctx);
2357	/* arg2: start time */
2358	emit(A64_MOV(1, A64_R(1), A64_R(20)), ctx);
2359	/* arg3: &run_ctx */
2360	emit(A64_ADD_I(1, A64_R(2), A64_SP, run_ctx_off), ctx);
2361
2362	emit_call(target: exit_prog, ctx);
2363	}
2364
2365	static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
2366	int bargs_off, int retval_off, int run_ctx_off,
2367	__le32 **branches)
2368	{
2369	int i;
2370
2371	/* The first fmod_ret program will receive a garbage return value.
2372	* Set this to 0 to avoid confusing the program.
2373	*/
2374	emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx);
2375	for (i = 0; i < tl->nr_links; i++) {
2376	invoke_bpf_prog(ctx, l: tl->links[i], bargs_off, retval_off,
2377	run_ctx_off, save_ret: true);
2378	/* if (*(u64 *)(sp + retval_off) != 0)
2379	* goto do_fexit;
2380	*/
2381	emit(A64_LDR64I(A64_R(10), A64_SP, retval_off), ctx);
2382	/* Save the location of branch, and generate a nop.
2383	* This nop will be replaced with a cbnz later.
2384	*/
2385	branches[i] = ctx->image + ctx->idx;
2386	emit(A64_NOP, ctx);
2387	}
2388	}
2389
2390	struct arg_aux {
2391	/* how many args are passed through registers, the rest of the args are
2392	* passed through stack
2393	*/
2394	int args_in_regs;
2395	/* how many registers are used to pass arguments */
2396	int regs_for_args;
2397	/* how much stack is used for additional args passed to bpf program
2398	* that did not fit in original function registers
2399	*/
2400	int bstack_for_args;
2401	/* home much stack is used for additional args passed to the
2402	* original function when called from trampoline (this one needs
2403	* arguments to be properly aligned)
2404	*/
2405	int ostack_for_args;
2406	};
2407
2408	static int calc_arg_aux(const struct btf_func_model *m,
2409	struct arg_aux *a)
2410	{
2411	int stack_slots, nregs, slots, i;
2412
2413	/* verifier ensures m->nr_args <= MAX_BPF_FUNC_ARGS */
2414	for (i = 0, nregs = 0; i < m->nr_args; i++) {
2415	slots = (m->arg_size[i] + 7) / 8;
2416	if (nregs + slots <= 8) /* passed through register ? */
2417	nregs += slots;
2418	else
2419	break;
2420	}
2421
2422	a->args_in_regs = i;
2423	a->regs_for_args = nregs;
2424	a->ostack_for_args = 0;
2425	a->bstack_for_args = 0;
2426
2427	/* the rest arguments are passed through stack */
2428	for (; i < m->nr_args; i++) {
2429	stack_slots = (m->arg_size[i] + 7) / 8;
2430	a->bstack_for_args += stack_slots * 8;
2431	a->ostack_for_args = a->ostack_for_args + stack_slots * 8;
2432	}
2433
2434	return 0;
2435	}
2436
2437	static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
2438	{
2439	if (effective_bytes) {
2440	int garbage_bits = 64 - 8 * effective_bytes;
2441	#ifdef CONFIG_CPU_BIG_ENDIAN
2442	/* garbage bits are at the right end */
2443	emit(A64_LSR(1, reg, reg, garbage_bits), ctx);
2444	emit(A64_LSL(1, reg, reg, garbage_bits), ctx);
2445	#else
2446	/* garbage bits are at the left end */
2447	emit(A64_LSL(1, reg, reg, garbage_bits), ctx);
2448	emit(A64_LSR(1, reg, reg, garbage_bits), ctx);
2449	#endif
2450	}
2451	}
2452
2453	static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
2454	const struct btf_func_model *m,
2455	const struct arg_aux *a,
2456	bool for_call_origin)
2457	{
2458	int i;
2459	int reg;
2460	int doff;
2461	int soff;
2462	int slots;
2463	u8 tmp = bpf2a64[TMP_REG_1];
2464
2465	/* store arguments to the stack for the bpf program, or restore
2466	* arguments from stack for the original function
2467	*/
2468	for (reg = 0; reg < a->regs_for_args; reg++) {
2469	emit(for_call_origin ?
2470	A64_LDR64I(reg, A64_SP, bargs_off) :
2471	A64_STR64I(reg, A64_SP, bargs_off),
2472	ctx);
2473	bargs_off += 8;
2474	}
2475
2476	soff = 32; /* on stack arguments start from FP + 32 */
2477	doff = (for_call_origin ? oargs_off : bargs_off);
2478
2479	/* save on stack arguments */
2480	for (i = a->args_in_regs; i < m->nr_args; i++) {
2481	slots = (m->arg_size[i] + 7) / 8;
2482	/* verifier ensures arg_size <= 16, so slots equals 1 or 2 */
2483	while (slots-- > 0) {
2484	emit(A64_LDR64I(tmp, A64_FP, soff), ctx);
2485	/* if there is unused space in the last slot, clear
2486	* the garbage contained in the space.
2487	*/
2488	if (slots == 0 && !for_call_origin)
2489	clear_garbage(ctx, reg: tmp, effective_bytes: m->arg_size[i] % 8);
2490	emit(A64_STR64I(tmp, A64_SP, doff), ctx);
2491	soff += 8;
2492	doff += 8;
2493	}
2494	}
2495	}
2496
2497	static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs)
2498	{
2499	int reg;
2500
2501	for (reg = 0; reg < nregs; reg++) {
2502	emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx);
2503	bargs_off += 8;
2504	}
2505	}
2506
2507	static bool is_struct_ops_tramp(const struct bpf_tramp_links *fentry_links)
2508	{
2509	return fentry_links->nr_links == 1 &&
2510	fentry_links->links[0]->link.type == BPF_LINK_TYPE_STRUCT_OPS;
2511	}
2512
2513	/* Based on the x86's implementation of arch_prepare_bpf_trampoline().
2514	*
2515	* bpf prog and function entry before bpf trampoline hooked:
2516	* mov x9, lr
2517	* nop
2518	*
2519	* bpf prog and function entry after bpf trampoline hooked:
2520	* mov x9, lr
2521	* bl <bpf_trampoline or plt>
2522	*
2523	*/
2524	static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
2525	struct bpf_tramp_links *tlinks, void *func_addr,
2526	const struct btf_func_model *m,
2527	const struct arg_aux *a,
2528	u32 flags)
2529	{
2530	int i;
2531	int stack_size;
2532	int retaddr_off;
2533	int regs_off;
2534	int retval_off;
2535	int bargs_off;
2536	int nfuncargs_off;
2537	int ip_off;
2538	int run_ctx_off;
2539	int oargs_off;
2540	int nfuncargs;
2541	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
2542	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
2543	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
2544	bool save_ret;
2545	__le32 **branches = NULL;
2546	bool is_struct_ops = is_struct_ops_tramp(fentry_links: fentry);
2547
2548	/* trampoline stack layout:
2549	* [ parent ip ]
2550	* [ FP ]
2551	* SP + retaddr_off [ self ip ]
2552	* [ FP ]
2553	*
2554	* [ padding ] align SP to multiples of 16
2555	*
2556	* [ x20 ] callee saved reg x20
2557	* SP + regs_off [ x19 ] callee saved reg x19
2558	*
2559	* SP + retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
2560	* BPF_TRAMP_F_RET_FENTRY_RET
2561	* [ arg reg N ]
2562	* [ ... ]
2563	* SP + bargs_off [ arg reg 1 ] for bpf
2564	*
2565	* SP + nfuncargs_off [ arg regs count ]
2566	*
2567	* SP + ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag
2568	*
2569	* SP + run_ctx_off [ bpf_tramp_run_ctx ]
2570	*
2571	* [ stack arg N ]
2572	* [ ... ]
2573	* SP + oargs_off [ stack arg 1 ] for original func
2574	*/
2575
2576	stack_size = 0;
2577	oargs_off = stack_size;
2578	if (flags & BPF_TRAMP_F_CALL_ORIG)
2579	stack_size += a->ostack_for_args;
2580
2581	run_ctx_off = stack_size;
2582	/* room for bpf_tramp_run_ctx */
2583	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
2584
2585	ip_off = stack_size;
2586	/* room for IP address argument */
2587	if (flags & BPF_TRAMP_F_IP_ARG)
2588	stack_size += 8;
2589
2590	nfuncargs_off = stack_size;
2591	/* room for args count */
2592	stack_size += 8;
2593
2594	bargs_off = stack_size;
2595	/* room for args */
2596	nfuncargs = a->regs_for_args + a->bstack_for_args / 8;
2597	stack_size += 8 * nfuncargs;
2598
2599	/* room for return value */
2600	retval_off = stack_size;
2601	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
2602	if (save_ret)
2603	stack_size += 8;
2604
2605	/* room for callee saved registers, currently x19 and x20 are used */
2606	regs_off = stack_size;
2607	stack_size += 16;
2608
2609	/* round up to multiples of 16 to avoid SPAlignmentFault */
2610	stack_size = round_up(stack_size, 16);
2611
2612	/* return address locates above FP */
2613	retaddr_off = stack_size + 8;
2614
2615	if (flags & BPF_TRAMP_F_INDIRECT) {
2616	/*
2617	* Indirect call for bpf_struct_ops
2618	*/
2619	emit_kcfi(hash: cfi_get_func_hash(func: func_addr), ctx);
2620	}
2621	/* bpf trampoline may be invoked by 3 instruction types:
2622	* 1. bl, attached to bpf prog or kernel function via short jump
2623	* 2. br, attached to bpf prog or kernel function via long jump
2624	* 3. blr, working as a function pointer, used by struct_ops.
2625	* So BTI_JC should used here to support both br and blr.
2626	*/
2627	emit_bti(A64_BTI_JC, ctx);
2628
2629	/* x9 is not set for struct_ops */
2630	if (!is_struct_ops) {
2631	/* frame for parent function */
2632	emit(A64_PUSH(A64_FP, A64_R(9), A64_SP), ctx);
2633	emit(A64_MOV(1, A64_FP, A64_SP), ctx);
2634	}
2635
2636	/* frame for patched function for tracing, or caller for struct_ops */
2637	emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
2638	emit(A64_MOV(1, A64_FP, A64_SP), ctx);
2639
2640	/* allocate stack space */
2641	emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx);
2642
2643	if (flags & BPF_TRAMP_F_IP_ARG) {
2644	/* save ip address of the traced function */
2645	emit_addr_mov_i64(A64_R(10), (const u64)func_addr, ctx);
2646	emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx);
2647	}
2648
2649	/* save arg regs count*/
2650	emit(A64_MOVZ(1, A64_R(10), nfuncargs, 0), ctx);
2651	emit(A64_STR64I(A64_R(10), A64_SP, nfuncargs_off), ctx);
2652
2653	/* save args for bpf */
2654	save_args(ctx, bargs_off, oargs_off, m, a, for_call_origin: false);
2655
2656	/* save callee saved registers */
2657	emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
2658	emit(A64_STR64I(A64_R(20), A64_SP, regs_off + 8), ctx);
2659
2660	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2661	/* for the first pass, assume the worst case */
2662	if (!ctx->image)
2663	ctx->idx += 4;
2664	else
2665	emit_a64_mov_i64(A64_R(0), (const u64)im, ctx);
2666	emit_call(target: (const u64)__bpf_tramp_enter, ctx);
2667	}
2668
2669	for (i = 0; i < fentry->nr_links; i++)
2670	invoke_bpf_prog(ctx, l: fentry->links[i], bargs_off,
2671	retval_off, run_ctx_off,
2672	save_ret: flags & BPF_TRAMP_F_RET_FENTRY_RET);
2673
2674	if (fmod_ret->nr_links) {
2675	branches = kcalloc(fmod_ret->nr_links, sizeof(__le32 *),
2676	GFP_KERNEL);
2677	if (!branches)
2678	return -ENOMEM;
2679
2680	invoke_bpf_mod_ret(ctx, tl: fmod_ret, bargs_off, retval_off,
2681	run_ctx_off, branches);
2682	}
2683
2684	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2685	/* save args for original func */
2686	save_args(ctx, bargs_off, oargs_off, m, a, for_call_origin: true);
2687	/* call original func */
2688	emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
2689	emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
2690	emit(A64_RET(A64_R(10)), ctx);
2691	/* store return value */
2692	emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx);
2693	/* reserve a nop for bpf_tramp_image_put */
2694	im->ip_after_call = ctx->ro_image + ctx->idx;
2695	emit(A64_NOP, ctx);
2696	}
2697
2698	/* update the branches saved in invoke_bpf_mod_ret with cbnz */
2699	for (i = 0; i < fmod_ret->nr_links && ctx->image != NULL; i++) {
2700	int offset = &ctx->image[ctx->idx] - branches[i];
2701	*branches[i] = cpu_to_le32(A64_CBNZ(1, A64_R(10), offset));
2702	}
2703
2704	for (i = 0; i < fexit->nr_links; i++)
2705	invoke_bpf_prog(ctx, l: fexit->links[i], bargs_off, retval_off,
2706	run_ctx_off, save_ret: false);
2707
2708	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2709	im->ip_epilogue = ctx->ro_image + ctx->idx;
2710	/* for the first pass, assume the worst case */
2711	if (!ctx->image)
2712	ctx->idx += 4;
2713	else
2714	emit_a64_mov_i64(A64_R(0), (const u64)im, ctx);
2715	emit_call(target: (const u64)__bpf_tramp_exit, ctx);
2716	}
2717
2718	if (flags & BPF_TRAMP_F_RESTORE_REGS)
2719	restore_args(ctx, bargs_off, nregs: a->regs_for_args);
2720
2721	/* restore callee saved register x19 and x20 */
2722	emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx);
2723	emit(A64_LDR64I(A64_R(20), A64_SP, regs_off + 8), ctx);
2724
2725	if (save_ret)
2726	emit(A64_LDR64I(A64_R(0), A64_SP, retval_off), ctx);
2727
2728	/* reset SP */
2729	emit(A64_MOV(1, A64_SP, A64_FP), ctx);
2730
2731	if (is_struct_ops) {
2732	emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
2733	emit(A64_RET(A64_LR), ctx);
2734	} else {
2735	/* pop frames */
2736	emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
2737	emit(A64_POP(A64_FP, A64_R(9), A64_SP), ctx);
2738
2739	if (flags & BPF_TRAMP_F_SKIP_FRAME) {
2740	/* skip patched function, return to parent */
2741	emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2742	emit(A64_RET(A64_R(9)), ctx);
2743	} else {
2744	/* return to patched function */
2745	emit(A64_MOV(1, A64_R(10), A64_LR), ctx);
2746	emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2747	emit(A64_RET(A64_R(10)), ctx);
2748	}
2749	}
2750
2751	kfree(objp: branches);
2752
2753	return ctx->idx;
2754	}
2755
2756	int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
2757	struct bpf_tramp_links *tlinks, void *func_addr)
2758	{
2759	struct jit_ctx ctx = {
2760	.image = NULL,
2761	.idx = 0,
2762	};
2763	struct bpf_tramp_image im;
2764	struct arg_aux aaux;
2765	int ret;
2766
2767	ret = calc_arg_aux(m, a: &aaux);
2768	if (ret < 0)
2769	return ret;
2770
2771	ret = prepare_trampoline(ctx: &ctx, im: &im, tlinks, func_addr, m, a: &aaux, flags);
2772	if (ret < 0)
2773	return ret;
2774
2775	return ret < 0 ? ret : ret * AARCH64_INSN_SIZE;
2776	}
2777
2778	void *arch_alloc_bpf_trampoline(unsigned int size)
2779	{
2780	return bpf_prog_pack_alloc(size, bpf_fill_ill_insns: jit_fill_hole);
2781	}
2782
2783	void arch_free_bpf_trampoline(void *image, unsigned int size)
2784	{
2785	bpf_prog_pack_free(ptr: image, size);
2786	}
2787
2788	int arch_protect_bpf_trampoline(void *image, unsigned int size)
2789	{
2790	return 0;
2791	}
2792
2793	int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
2794	void *ro_image_end, const struct btf_func_model *m,
2795	u32 flags, struct bpf_tramp_links *tlinks,
2796	void *func_addr)
2797	{
2798	u32 size = ro_image_end - ro_image;
2799	struct arg_aux aaux;
2800	void *image, *tmp;
2801	int ret;
2802
2803	/* image doesn't need to be in module memory range, so we can
2804	* use kvmalloc.
2805	*/
2806	image = kvmalloc(size, GFP_KERNEL);
2807	if (!image)
2808	return -ENOMEM;
2809
2810	struct jit_ctx ctx = {
2811	.image = image,
2812	.ro_image = ro_image,
2813	.idx = 0,
2814	.write = true,
2815	};
2816
2817
2818	jit_fill_hole(area: image, size: (unsigned int)(ro_image_end - ro_image));
2819	ret = calc_arg_aux(m, a: &aaux);
2820	if (ret)
2821	goto out;
2822	ret = prepare_trampoline(ctx: &ctx, im, tlinks, func_addr, m, a: &aaux, flags);
2823
2824	if (ret > 0 && validate_code(ctx: &ctx) < 0) {
2825	ret = -EINVAL;
2826	goto out;
2827	}
2828
2829	if (ret > 0)
2830	ret *= AARCH64_INSN_SIZE;
2831
2832	tmp = bpf_arch_text_copy(dst: ro_image, src: image, len: size);
2833	if (IS_ERR(ptr: tmp)) {
2834	ret = PTR_ERR(ptr: tmp);
2835	goto out;
2836	}
2837
2838	out:
2839	kvfree(addr: image);
2840	return ret;
2841	}
2842
2843	static bool is_long_jump(void *ip, void *target)
2844	{
2845	long offset;
2846
2847	/* NULL target means this is a NOP */
2848	if (!target)
2849	return false;
2850
2851	offset = (long)target - (long)ip;
2852	return offset < -SZ_128M || offset >= SZ_128M;
2853	}
2854
2855	static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip,
2856	void *addr, void *plt, u32 *insn)
2857	{
2858	void *target;
2859
2860	if (!addr) {
2861	*insn = aarch64_insn_gen_nop();
2862	return 0;
2863	}
2864
2865	if (is_long_jump(ip, target: addr))
2866	target = plt;
2867	else
2868	target = addr;
2869
2870	*insn = aarch64_insn_gen_branch_imm((unsigned long)ip,
2871	(unsigned long)target,
2872	type);
2873
2874	return *insn != AARCH64_BREAK_FAULT ? 0 : -EFAULT;
2875	}
2876
2877	/* Replace the branch instruction from @ip to @old_addr in a bpf prog or a bpf
2878	* trampoline with the branch instruction from @ip to @new_addr. If @old_addr
2879	* or @new_addr is NULL, the old or new instruction is NOP.
2880	*
2881	* When @ip is the bpf prog entry, a bpf trampoline is being attached or
2882	* detached. Since bpf trampoline and bpf prog are allocated separately with
2883	* vmalloc, the address distance may exceed 128MB, the maximum branch range.
2884	* So long jump should be handled.
2885	*
2886	* When a bpf prog is constructed, a plt pointing to empty trampoline
2887	* dummy_tramp is placed at the end:
2888	*
2889	* bpf_prog:
2890	* mov x9, lr
2891	* nop // patchsite
2892	* ...
2893	* ret
2894	*
2895	* plt:
2896	* ldr x10, target
2897	* br x10
2898	* target:
2899	* .quad dummy_tramp // plt target
2900	*
2901	* This is also the state when no trampoline is attached.
2902	*
2903	* When a short-jump bpf trampoline is attached, the patchsite is patched
2904	* to a bl instruction to the trampoline directly:
2905	*
2906	* bpf_prog:
2907	* mov x9, lr
2908	* bl <short-jump bpf trampoline address> // patchsite
2909	* ...
2910	* ret
2911	*
2912	* plt:
2913	* ldr x10, target
2914	* br x10
2915	* target:
2916	* .quad dummy_tramp // plt target
2917	*
2918	* When a long-jump bpf trampoline is attached, the plt target is filled with
2919	* the trampoline address and the patchsite is patched to a bl instruction to
2920	* the plt:
2921	*
2922	* bpf_prog:
2923	* mov x9, lr
2924	* bl plt // patchsite
2925	* ...
2926	* ret
2927	*
2928	* plt:
2929	* ldr x10, target
2930	* br x10
2931	* target:
2932	* .quad <long-jump bpf trampoline address> // plt target
2933	*
2934	* The dummy_tramp is used to prevent another CPU from jumping to unknown
2935	* locations during the patching process, making the patching process easier.
2936	*/
2937	int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
2938	enum bpf_text_poke_type new_t, void *old_addr,
2939	void *new_addr)
2940	{
2941	int ret;
2942	u32 old_insn;
2943	u32 new_insn;
2944	u32 replaced;
2945	struct bpf_plt *plt = NULL;
2946	unsigned long size = 0UL;
2947	unsigned long offset = ~0UL;
2948	enum aarch64_insn_branch_type branch_type;
2949	char namebuf[KSYM_NAME_LEN];
2950	void *image = NULL;
2951	u64 plt_target = 0ULL;
2952	bool poking_bpf_entry;
2953
2954	if (!__bpf_address_lookup(addr: (unsigned long)ip, size: &size, off: &offset, sym: namebuf))
2955	/* Only poking bpf text is supported. Since kernel function
2956	* entry is set up by ftrace, we reply on ftrace to poke kernel
2957	* functions.
2958	*/
2959	return -ENOTSUPP;
2960
2961	image = ip - offset;
2962	/* zero offset means we're poking bpf prog entry */
2963	poking_bpf_entry = (offset == 0UL);
2964
2965	/* bpf prog entry, find plt and the real patchsite */
2966	if (poking_bpf_entry) {
2967	/* plt locates at the end of bpf prog */
2968	plt = image + size - PLT_TARGET_OFFSET;
2969
2970	/* skip to the nop instruction in bpf prog entry:
2971	* bti c // if BTI enabled
2972	* mov x9, x30
2973	* nop
2974	*/
2975	ip = image + POKE_OFFSET * AARCH64_INSN_SIZE;
2976	}
2977
2978	/* long jump is only possible at bpf prog entry */
2979	if (WARN_ON((is_long_jump(ip, new_addr) || is_long_jump(ip, old_addr)) &&
2980	!poking_bpf_entry))
2981	return -EINVAL;
2982
2983	branch_type = old_t == BPF_MOD_CALL ? AARCH64_INSN_BRANCH_LINK :
2984	AARCH64_INSN_BRANCH_NOLINK;
2985	if (gen_branch_or_nop(type: branch_type, ip, addr: old_addr, plt, insn: &old_insn) < 0)
2986	return -EFAULT;
2987
2988	branch_type = new_t == BPF_MOD_CALL ? AARCH64_INSN_BRANCH_LINK :
2989	AARCH64_INSN_BRANCH_NOLINK;
2990	if (gen_branch_or_nop(type: branch_type, ip, addr: new_addr, plt, insn: &new_insn) < 0)
2991	return -EFAULT;
2992
2993	if (is_long_jump(ip, target: new_addr))
2994	plt_target = (u64)new_addr;
2995	else if (is_long_jump(ip, target: old_addr))
2996	/* if the old target is a long jump and the new target is not,
2997	* restore the plt target to dummy_tramp, so there is always a
2998	* legal and harmless address stored in plt target, and we'll
2999	* never jump from plt to an unknown place.
3000	*/
3001	plt_target = (u64)&dummy_tramp;
3002
3003	if (plt_target) {
3004	/* non-zero plt_target indicates we're patching a bpf prog,
3005	* which is read only.
3006	*/
3007	if (set_memory_rw(PAGE_MASK & ((uintptr_t)&plt->target), numpages: 1))
3008	return -EFAULT;
3009	WRITE_ONCE(plt->target, plt_target);
3010	set_memory_ro(PAGE_MASK & ((uintptr_t)&plt->target), numpages: 1);
3011	/* since plt target points to either the new trampoline
3012	* or dummy_tramp, even if another CPU reads the old plt
3013	* target value before fetching the bl instruction to plt,
3014	* it will be brought back by dummy_tramp, so no barrier is
3015	* required here.
3016	*/
3017	}
3018
3019	/* if the old target and the new target are both long jumps, no
3020	* patching is required
3021	*/
3022	if (old_insn == new_insn)
3023	return 0;
3024
3025	mutex_lock(&text_mutex);
3026	if (aarch64_insn_read(ip, &replaced)) {
3027	ret = -EFAULT;
3028	goto out;
3029	}
3030
3031	if (replaced != old_insn) {
3032	ret = -EFAULT;
3033	goto out;
3034	}
3035
3036	/* We call aarch64_insn_patch_text_nosync() to replace instruction
3037	* atomically, so no other CPUs will fetch a half-new and half-old
3038	* instruction. But there is chance that another CPU executes the
3039	* old instruction after the patching operation finishes (e.g.,
3040	* pipeline not flushed, or icache not synchronized yet).
3041	*
3042	* 1. when a new trampoline is attached, it is not a problem for
3043	* different CPUs to jump to different trampolines temporarily.
3044	*
3045	* 2. when an old trampoline is freed, we should wait for all other
3046	* CPUs to exit the trampoline and make sure the trampoline is no
3047	* longer reachable, since bpf_tramp_image_put() function already
3048	* uses percpu_ref and task-based rcu to do the sync, no need to call
3049	* the sync version here, see bpf_tramp_image_put() for details.
3050	*/
3051	ret = aarch64_insn_patch_text_nosync(ip, new_insn);
3052	out:
3053	mutex_unlock(lock: &text_mutex);
3054
3055	return ret;
3056	}
3057
3058	bool bpf_jit_supports_ptr_xchg(void)
3059	{
3060	return true;
3061	}
3062
3063	bool bpf_jit_supports_exceptions(void)
3064	{
3065	/* We unwind through both kernel frames starting from within bpf_throw
3066	* call and BPF frames. Therefore we require FP unwinder to be enabled
3067	* to walk kernel frames and reach BPF frames in the stack trace.
3068	* ARM64 kernel is always compiled with CONFIG_FRAME_POINTER=y
3069	*/
3070	return true;
3071	}
3072
3073	bool bpf_jit_supports_arena(void)
3074	{
3075	return true;
3076	}
3077
3078	bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
3079	{
3080	if (!in_arena)
3081	return true;
3082	switch (insn->code) {
3083	case BPF_STX | BPF_ATOMIC | BPF_W:
3084	case BPF_STX | BPF_ATOMIC | BPF_DW:
3085	if (!bpf_atomic_is_load_store(insn) &&
3086	!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
3087	return false;
3088	}
3089	return true;
3090	}
3091
3092	bool bpf_jit_supports_percpu_insn(void)
3093	{
3094	return true;
3095	}
3096
3097	bool bpf_jit_bypass_spec_v4(void)
3098	{
3099	/* In case of arm64, we rely on the firmware mitigation of Speculative
3100	* Store Bypass as controlled via the ssbd kernel parameter. Whenever
3101	* the mitigation is enabled, it works for all of the kernel code with
3102	* no need to provide any additional instructions. Therefore, skip
3103	* inserting nospec insns against Spectre v4.
3104	*/
3105	return true;
3106	}
3107
3108	bool bpf_jit_supports_timed_may_goto(void)
3109	{
3110	return true;
3111	}
3112
3113	bool bpf_jit_inlines_helper_call(s32 imm)
3114	{
3115	switch (imm) {
3116	case BPF_FUNC_get_smp_processor_id:
3117	case BPF_FUNC_get_current_task:
3118	case BPF_FUNC_get_current_task_btf:
3119	return true;
3120	default:
3121	return false;
3122	}
3123	}
3124
3125	void bpf_jit_free(struct bpf_prog *prog)
3126	{
3127	if (prog->jited) {
3128	struct arm64_jit_data *jit_data = prog->aux->jit_data;
3129	struct bpf_binary_header *hdr;
3130	void __percpu *priv_stack_ptr;
3131	int priv_stack_alloc_sz;
3132
3133	/*
3134	* If we fail the final pass of JIT (from jit_subprogs),
3135	* the program may not be finalized yet. Call finalize here
3136	* before freeing it.
3137	*/
3138	if (jit_data) {
3139	bpf_jit_binary_pack_finalize(ro_header: jit_data->ro_header, rw_header: jit_data->header);
3140	kfree(objp: jit_data);
3141	}
3142	prog->bpf_func -= cfi_get_offset();
3143	hdr = bpf_jit_binary_pack_hdr(fp: prog);
3144	bpf_jit_binary_pack_free(ro_header: hdr, NULL);
3145	priv_stack_ptr = prog->aux->priv_stack_ptr;
3146	if (priv_stack_ptr) {
3147	priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +
3148	2 * PRIV_STACK_GUARD_SZ;
3149	priv_stack_check_guard(priv_stack_ptr, alloc_size: priv_stack_alloc_sz, prog);
3150	free_percpu(pdata: prog->aux->priv_stack_ptr);
3151	}
3152	WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
3153	}
3154
3155	bpf_prog_unlock_free(fp: prog);
3156	}
3157