1	// SPDX-License-Identifier: GPL-2.0
2	#include <linux/moduleloader.h>
3	#include <linux/workqueue.h>
4	#include <linux/netdevice.h>
5	#include <linux/filter.h>
6	#include <linux/bpf.h>
7	#include <linux/cache.h>
8	#include <linux/if_vlan.h>
9
10	#include <asm/cacheflush.h>
11	#include <asm/ptrace.h>
12
13	#include "bpf_jit_64.h"
14
15	static inline bool is_simm13(unsigned int value)
16	{
17	return value + 0x1000 < 0x2000;
18	}
19
20	static inline bool is_simm10(unsigned int value)
21	{
22	return value + 0x200 < 0x400;
23	}
24
25	static inline bool is_simm5(unsigned int value)
26	{
27	return value + 0x10 < 0x20;
28	}
29
30	static inline bool is_sethi(unsigned int value)
31	{
32	return (value & ~0x3fffff) == 0;
33	}
34
35	static void bpf_flush_icache(void *start_, void *end_)
36	{
37	/* Cheetah's I-cache is fully coherent. */
38	if (tlb_type == spitfire) {
39	unsigned long start = (unsigned long) start_;
40	unsigned long end = (unsigned long) end_;
41
42	start &= ~7UL;
43	end = (end + 7UL) & ~7UL;
44	while (start < end) {
45	flushi(start);
46	start += 32;
47	}
48	}
49	}
50
51	#define S13(X) ((X) & 0x1fff)
52	#define S5(X) ((X) & 0x1f)
53	#define IMMED 0x00002000
54	#define RD(X) ((X) << 25)
55	#define RS1(X) ((X) << 14)
56	#define RS2(X) ((X))
57	#define OP(X) ((X) << 30)
58	#define OP2(X) ((X) << 22)
59	#define OP3(X) ((X) << 19)
60	#define COND(X) (((X) & 0xf) << 25)
61	#define CBCOND(X) (((X) & 0x1f) << 25)
62	#define F1(X) OP(X)
63	#define F2(X, Y) (OP(X) | OP2(Y))
64	#define F3(X, Y) (OP(X) | OP3(Y))
65	#define ASI(X) (((X) & 0xff) << 5)
66
67	#define CONDN COND(0x0)
68	#define CONDE COND(0x1)
69	#define CONDLE COND(0x2)
70	#define CONDL COND(0x3)
71	#define CONDLEU COND(0x4)
72	#define CONDCS COND(0x5)
73	#define CONDNEG COND(0x6)
74	#define CONDVC COND(0x7)
75	#define CONDA COND(0x8)
76	#define CONDNE COND(0x9)
77	#define CONDG COND(0xa)
78	#define CONDGE COND(0xb)
79	#define CONDGU COND(0xc)
80	#define CONDCC COND(0xd)
81	#define CONDPOS COND(0xe)
82	#define CONDVS COND(0xf)
83
84	#define CONDGEU CONDCC
85	#define CONDLU CONDCS
86
87	#define WDISP22(X) (((X) >> 2) & 0x3fffff)
88	#define WDISP19(X) (((X) >> 2) & 0x7ffff)
89
90	/* The 10-bit branch displacement for CBCOND is split into two fields */
91	static u32 WDISP10(u32 off)
92	{
93	u32 ret = ((off >> 2) & 0xff) << 5;
94
95	ret |= ((off >> (2 + 8)) & 0x03) << 19;
96
97	return ret;
98	}
99
100	#define CBCONDE CBCOND(0x09)
101	#define CBCONDLE CBCOND(0x0a)
102	#define CBCONDL CBCOND(0x0b)
103	#define CBCONDLEU CBCOND(0x0c)
104	#define CBCONDCS CBCOND(0x0d)
105	#define CBCONDN CBCOND(0x0e)
106	#define CBCONDVS CBCOND(0x0f)
107	#define CBCONDNE CBCOND(0x19)
108	#define CBCONDG CBCOND(0x1a)
109	#define CBCONDGE CBCOND(0x1b)
110	#define CBCONDGU CBCOND(0x1c)
111	#define CBCONDCC CBCOND(0x1d)
112	#define CBCONDPOS CBCOND(0x1e)
113	#define CBCONDVC CBCOND(0x1f)
114
115	#define CBCONDGEU CBCONDCC
116	#define CBCONDLU CBCONDCS
117
118	#define ANNUL (1 << 29)
119	#define XCC (1 << 21)
120
121	#define BRANCH (F2(0, 1) | XCC)
122	#define CBCOND_OP (F2(0, 3) | XCC)
123
124	#define BA (BRANCH | CONDA)
125	#define BG (BRANCH | CONDG)
126	#define BL (BRANCH | CONDL)
127	#define BLE (BRANCH | CONDLE)
128	#define BGU (BRANCH | CONDGU)
129	#define BLEU (BRANCH | CONDLEU)
130	#define BGE (BRANCH | CONDGE)
131	#define BGEU (BRANCH | CONDGEU)
132	#define BLU (BRANCH | CONDLU)
133	#define BE (BRANCH | CONDE)
134	#define BNE (BRANCH | CONDNE)
135
136	#define SETHI(K, REG) \
137	(F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
138	#define OR_LO(K, REG) \
139	(F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
140
141	#define ADD F3(2, 0x00)
142	#define AND F3(2, 0x01)
143	#define ANDCC F3(2, 0x11)
144	#define OR F3(2, 0x02)
145	#define XOR F3(2, 0x03)
146	#define SUB F3(2, 0x04)
147	#define SUBCC F3(2, 0x14)
148	#define MUL F3(2, 0x0a)
149	#define MULX F3(2, 0x09)
150	#define UDIVX F3(2, 0x0d)
151	#define DIV F3(2, 0x0e)
152	#define SLL F3(2, 0x25)
153	#define SLLX (F3(2, 0x25)|(1<<12))
154	#define SRA F3(2, 0x27)
155	#define SRAX (F3(2, 0x27)|(1<<12))
156	#define SRL F3(2, 0x26)
157	#define SRLX (F3(2, 0x26)|(1<<12))
158	#define JMPL F3(2, 0x38)
159	#define SAVE F3(2, 0x3c)
160	#define RESTORE F3(2, 0x3d)
161	#define CALL F1(1)
162	#define BR F2(0, 0x01)
163	#define RD_Y F3(2, 0x28)
164	#define WR_Y F3(2, 0x30)
165
166	#define LD32 F3(3, 0x00)
167	#define LD8 F3(3, 0x01)
168	#define LD16 F3(3, 0x02)
169	#define LD64 F3(3, 0x0b)
170	#define LD64A F3(3, 0x1b)
171	#define ST8 F3(3, 0x05)
172	#define ST16 F3(3, 0x06)
173	#define ST32 F3(3, 0x04)
174	#define ST64 F3(3, 0x0e)
175
176	#define CAS F3(3, 0x3c)
177	#define CASX F3(3, 0x3e)
178
179	#define LDPTR LD64
180	#define BASE_STACKFRAME 176
181
182	#define LD32I (LD32 | IMMED)
183	#define LD8I (LD8 | IMMED)
184	#define LD16I (LD16 | IMMED)
185	#define LD64I (LD64 | IMMED)
186	#define LDPTRI (LDPTR | IMMED)
187	#define ST32I (ST32 | IMMED)
188
189	struct jit_ctx {
190	struct bpf_prog *prog;
191	unsigned int *offset;
192	int idx;
193	int epilogue_offset;
194	bool tmp_1_used;
195	bool tmp_2_used;
196	bool tmp_3_used;
197	bool saw_frame_pointer;
198	bool saw_call;
199	bool saw_tail_call;
200	u32 *image;
201	};
202
203	#define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
204	#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
205	#define TMP_REG_3 (MAX_BPF_JIT_REG + 2)
206
207	/* Map BPF registers to SPARC registers */
208	static const int bpf2sparc[] = {
209	/* return value from in-kernel function, and exit value from eBPF */
210	[BPF_REG_0] = O5,
211
212	/* arguments from eBPF program to in-kernel function */
213	[BPF_REG_1] = O0,
214	[BPF_REG_2] = O1,
215	[BPF_REG_3] = O2,
216	[BPF_REG_4] = O3,
217	[BPF_REG_5] = O4,
218
219	/* callee saved registers that in-kernel function will preserve */
220	[BPF_REG_6] = L0,
221	[BPF_REG_7] = L1,
222	[BPF_REG_8] = L2,
223	[BPF_REG_9] = L3,
224
225	/* read-only frame pointer to access stack */
226	[BPF_REG_FP] = L6,
227
228	[BPF_REG_AX] = G7,
229
230	/* temporary register for BPF JIT */
231	[TMP_REG_1] = G1,
232	[TMP_REG_2] = G2,
233	[TMP_REG_3] = G3,
234	};
235
236	static void emit(const u32 insn, struct jit_ctx *ctx)
237	{
238	if (ctx->image != NULL)
239	ctx->image[ctx->idx] = insn;
240
241	ctx->idx++;
242	}
243
244	static void emit_call(u32 *func, struct jit_ctx *ctx)
245	{
246	if (ctx->image != NULL) {
247	void *here = &ctx->image[ctx->idx];
248	unsigned int off;
249
250	off = (void *)func - here;
251	ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
252	}
253	ctx->idx++;
254	}
255
256	static void emit_nop(struct jit_ctx *ctx)
257	{
258	emit(SETHI(0, G0), ctx);
259	}
260
261	static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
262	{
263	emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
264	}
265
266	/* Emit 32-bit constant, zero extended. */
267	static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
268	{
269	emit(SETHI(K, reg), ctx);
270	emit(OR_LO(K, reg), ctx);
271	}
272
273	/* Emit 32-bit constant, sign extended. */
274	static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
275	{
276	if (K >= 0) {
277	emit(SETHI(K, reg), ctx);
278	emit(OR_LO(K, reg), ctx);
279	} else {
280	u32 hbits = ~(u32) K;
281	u32 lbits = -0x400 | (u32) K;
282
283	emit(SETHI(hbits, reg), ctx);
284	emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
285	}
286	}
287
288	static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
289	{
290	emit(insn: opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
291	}
292
293	static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
294	{
295	emit(insn: opcode | RS1(a) | RS2(b) | RD(c), ctx);
296	}
297
298	static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
299	struct jit_ctx *ctx)
300	{
301	bool small_immed = is_simm13(value: imm);
302	unsigned int insn = opcode;
303
304	insn |= RS1(dst) | RD(dst);
305	if (small_immed) {
306	emit(insn: insn | IMMED | S13(imm), ctx);
307	} else {
308	unsigned int tmp = bpf2sparc[TMP_REG_1];
309
310	ctx->tmp_1_used = true;
311
312	emit_set_const_sext(K: imm, reg: tmp, ctx);
313	emit(insn: insn | RS2(tmp), ctx);
314	}
315	}
316
317	static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
318	unsigned int dst, struct jit_ctx *ctx)
319	{
320	bool small_immed = is_simm13(value: imm);
321	unsigned int insn = opcode;
322
323	insn |= RS1(src) | RD(dst);
324	if (small_immed) {
325	emit(insn: insn | IMMED | S13(imm), ctx);
326	} else {
327	unsigned int tmp = bpf2sparc[TMP_REG_1];
328
329	ctx->tmp_1_used = true;
330
331	emit_set_const_sext(K: imm, reg: tmp, ctx);
332	emit(insn: insn | RS2(tmp), ctx);
333	}
334	}
335
336	static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
337	{
338	if (K >= 0 && is_simm13(value: K)) {
339	/* or %g0, K, DEST */
340	emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
341	} else {
342	emit_set_const(K, reg: dest, ctx);
343	}
344	}
345
346	static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
347	{
348	if (is_simm13(value: K)) {
349	/* or %g0, K, DEST */
350	emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
351	} else {
352	emit_set_const(K, reg: dest, ctx);
353	}
354	}
355
356	static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
357	{
358	if (is_simm13(value: K)) {
359	/* or %g0, K, DEST */
360	emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
361	} else {
362	emit_set_const_sext(K, reg: dest, ctx);
363	}
364	}
365
366	static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
367	int *hbsp, int *lbsp, int *abbasp)
368	{
369	int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
370	int i;
371
372	lowest_bit_set = highest_bit_set = -1;
373	i = 0;
374	do {
375	if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
376	lowest_bit_set = i;
377	if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
378	highest_bit_set = (64 - i - 1);
379	} while (++i < 32 && (highest_bit_set == -1 ||
380	lowest_bit_set == -1));
381	if (i == 32) {
382	i = 0;
383	do {
384	if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
385	lowest_bit_set = i + 32;
386	if (highest_bit_set == -1 &&
387	((low_bits >> (32 - i - 1)) & 1))
388	highest_bit_set = 32 - i - 1;
389	} while (++i < 32 && (highest_bit_set == -1 ||
390	lowest_bit_set == -1));
391	}
392
393	all_bits_between_are_set = 1;
394	for (i = lowest_bit_set; i <= highest_bit_set; i++) {
395	if (i < 32) {
396	if ((low_bits & (1 << i)) != 0)
397	continue;
398	} else {
399	if ((high_bits & (1 << (i - 32))) != 0)
400	continue;
401	}
402	all_bits_between_are_set = 0;
403	break;
404	}
405	*hbsp = highest_bit_set;
406	*lbsp = lowest_bit_set;
407	*abbasp = all_bits_between_are_set;
408	}
409
410	static unsigned long create_simple_focus_bits(unsigned long high_bits,
411	unsigned long low_bits,
412	int lowest_bit_set, int shift)
413	{
414	long hi, lo;
415
416	if (lowest_bit_set < 32) {
417	lo = (low_bits >> lowest_bit_set) << shift;
418	hi = ((high_bits << (32 - lowest_bit_set)) << shift);
419	} else {
420	lo = 0;
421	hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
422	}
423	return hi | lo;
424	}
425
426	static bool const64_is_2insns(unsigned long high_bits,
427	unsigned long low_bits)
428	{
429	int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
430
431	if (high_bits == 0 || high_bits == 0xffffffff)
432	return true;
433
434	analyze_64bit_constant(high_bits, low_bits,
435	hbsp: &highest_bit_set, lbsp: &lowest_bit_set,
436	abbasp: &all_bits_between_are_set);
437
438	if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
439	all_bits_between_are_set != 0)
440	return true;
441
442	if (highest_bit_set - lowest_bit_set < 21)
443	return true;
444
445	return false;
446	}
447
448	static void sparc_emit_set_const64_quick2(unsigned long high_bits,
449	unsigned long low_imm,
450	unsigned int dest,
451	int shift_count, struct jit_ctx *ctx)
452	{
453	emit_loadimm32(K: high_bits, dest, ctx);
454
455	/* Now shift it up into place. */
456	emit_alu_K(SLLX, dst: dest, imm: shift_count, ctx);
457
458	/* If there is a low immediate part piece, finish up by
459	* putting that in as well.
460	*/
461	if (low_imm != 0)
462	emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
463	}
464
465	static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
466	{
467	int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
468	unsigned int tmp = bpf2sparc[TMP_REG_1];
469	u32 low_bits = (K & 0xffffffff);
470	u32 high_bits = (K >> 32);
471
472	/* These two tests also take care of all of the one
473	* instruction cases.
474	*/
475	if (high_bits == 0xffffffff && (low_bits & 0x80000000))
476	return emit_loadimm_sext(K, dest, ctx);
477	if (high_bits == 0x00000000)
478	return emit_loadimm32(K, dest, ctx);
479
480	analyze_64bit_constant(high_bits, low_bits, hbsp: &highest_bit_set,
481	lbsp: &lowest_bit_set, abbasp: &all_bits_between_are_set);
482
483	/* 1) mov -1, %reg
484	* sllx %reg, shift, %reg
485	* 2) mov -1, %reg
486	* srlx %reg, shift, %reg
487	* 3) mov some_small_const, %reg
488	* sllx %reg, shift, %reg
489	*/
490	if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
491	all_bits_between_are_set != 0) ||
492	((highest_bit_set - lowest_bit_set) < 12)) {
493	int shift = lowest_bit_set;
494	long the_const = -1;
495
496	if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
497	all_bits_between_are_set == 0) {
498	the_const =
499	create_simple_focus_bits(high_bits, low_bits,
500	lowest_bit_set, shift: 0);
501	} else if (lowest_bit_set == 0)
502	shift = -(63 - highest_bit_set);
503
504	emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
505	if (shift > 0)
506	emit_alu_K(SLLX, dst: dest, imm: shift, ctx);
507	else if (shift < 0)
508	emit_alu_K(SRLX, dst: dest, imm: -shift, ctx);
509
510	return;
511	}
512
513	/* Now a range of 22 or less bits set somewhere.
514	* 1) sethi %hi(focus_bits), %reg
515	* sllx %reg, shift, %reg
516	* 2) sethi %hi(focus_bits), %reg
517	* srlx %reg, shift, %reg
518	*/
519	if ((highest_bit_set - lowest_bit_set) < 21) {
520	unsigned long focus_bits =
521	create_simple_focus_bits(high_bits, low_bits,
522	lowest_bit_set, shift: 10);
523
524	emit(SETHI(focus_bits, dest), ctx);
525
526	/* If lowest_bit_set == 10 then a sethi alone could
527	* have done it.
528	*/
529	if (lowest_bit_set < 10)
530	emit_alu_K(SRLX, dst: dest, imm: 10 - lowest_bit_set, ctx);
531	else if (lowest_bit_set > 10)
532	emit_alu_K(SLLX, dst: dest, imm: lowest_bit_set - 10, ctx);
533	return;
534	}
535
536	/* Ok, now 3 instruction sequences. */
537	if (low_bits == 0) {
538	emit_loadimm32(K: high_bits, dest, ctx);
539	emit_alu_K(SLLX, dst: dest, imm: 32, ctx);
540	return;
541	}
542
543	/* We may be able to do something quick
544	* when the constant is negated, so try that.
545	*/
546	if (const64_is_2insns(high_bits: (~high_bits) & 0xffffffff,
547	low_bits: (~low_bits) & 0xfffffc00)) {
548	/* NOTE: The trailing bits get XOR'd so we need the
549	* non-negated bits, not the negated ones.
550	*/
551	unsigned long trailing_bits = low_bits & 0x3ff;
552
553	if ((((~high_bits) & 0xffffffff) == 0 &&
554	((~low_bits) & 0x80000000) == 0) ||
555	(((~high_bits) & 0xffffffff) == 0xffffffff &&
556	((~low_bits) & 0x80000000) != 0)) {
557	unsigned long fast_int = (~low_bits & 0xffffffff);
558
559	if ((is_sethi(value: fast_int) &&
560	(~high_bits & 0xffffffff) == 0)) {
561	emit(SETHI(fast_int, dest), ctx);
562	} else if (is_simm13(value: fast_int)) {
563	emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
564	} else {
565	emit_loadimm64(K: fast_int, dest, ctx);
566	}
567	} else {
568	u64 n = ((~low_bits) & 0xfffffc00) |
569	(((unsigned long)((~high_bits) & 0xffffffff))<<32);
570	emit_loadimm64(K: n, dest, ctx);
571	}
572
573	low_bits = -0x400 | trailing_bits;
574
575	emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
576	return;
577	}
578
579	/* 1) sethi %hi(xxx), %reg
580	* or %reg, %lo(xxx), %reg
581	* sllx %reg, yyy, %reg
582	*/
583	if ((highest_bit_set - lowest_bit_set) < 32) {
584	unsigned long focus_bits =
585	create_simple_focus_bits(high_bits, low_bits,
586	lowest_bit_set, shift: 0);
587
588	/* So what we know is that the set bits straddle the
589	* middle of the 64-bit word.
590	*/
591	sparc_emit_set_const64_quick2(high_bits: focus_bits, low_imm: 0, dest,
592	shift_count: lowest_bit_set, ctx);
593	return;
594	}
595
596	/* 1) sethi %hi(high_bits), %reg
597	* or %reg, %lo(high_bits), %reg
598	* sllx %reg, 32, %reg
599	* or %reg, low_bits, %reg
600	*/
601	if (is_simm13(value: low_bits) && ((int)low_bits > 0)) {
602	sparc_emit_set_const64_quick2(high_bits, low_imm: low_bits,
603	dest, shift_count: 32, ctx);
604	return;
605	}
606
607	/* Oh well, we tried... Do a full 64-bit decomposition. */
608	ctx->tmp_1_used = true;
609
610	emit_loadimm32(K: high_bits, dest: tmp, ctx);
611	emit_loadimm32(K: low_bits, dest, ctx);
612	emit_alu_K(SLLX, dst: tmp, imm: 32, ctx);
613	emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
614	}
615
616	static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
617	struct jit_ctx *ctx)
618	{
619	unsigned int off = to_idx - from_idx;
620
621	if (br_opc & XCC)
622	emit(insn: br_opc | WDISP19(off << 2), ctx);
623	else
624	emit(insn: br_opc | WDISP22(off << 2), ctx);
625	}
626
627	static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
628	const u8 dst, const u8 src, struct jit_ctx *ctx)
629	{
630	unsigned int off = to_idx - from_idx;
631
632	emit(insn: cb_opc | WDISP10(off: off << 2) | RS1(dst) | RS2(src), ctx);
633	}
634
635	static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
636	const u8 dst, s32 imm, struct jit_ctx *ctx)
637	{
638	unsigned int off = to_idx - from_idx;
639
640	emit(insn: cb_opc | IMMED | WDISP10(off: off << 2) | RS1(dst) | S5(imm), ctx);
641	}
642
643	#define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX)
644	#define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
645
646	#define emit_cmp(R1, R2, CTX) \
647	emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
648
649	#define emit_cmpi(R1, IMM, CTX) \
650	emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
651
652	#define emit_btst(R1, R2, CTX) \
653	emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
654
655	#define emit_btsti(R1, IMM, CTX) \
656	emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
657
658	static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
659	const s32 imm, bool is_imm, int branch_dst,
660	struct jit_ctx *ctx)
661	{
662	bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
663	const u8 tmp = bpf2sparc[TMP_REG_1];
664
665	branch_dst = ctx->offset[branch_dst];
666
667	if (!is_simm10(value: branch_dst - ctx->idx) ||
668	BPF_OP(code) == BPF_JSET)
669	use_cbcond = false;
670
671	if (is_imm) {
672	bool fits = true;
673
674	if (use_cbcond) {
675	if (!is_simm5(value: imm))
676	fits = false;
677	} else if (!is_simm13(value: imm)) {
678	fits = false;
679	}
680	if (!fits) {
681	ctx->tmp_1_used = true;
682	emit_loadimm_sext(K: imm, dest: tmp, ctx);
683	src = tmp;
684	is_imm = false;
685	}
686	}
687
688	if (!use_cbcond) {
689	u32 br_opcode;
690
691	if (BPF_OP(code) == BPF_JSET) {
692	if (is_imm)
693	emit_btsti(dst, imm, ctx);
694	else
695	emit_btst(dst, src, ctx);
696	} else {
697	if (is_imm)
698	emit_cmpi(dst, imm, ctx);
699	else
700	emit_cmp(dst, src, ctx);
701	}
702	switch (BPF_OP(code)) {
703	case BPF_JEQ:
704	br_opcode = BE;
705	break;
706	case BPF_JGT:
707	br_opcode = BGU;
708	break;
709	case BPF_JLT:
710	br_opcode = BLU;
711	break;
712	case BPF_JGE:
713	br_opcode = BGEU;
714	break;
715	case BPF_JLE:
716	br_opcode = BLEU;
717	break;
718	case BPF_JSET:
719	case BPF_JNE:
720	br_opcode = BNE;
721	break;
722	case BPF_JSGT:
723	br_opcode = BG;
724	break;
725	case BPF_JSLT:
726	br_opcode = BL;
727	break;
728	case BPF_JSGE:
729	br_opcode = BGE;
730	break;
731	case BPF_JSLE:
732	br_opcode = BLE;
733	break;
734	default:
735	/* Make sure we dont leak kernel information to the
736	* user.
737	*/
738	return -EFAULT;
739	}
740	emit_branch(br_opc: br_opcode, from_idx: ctx->idx, to_idx: branch_dst, ctx);
741	emit_nop(ctx);
742	} else {
743	u32 cbcond_opcode;
744
745	switch (BPF_OP(code)) {
746	case BPF_JEQ:
747	cbcond_opcode = CBCONDE;
748	break;
749	case BPF_JGT:
750	cbcond_opcode = CBCONDGU;
751	break;
752	case BPF_JLT:
753	cbcond_opcode = CBCONDLU;
754	break;
755	case BPF_JGE:
756	cbcond_opcode = CBCONDGEU;
757	break;
758	case BPF_JLE:
759	cbcond_opcode = CBCONDLEU;
760	break;
761	case BPF_JNE:
762	cbcond_opcode = CBCONDNE;
763	break;
764	case BPF_JSGT:
765	cbcond_opcode = CBCONDG;
766	break;
767	case BPF_JSLT:
768	cbcond_opcode = CBCONDL;
769	break;
770	case BPF_JSGE:
771	cbcond_opcode = CBCONDGE;
772	break;
773	case BPF_JSLE:
774	cbcond_opcode = CBCONDLE;
775	break;
776	default:
777	/* Make sure we dont leak kernel information to the
778	* user.
779	*/
780	return -EFAULT;
781	}
782	cbcond_opcode |= CBCOND_OP;
783	if (is_imm)
784	emit_cbcondi(cb_opc: cbcond_opcode, from_idx: ctx->idx, to_idx: branch_dst,
785	dst, imm, ctx);
786	else
787	emit_cbcond(cb_opc: cbcond_opcode, from_idx: ctx->idx, to_idx: branch_dst,
788	dst, src, ctx);
789	}
790	return 0;
791	}
792
793	/* Just skip the save instruction and the ctx register move. */
794	#define BPF_TAILCALL_PROLOGUE_SKIP 32
795	#define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128)
796
797	static void build_prologue(struct jit_ctx *ctx)
798	{
799	s32 stack_needed = BASE_STACKFRAME;
800
801	if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
802	struct bpf_prog *prog = ctx->prog;
803	u32 stack_depth;
804
805	stack_depth = prog->aux->stack_depth;
806	stack_needed += round_up(stack_depth, 16);
807	}
808
809	if (ctx->saw_tail_call)
810	stack_needed += 8;
811
812	/* save %sp, -176, %sp */
813	emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
814
815	/* tail_call_cnt = 0 */
816	if (ctx->saw_tail_call) {
817	u32 off = BPF_TAILCALL_CNT_SP_OFF;
818
819	emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
820	} else {
821	emit_nop(ctx);
822	}
823	if (ctx->saw_frame_pointer) {
824	const u8 vfp = bpf2sparc[BPF_REG_FP];
825
826	emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
827	} else {
828	emit_nop(ctx);
829	}
830
831	emit_reg_move(I0, O0, ctx);
832	emit_reg_move(I1, O1, ctx);
833	emit_reg_move(I2, O2, ctx);
834	emit_reg_move(I3, O3, ctx);
835	emit_reg_move(I4, O4, ctx);
836	/* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
837	}
838
839	static void build_epilogue(struct jit_ctx *ctx)
840	{
841	ctx->epilogue_offset = ctx->idx;
842
843	/* ret (jmpl %i7 + 8, %g0) */
844	emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
845
846	/* restore %i5, %g0, %o0 */
847	emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
848	}
849
850	static void emit_tail_call(struct jit_ctx *ctx)
851	{
852	const u8 bpf_array = bpf2sparc[BPF_REG_2];
853	const u8 bpf_index = bpf2sparc[BPF_REG_3];
854	const u8 tmp = bpf2sparc[TMP_REG_1];
855	u32 off;
856
857	ctx->saw_tail_call = true;
858
859	off = offsetof(struct bpf_array, map.max_entries);
860	emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
861	emit_cmp(bpf_index, tmp, ctx);
862	#define OFFSET1 17
863	emit_branch(BGEU, from_idx: ctx->idx, to_idx: ctx->idx + OFFSET1, ctx);
864	emit_nop(ctx);
865
866	off = BPF_TAILCALL_CNT_SP_OFF;
867	emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
868	emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
869	#define OFFSET2 13
870	emit_branch(BGEU, from_idx: ctx->idx, to_idx: ctx->idx + OFFSET2, ctx);
871	emit_nop(ctx);
872
873	emit_alu_K(ADD, dst: tmp, imm: 1, ctx);
874	off = BPF_TAILCALL_CNT_SP_OFF;
875	emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
876
877	emit_alu3_K(SLL, src: bpf_index, imm: 3, dst: tmp, ctx);
878	emit_alu(ADD, src: bpf_array, dst: tmp, ctx);
879	off = offsetof(struct bpf_array, ptrs);
880	emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
881
882	emit_cmpi(tmp, 0, ctx);
883	#define OFFSET3 5
884	emit_branch(BE, from_idx: ctx->idx, to_idx: ctx->idx + OFFSET3, ctx);
885	emit_nop(ctx);
886
887	off = offsetof(struct bpf_prog, bpf_func);
888	emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
889
890	off = BPF_TAILCALL_PROLOGUE_SKIP;
891	emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
892	emit_nop(ctx);
893	}
894
895	static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
896	{
897	const u8 code = insn->code;
898	const u8 dst = bpf2sparc[insn->dst_reg];
899	const u8 src = bpf2sparc[insn->src_reg];
900	const int i = insn - ctx->prog->insnsi;
901	const s16 off = insn->off;
902	const s32 imm = insn->imm;
903
904	if (insn->src_reg == BPF_REG_FP)
905	ctx->saw_frame_pointer = true;
906
907	switch (code) {
908	/* dst = src */
909	case BPF_ALU | BPF_MOV | BPF_X:
910	emit_alu3_K(SRL, src, imm: 0, dst, ctx);
911	if (insn_is_zext(insn: &insn[1]))
912	return 1;
913	break;
914	case BPF_ALU64 | BPF_MOV | BPF_X:
915	emit_reg_move(from: src, to: dst, ctx);
916	break;
917	/* dst = dst OP src */
918	case BPF_ALU | BPF_ADD | BPF_X:
919	case BPF_ALU64 | BPF_ADD | BPF_X:
920	emit_alu(ADD, src, dst, ctx);
921	goto do_alu32_trunc;
922	case BPF_ALU | BPF_SUB | BPF_X:
923	case BPF_ALU64 | BPF_SUB | BPF_X:
924	emit_alu(SUB, src, dst, ctx);
925	goto do_alu32_trunc;
926	case BPF_ALU | BPF_AND | BPF_X:
927	case BPF_ALU64 | BPF_AND | BPF_X:
928	emit_alu(AND, src, dst, ctx);
929	goto do_alu32_trunc;
930	case BPF_ALU | BPF_OR | BPF_X:
931	case BPF_ALU64 | BPF_OR | BPF_X:
932	emit_alu(OR, src, dst, ctx);
933	goto do_alu32_trunc;
934	case BPF_ALU | BPF_XOR | BPF_X:
935	case BPF_ALU64 | BPF_XOR | BPF_X:
936	emit_alu(XOR, src, dst, ctx);
937	goto do_alu32_trunc;
938	case BPF_ALU | BPF_MUL | BPF_X:
939	emit_alu(MUL, src, dst, ctx);
940	goto do_alu32_trunc;
941	case BPF_ALU64 | BPF_MUL | BPF_X:
942	emit_alu(MULX, src, dst, ctx);
943	break;
944	case BPF_ALU | BPF_DIV | BPF_X:
945	emit_write_y(G0, ctx);
946	emit_alu(DIV, src, dst, ctx);
947	if (insn_is_zext(insn: &insn[1]))
948	return 1;
949	break;
950	case BPF_ALU64 | BPF_DIV | BPF_X:
951	emit_alu(UDIVX, src, dst, ctx);
952	break;
953	case BPF_ALU | BPF_MOD | BPF_X: {
954	const u8 tmp = bpf2sparc[TMP_REG_1];
955
956	ctx->tmp_1_used = true;
957
958	emit_write_y(G0, ctx);
959	emit_alu3(DIV, a: dst, b: src, c: tmp, ctx);
960	emit_alu3(MULX, a: tmp, b: src, c: tmp, ctx);
961	emit_alu3(SUB, a: dst, b: tmp, c: dst, ctx);
962	goto do_alu32_trunc;
963	}
964	case BPF_ALU64 | BPF_MOD | BPF_X: {
965	const u8 tmp = bpf2sparc[TMP_REG_1];
966
967	ctx->tmp_1_used = true;
968
969	emit_alu3(UDIVX, a: dst, b: src, c: tmp, ctx);
970	emit_alu3(MULX, a: tmp, b: src, c: tmp, ctx);
971	emit_alu3(SUB, a: dst, b: tmp, c: dst, ctx);
972	break;
973	}
974	case BPF_ALU | BPF_LSH | BPF_X:
975	emit_alu(SLL, src, dst, ctx);
976	goto do_alu32_trunc;
977	case BPF_ALU64 | BPF_LSH | BPF_X:
978	emit_alu(SLLX, src, dst, ctx);
979	break;
980	case BPF_ALU | BPF_RSH | BPF_X:
981	emit_alu(SRL, src, dst, ctx);
982	if (insn_is_zext(insn: &insn[1]))
983	return 1;
984	break;
985	case BPF_ALU64 | BPF_RSH | BPF_X:
986	emit_alu(SRLX, src, dst, ctx);
987	break;
988	case BPF_ALU | BPF_ARSH | BPF_X:
989	emit_alu(SRA, src, dst, ctx);
990	goto do_alu32_trunc;
991	case BPF_ALU64 | BPF_ARSH | BPF_X:
992	emit_alu(SRAX, src, dst, ctx);
993	break;
994
995	/* dst = -dst */
996	case BPF_ALU | BPF_NEG:
997	case BPF_ALU64 | BPF_NEG:
998	emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
999	goto do_alu32_trunc;
1000
1001	case BPF_ALU | BPF_END | BPF_FROM_BE:
1002	switch (imm) {
1003	case 16:
1004	emit_alu_K(SLL, dst, imm: 16, ctx);
1005	emit_alu_K(SRL, dst, imm: 16, ctx);
1006	if (insn_is_zext(insn: &insn[1]))
1007	return 1;
1008	break;
1009	case 32:
1010	if (!ctx->prog->aux->verifier_zext)
1011	emit_alu_K(SRL, dst, imm: 0, ctx);
1012	break;
1013	case 64:
1014	/* nop */
1015	break;
1016
1017	}
1018	break;
1019
1020	/* dst = BSWAP##imm(dst) */
1021	case BPF_ALU | BPF_END | BPF_FROM_LE: {
1022	const u8 tmp = bpf2sparc[TMP_REG_1];
1023	const u8 tmp2 = bpf2sparc[TMP_REG_2];
1024
1025	ctx->tmp_1_used = true;
1026	switch (imm) {
1027	case 16:
1028	emit_alu3_K(AND, src: dst, imm: 0xff, dst: tmp, ctx);
1029	emit_alu3_K(SRL, src: dst, imm: 8, dst, ctx);
1030	emit_alu3_K(AND, src: dst, imm: 0xff, dst, ctx);
1031	emit_alu3_K(SLL, src: tmp, imm: 8, dst: tmp, ctx);
1032	emit_alu(OR, src: tmp, dst, ctx);
1033	if (insn_is_zext(insn: &insn[1]))
1034	return 1;
1035	break;
1036
1037	case 32:
1038	ctx->tmp_2_used = true;
1039	emit_alu3_K(SRL, src: dst, imm: 24, dst: tmp, ctx); /* tmp = dst >> 24 */
1040	emit_alu3_K(SRL, src: dst, imm: 16, dst: tmp2, ctx); /* tmp2 = dst >> 16 */
1041	emit_alu3_K(AND, src: tmp2, imm: 0xff, dst: tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1042	emit_alu3_K(SLL, src: tmp2, imm: 8, dst: tmp2, ctx); /* tmp2 = tmp2 << 8 */
1043	emit_alu(OR, src: tmp2, dst: tmp, ctx); /* tmp = tmp | tmp2 */
1044	emit_alu3_K(SRL, src: dst, imm: 8, dst: tmp2, ctx); /* tmp2 = dst >> 8 */
1045	emit_alu3_K(AND, src: tmp2, imm: 0xff, dst: tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1046	emit_alu3_K(SLL, src: tmp2, imm: 16, dst: tmp2, ctx); /* tmp2 = tmp2 << 16 */
1047	emit_alu(OR, src: tmp2, dst: tmp, ctx); /* tmp = tmp | tmp2 */
1048	emit_alu3_K(AND, src: dst, imm: 0xff, dst, ctx); /* dst = dst & 0xff */
1049	emit_alu3_K(SLL, src: dst, imm: 24, dst, ctx); /* dst = dst << 24 */
1050	emit_alu(OR, src: tmp, dst, ctx); /* dst = dst | tmp */
1051	if (insn_is_zext(insn: &insn[1]))
1052	return 1;
1053	break;
1054
1055	case 64:
1056	emit_alu3_K(ADD, SP, imm: STACK_BIAS + 128, dst: tmp, ctx);
1057	emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1058	emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1059	break;
1060	}
1061	break;
1062	}
1063	/* dst = imm */
1064	case BPF_ALU | BPF_MOV | BPF_K:
1065	emit_loadimm32(K: imm, dest: dst, ctx);
1066	if (insn_is_zext(insn: &insn[1]))
1067	return 1;
1068	break;
1069	case BPF_ALU64 | BPF_MOV | BPF_K:
1070	emit_loadimm_sext(K: imm, dest: dst, ctx);
1071	break;
1072	/* dst = dst OP imm */
1073	case BPF_ALU | BPF_ADD | BPF_K:
1074	case BPF_ALU64 | BPF_ADD | BPF_K:
1075	emit_alu_K(ADD, dst, imm, ctx);
1076	goto do_alu32_trunc;
1077	case BPF_ALU | BPF_SUB | BPF_K:
1078	case BPF_ALU64 | BPF_SUB | BPF_K:
1079	emit_alu_K(SUB, dst, imm, ctx);
1080	goto do_alu32_trunc;
1081	case BPF_ALU | BPF_AND | BPF_K:
1082	case BPF_ALU64 | BPF_AND | BPF_K:
1083	emit_alu_K(AND, dst, imm, ctx);
1084	goto do_alu32_trunc;
1085	case BPF_ALU | BPF_OR | BPF_K:
1086	case BPF_ALU64 | BPF_OR | BPF_K:
1087	emit_alu_K(OR, dst, imm, ctx);
1088	goto do_alu32_trunc;
1089	case BPF_ALU | BPF_XOR | BPF_K:
1090	case BPF_ALU64 | BPF_XOR | BPF_K:
1091	emit_alu_K(XOR, dst, imm, ctx);
1092	goto do_alu32_trunc;
1093	case BPF_ALU | BPF_MUL | BPF_K:
1094	emit_alu_K(MUL, dst, imm, ctx);
1095	goto do_alu32_trunc;
1096	case BPF_ALU64 | BPF_MUL | BPF_K:
1097	emit_alu_K(MULX, dst, imm, ctx);
1098	break;
1099	case BPF_ALU | BPF_DIV | BPF_K:
1100	if (imm == 0)
1101	return -EINVAL;
1102
1103	emit_write_y(G0, ctx);
1104	emit_alu_K(DIV, dst, imm, ctx);
1105	goto do_alu32_trunc;
1106	case BPF_ALU64 | BPF_DIV | BPF_K:
1107	if (imm == 0)
1108	return -EINVAL;
1109
1110	emit_alu_K(UDIVX, dst, imm, ctx);
1111	break;
1112	case BPF_ALU64 | BPF_MOD | BPF_K:
1113	case BPF_ALU | BPF_MOD | BPF_K: {
1114	const u8 tmp = bpf2sparc[TMP_REG_2];
1115	unsigned int div;
1116
1117	if (imm == 0)
1118	return -EINVAL;
1119
1120	div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1121
1122	ctx->tmp_2_used = true;
1123
1124	if (BPF_CLASS(code) != BPF_ALU64)
1125	emit_write_y(G0, ctx);
1126	if (is_simm13(value: imm)) {
1127	emit(insn: div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1128	emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1129	emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1130	} else {
1131	const u8 tmp1 = bpf2sparc[TMP_REG_1];
1132
1133	ctx->tmp_1_used = true;
1134
1135	emit_set_const_sext(K: imm, reg: tmp1, ctx);
1136	emit(insn: div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1137	emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1138	emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1139	}
1140	goto do_alu32_trunc;
1141	}
1142	case BPF_ALU | BPF_LSH | BPF_K:
1143	emit_alu_K(SLL, dst, imm, ctx);
1144	goto do_alu32_trunc;
1145	case BPF_ALU64 | BPF_LSH | BPF_K:
1146	emit_alu_K(SLLX, dst, imm, ctx);
1147	break;
1148	case BPF_ALU | BPF_RSH | BPF_K:
1149	emit_alu_K(SRL, dst, imm, ctx);
1150	if (insn_is_zext(insn: &insn[1]))
1151	return 1;
1152	break;
1153	case BPF_ALU64 | BPF_RSH | BPF_K:
1154	emit_alu_K(SRLX, dst, imm, ctx);
1155	break;
1156	case BPF_ALU | BPF_ARSH | BPF_K:
1157	emit_alu_K(SRA, dst, imm, ctx);
1158	goto do_alu32_trunc;
1159	case BPF_ALU64 | BPF_ARSH | BPF_K:
1160	emit_alu_K(SRAX, dst, imm, ctx);
1161	break;
1162
1163	do_alu32_trunc:
1164	if (BPF_CLASS(code) == BPF_ALU &&
1165	!ctx->prog->aux->verifier_zext)
1166	emit_alu_K(SRL, dst, imm: 0, ctx);
1167	break;
1168
1169	/* JUMP off */
1170	case BPF_JMP | BPF_JA:
1171	emit_branch(BA, from_idx: ctx->idx, to_idx: ctx->offset[i + off], ctx);
1172	emit_nop(ctx);
1173	break;
1174	/* IF (dst COND src) JUMP off */
1175	case BPF_JMP | BPF_JEQ | BPF_X:
1176	case BPF_JMP | BPF_JGT | BPF_X:
1177	case BPF_JMP | BPF_JLT | BPF_X:
1178	case BPF_JMP | BPF_JGE | BPF_X:
1179	case BPF_JMP | BPF_JLE | BPF_X:
1180	case BPF_JMP | BPF_JNE | BPF_X:
1181	case BPF_JMP | BPF_JSGT | BPF_X:
1182	case BPF_JMP | BPF_JSLT | BPF_X:
1183	case BPF_JMP | BPF_JSGE | BPF_X:
1184	case BPF_JMP | BPF_JSLE | BPF_X:
1185	case BPF_JMP | BPF_JSET | BPF_X: {
1186	int err;
1187
1188	err = emit_compare_and_branch(code, dst, src, imm: 0, is_imm: false, branch_dst: i + off, ctx);
1189	if (err)
1190	return err;
1191	break;
1192	}
1193	/* IF (dst COND imm) JUMP off */
1194	case BPF_JMP | BPF_JEQ | BPF_K:
1195	case BPF_JMP | BPF_JGT | BPF_K:
1196	case BPF_JMP | BPF_JLT | BPF_K:
1197	case BPF_JMP | BPF_JGE | BPF_K:
1198	case BPF_JMP | BPF_JLE | BPF_K:
1199	case BPF_JMP | BPF_JNE | BPF_K:
1200	case BPF_JMP | BPF_JSGT | BPF_K:
1201	case BPF_JMP | BPF_JSLT | BPF_K:
1202	case BPF_JMP | BPF_JSGE | BPF_K:
1203	case BPF_JMP | BPF_JSLE | BPF_K:
1204	case BPF_JMP | BPF_JSET | BPF_K: {
1205	int err;
1206
1207	err = emit_compare_and_branch(code, dst, src: 0, imm, is_imm: true, branch_dst: i + off, ctx);
1208	if (err)
1209	return err;
1210	break;
1211	}
1212
1213	/* function call */
1214	case BPF_JMP | BPF_CALL:
1215	{
1216	u8 *func = ((u8 *)__bpf_call_base) + imm;
1217
1218	ctx->saw_call = true;
1219
1220	emit_call(func: (u32 *)func, ctx);
1221	emit_nop(ctx);
1222
1223	emit_reg_move(O0, to: bpf2sparc[BPF_REG_0], ctx);
1224	break;
1225	}
1226
1227	/* tail call */
1228	case BPF_JMP | BPF_TAIL_CALL:
1229	emit_tail_call(ctx);
1230	break;
1231
1232	/* function return */
1233	case BPF_JMP | BPF_EXIT:
1234	/* Optimization: when last instruction is EXIT,
1235	simply fallthrough to epilogue. */
1236	if (i == ctx->prog->len - 1)
1237	break;
1238	emit_branch(BA, from_idx: ctx->idx, to_idx: ctx->epilogue_offset, ctx);
1239	emit_nop(ctx);
1240	break;
1241
1242	/* dst = imm64 */
1243	case BPF_LD | BPF_IMM | BPF_DW:
1244	{
1245	const struct bpf_insn insn1 = insn[1];
1246	u64 imm64;
1247
1248	imm64 = (u64)insn1.imm << 32 | (u32)imm;
1249	emit_loadimm64(K: imm64, dest: dst, ctx);
1250
1251	return 1;
1252	}
1253
1254	/* LDX: dst = *(size *)(src + off) */
1255	case BPF_LDX | BPF_MEM | BPF_W:
1256	case BPF_LDX | BPF_MEM | BPF_H:
1257	case BPF_LDX | BPF_MEM | BPF_B:
1258	case BPF_LDX | BPF_MEM | BPF_DW: {
1259	const u8 tmp = bpf2sparc[TMP_REG_1];
1260	u32 opcode = 0, rs2;
1261
1262	ctx->tmp_1_used = true;
1263	switch (BPF_SIZE(code)) {
1264	case BPF_W:
1265	opcode = LD32;
1266	break;
1267	case BPF_H:
1268	opcode = LD16;
1269	break;
1270	case BPF_B:
1271	opcode = LD8;
1272	break;
1273	case BPF_DW:
1274	opcode = LD64;
1275	break;
1276	}
1277
1278	if (is_simm13(value: off)) {
1279	opcode |= IMMED;
1280	rs2 = S13(off);
1281	} else {
1282	emit_loadimm(K: off, dest: tmp, ctx);
1283	rs2 = RS2(tmp);
1284	}
1285	emit(insn: opcode | RS1(src) | rs2 | RD(dst), ctx);
1286	if (opcode != LD64 && insn_is_zext(insn: &insn[1]))
1287	return 1;
1288	break;
1289	}
1290	/* speculation barrier */
1291	case BPF_ST | BPF_NOSPEC:
1292	break;
1293	/* ST: *(size *)(dst + off) = imm */
1294	case BPF_ST | BPF_MEM | BPF_W:
1295	case BPF_ST | BPF_MEM | BPF_H:
1296	case BPF_ST | BPF_MEM | BPF_B:
1297	case BPF_ST | BPF_MEM | BPF_DW: {
1298	const u8 tmp = bpf2sparc[TMP_REG_1];
1299	const u8 tmp2 = bpf2sparc[TMP_REG_2];
1300	u32 opcode = 0, rs2;
1301
1302	if (insn->dst_reg == BPF_REG_FP)
1303	ctx->saw_frame_pointer = true;
1304
1305	ctx->tmp_2_used = true;
1306	emit_loadimm(K: imm, dest: tmp2, ctx);
1307
1308	switch (BPF_SIZE(code)) {
1309	case BPF_W:
1310	opcode = ST32;
1311	break;
1312	case BPF_H:
1313	opcode = ST16;
1314	break;
1315	case BPF_B:
1316	opcode = ST8;
1317	break;
1318	case BPF_DW:
1319	opcode = ST64;
1320	break;
1321	}
1322
1323	if (is_simm13(value: off)) {
1324	opcode |= IMMED;
1325	rs2 = S13(off);
1326	} else {
1327	ctx->tmp_1_used = true;
1328	emit_loadimm(K: off, dest: tmp, ctx);
1329	rs2 = RS2(tmp);
1330	}
1331	emit(insn: opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1332	break;
1333	}
1334
1335	/* STX: *(size *)(dst + off) = src */
1336	case BPF_STX | BPF_MEM | BPF_W:
1337	case BPF_STX | BPF_MEM | BPF_H:
1338	case BPF_STX | BPF_MEM | BPF_B:
1339	case BPF_STX | BPF_MEM | BPF_DW: {
1340	const u8 tmp = bpf2sparc[TMP_REG_1];
1341	u32 opcode = 0, rs2;
1342
1343	if (insn->dst_reg == BPF_REG_FP)
1344	ctx->saw_frame_pointer = true;
1345
1346	switch (BPF_SIZE(code)) {
1347	case BPF_W:
1348	opcode = ST32;
1349	break;
1350	case BPF_H:
1351	opcode = ST16;
1352	break;
1353	case BPF_B:
1354	opcode = ST8;
1355	break;
1356	case BPF_DW:
1357	opcode = ST64;
1358	break;
1359	}
1360	if (is_simm13(value: off)) {
1361	opcode |= IMMED;
1362	rs2 = S13(off);
1363	} else {
1364	ctx->tmp_1_used = true;
1365	emit_loadimm(K: off, dest: tmp, ctx);
1366	rs2 = RS2(tmp);
1367	}
1368	emit(insn: opcode | RS1(dst) | rs2 | RD(src), ctx);
1369	break;
1370	}
1371
1372	case BPF_STX | BPF_ATOMIC | BPF_W: {
1373	const u8 tmp = bpf2sparc[TMP_REG_1];
1374	const u8 tmp2 = bpf2sparc[TMP_REG_2];
1375	const u8 tmp3 = bpf2sparc[TMP_REG_3];
1376
1377	if (insn->imm != BPF_ADD) {
1378	pr_err_once("unknown atomic op %02x\n", insn->imm);
1379	return -EINVAL;
1380	}
1381
1382	/* lock *(u32 *)(dst + off) += src */
1383
1384	if (insn->dst_reg == BPF_REG_FP)
1385	ctx->saw_frame_pointer = true;
1386
1387	ctx->tmp_1_used = true;
1388	ctx->tmp_2_used = true;
1389	ctx->tmp_3_used = true;
1390	emit_loadimm(K: off, dest: tmp, ctx);
1391	emit_alu3(ADD, a: dst, b: tmp, c: tmp, ctx);
1392
1393	emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1394	emit_alu3(ADD, a: tmp2, b: src, c: tmp3, ctx);
1395	emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1396	emit_cmp(tmp2, tmp3, ctx);
1397	emit_branch(BNE, from_idx: 4, to_idx: 0, ctx);
1398	emit_nop(ctx);
1399	break;
1400	}
1401	/* STX XADD: lock *(u64 *)(dst + off) += src */
1402	case BPF_STX | BPF_ATOMIC | BPF_DW: {
1403	const u8 tmp = bpf2sparc[TMP_REG_1];
1404	const u8 tmp2 = bpf2sparc[TMP_REG_2];
1405	const u8 tmp3 = bpf2sparc[TMP_REG_3];
1406
1407	if (insn->imm != BPF_ADD) {
1408	pr_err_once("unknown atomic op %02x\n", insn->imm);
1409	return -EINVAL;
1410	}
1411
1412	if (insn->dst_reg == BPF_REG_FP)
1413	ctx->saw_frame_pointer = true;
1414
1415	ctx->tmp_1_used = true;
1416	ctx->tmp_2_used = true;
1417	ctx->tmp_3_used = true;
1418	emit_loadimm(K: off, dest: tmp, ctx);
1419	emit_alu3(ADD, a: dst, b: tmp, c: tmp, ctx);
1420
1421	emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1422	emit_alu3(ADD, a: tmp2, b: src, c: tmp3, ctx);
1423	emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1424	emit_cmp(tmp2, tmp3, ctx);
1425	emit_branch(BNE, from_idx: 4, to_idx: 0, ctx);
1426	emit_nop(ctx);
1427	break;
1428	}
1429
1430	default:
1431	pr_err_once("unknown opcode %02x\n", code);
1432	return -EINVAL;
1433	}
1434
1435	return 0;
1436	}
1437
1438	static int build_body(struct jit_ctx *ctx)
1439	{
1440	const struct bpf_prog *prog = ctx->prog;
1441	int i;
1442
1443	for (i = 0; i < prog->len; i++) {
1444	const struct bpf_insn *insn = &prog->insnsi[i];
1445	int ret;
1446
1447	ret = build_insn(insn, ctx);
1448
1449	if (ret > 0) {
1450	i++;
1451	ctx->offset[i] = ctx->idx;
1452	continue;
1453	}
1454	ctx->offset[i] = ctx->idx;
1455	if (ret)
1456	return ret;
1457	}
1458	return 0;
1459	}
1460
1461	static void jit_fill_hole(void *area, unsigned int size)
1462	{
1463	u32 *ptr;
1464	/* We are guaranteed to have aligned memory. */
1465	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1466	*ptr++ = 0x91d02005; /* ta 5 */
1467	}
1468
1469	bool bpf_jit_needs_zext(void)
1470	{
1471	return true;
1472	}
1473
1474	struct sparc64_jit_data {
1475	struct bpf_binary_header *header;
1476	u8 *image;
1477	struct jit_ctx ctx;
1478	};
1479
1480	struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1481	{
1482	struct bpf_prog *tmp, *orig_prog = prog;
1483	struct sparc64_jit_data *jit_data;
1484	struct bpf_binary_header *header;
1485	u32 prev_image_size, image_size;
1486	bool tmp_blinded = false;
1487	bool extra_pass = false;
1488	struct jit_ctx ctx;
1489	u8 *image_ptr;
1490	int pass, i;
1491
1492	if (!prog->jit_requested)
1493	return orig_prog;
1494
1495	tmp = bpf_jit_blind_constants(fp: prog);
1496	/* If blinding was requested and we failed during blinding,
1497	* we must fall back to the interpreter.
1498	*/
1499	if (IS_ERR(ptr: tmp))
1500	return orig_prog;
1501	if (tmp != prog) {
1502	tmp_blinded = true;
1503	prog = tmp;
1504	}
1505
1506	jit_data = prog->aux->jit_data;
1507	if (!jit_data) {
1508	jit_data = kzalloc(size: sizeof(*jit_data), GFP_KERNEL);
1509	if (!jit_data) {
1510	prog = orig_prog;
1511	goto out;
1512	}
1513	prog->aux->jit_data = jit_data;
1514	}
1515	if (jit_data->ctx.offset) {
1516	ctx = jit_data->ctx;
1517	image_ptr = jit_data->image;
1518	header = jit_data->header;
1519	extra_pass = true;
1520	image_size = sizeof(u32) * ctx.idx;
1521	prev_image_size = image_size;
1522	pass = 1;
1523	goto skip_init_ctx;
1524	}
1525
1526	memset(&ctx, 0, sizeof(ctx));
1527	ctx.prog = prog;
1528
1529	ctx.offset = kmalloc_array(n: prog->len, size: sizeof(unsigned int), GFP_KERNEL);
1530	if (ctx.offset == NULL) {
1531	prog = orig_prog;
1532	goto out_off;
1533	}
1534
1535	/* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook
1536	* the offset array so that we converge faster.
1537	*/
1538	for (i = 0; i < prog->len; i++)
1539	ctx.offset[i] = i * (12 * 4);
1540
1541	prev_image_size = ~0U;
1542	for (pass = 1; pass < 40; pass++) {
1543	ctx.idx = 0;
1544
1545	build_prologue(ctx: &ctx);
1546	if (build_body(ctx: &ctx)) {
1547	prog = orig_prog;
1548	goto out_off;
1549	}
1550	build_epilogue(ctx: &ctx);
1551
1552	if (bpf_jit_enable > 1)
1553	pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,
1554	ctx.idx * 4,
1555	ctx.tmp_1_used ? '1' : ' ',
1556	ctx.tmp_2_used ? '2' : ' ',
1557	ctx.tmp_3_used ? '3' : ' ',
1558	ctx.saw_frame_pointer ? 'F' : ' ',
1559	ctx.saw_call ? 'C' : ' ',
1560	ctx.saw_tail_call ? 'T' : ' ');
1561
1562	if (ctx.idx * 4 == prev_image_size)
1563	break;
1564	prev_image_size = ctx.idx * 4;
1565	cond_resched();
1566	}
1567
1568	/* Now we know the actual image size. */
1569	image_size = sizeof(u32) * ctx.idx;
1570	header = bpf_jit_binary_alloc(proglen: image_size, image_ptr: &image_ptr,
1571	alignment: sizeof(u32), bpf_fill_ill_insns: jit_fill_hole);
1572	if (header == NULL) {
1573	prog = orig_prog;
1574	goto out_off;
1575	}
1576
1577	ctx.image = (u32 *)image_ptr;
1578	skip_init_ctx:
1579	ctx.idx = 0;
1580
1581	build_prologue(ctx: &ctx);
1582
1583	if (build_body(ctx: &ctx)) {
1584	bpf_jit_binary_free(hdr: header);
1585	prog = orig_prog;
1586	goto out_off;
1587	}
1588
1589	build_epilogue(ctx: &ctx);
1590
1591	if (ctx.idx * 4 != prev_image_size) {
1592	pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
1593	prev_image_size, ctx.idx * 4);
1594	bpf_jit_binary_free(hdr: header);
1595	prog = orig_prog;
1596	goto out_off;
1597	}
1598
1599	if (bpf_jit_enable > 1)
1600	bpf_jit_dump(flen: prog->len, proglen: image_size, pass, image: ctx.image);
1601
1602	bpf_flush_icache(start_: header, end_: (u8 *)header + header->size);
1603
1604	if (!prog->is_func || extra_pass) {
1605	bpf_jit_binary_lock_ro(hdr: header);
1606	} else {
1607	jit_data->ctx = ctx;
1608	jit_data->image = image_ptr;
1609	jit_data->header = header;
1610	}
1611
1612	prog->bpf_func = (void *)ctx.image;
1613	prog->jited = 1;
1614	prog->jited_len = image_size;
1615
1616	if (!prog->is_func || extra_pass) {
1617	bpf_prog_fill_jited_linfo(prog, insn_to_jit_off: ctx.offset);
1618	out_off:
1619	kfree(objp: ctx.offset);
1620	kfree(objp: jit_data);
1621	prog->aux->jit_data = NULL;
1622	}
1623	out:
1624	if (tmp_blinded)
1625	bpf_jit_prog_release_other(fp: prog, fp_other: prog == orig_prog ?
1626	tmp : orig_prog);
1627	return prog;
1628	}
1629