1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* BPF Jit compiler for s390.
4	*
5	* Minimum build requirements:
6	*
7	* - HAVE_MARCH_Z196_FEATURES: laal, laalg
8	* - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj
9	* - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf
10	* - 64BIT
11	*
12	* Copyright IBM Corp. 2012,2015
13	*
14	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
15	* Michael Holzheu <holzheu@linux.vnet.ibm.com>
16	*/
17
18	#define KMSG_COMPONENT "bpf_jit"
19	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20
21	#include <linux/netdevice.h>
22	#include <linux/filter.h>
23	#include <linux/init.h>
24	#include <linux/bpf.h>
25	#include <linux/mm.h>
26	#include <linux/kernel.h>
27	#include <asm/cacheflush.h>
28	#include <asm/extable.h>
29	#include <asm/dis.h>
30	#include <asm/facility.h>
31	#include <asm/nospec-branch.h>
32	#include <asm/set_memory.h>
33	#include <asm/text-patching.h>
34	#include "bpf_jit.h"
35
36	struct bpf_jit {
37	u32 seen; /* Flags to remember seen eBPF instructions */
38	u32 seen_reg[16]; /* Array to remember which registers are used */
39	u32 *addrs; /* Array with relative instruction addresses */
40	u8 *prg_buf; /* Start of program */
41	int size; /* Size of program and literal pool */
42	int size_prg; /* Size of program */
43	int prg; /* Current position in program */
44	int lit32_start; /* Start of 32-bit literal pool */
45	int lit32; /* Current position in 32-bit literal pool */
46	int lit64_start; /* Start of 64-bit literal pool */
47	int lit64; /* Current position in 64-bit literal pool */
48	int base_ip; /* Base address for literal pool */
49	int exit_ip; /* Address of exit */
50	int r1_thunk_ip; /* Address of expoline thunk for 'br %r1' */
51	int r14_thunk_ip; /* Address of expoline thunk for 'br %r14' */
52	int tail_call_start; /* Tail call start offset */
53	int excnt; /* Number of exception table entries */
54	int prologue_plt_ret; /* Return address for prologue hotpatch PLT */
55	int prologue_plt; /* Start of prologue hotpatch PLT */
56	};
57
58	#define SEEN_MEM BIT(0) /* use mem[] for temporary storage */
59	#define SEEN_LITERAL BIT(1) /* code uses literals */
60	#define SEEN_FUNC BIT(2) /* calls C functions */
61	#define SEEN_STACK (SEEN_FUNC | SEEN_MEM)
62
63	/*
64	* s390 registers
65	*/
66	#define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */
67	#define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */
68	#define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */
69	#define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */
70	#define REG_0 REG_W0 /* Register 0 */
71	#define REG_1 REG_W1 /* Register 1 */
72	#define REG_2 BPF_REG_1 /* Register 2 */
73	#define REG_3 BPF_REG_2 /* Register 3 */
74	#define REG_4 BPF_REG_3 /* Register 4 */
75	#define REG_7 BPF_REG_6 /* Register 7 */
76	#define REG_8 BPF_REG_7 /* Register 8 */
77	#define REG_14 BPF_REG_0 /* Register 14 */
78
79	/*
80	* Mapping of BPF registers to s390 registers
81	*/
82	static const int reg2hex[] = {
83	/* Return code */
84	[BPF_REG_0] = 14,
85	/* Function parameters */
86	[BPF_REG_1] = 2,
87	[BPF_REG_2] = 3,
88	[BPF_REG_3] = 4,
89	[BPF_REG_4] = 5,
90	[BPF_REG_5] = 6,
91	/* Call saved registers */
92	[BPF_REG_6] = 7,
93	[BPF_REG_7] = 8,
94	[BPF_REG_8] = 9,
95	[BPF_REG_9] = 10,
96	/* BPF stack pointer */
97	[BPF_REG_FP] = 13,
98	/* Register for blinding */
99	[BPF_REG_AX] = 12,
100	/* Work registers for s390x backend */
101	[REG_W0] = 0,
102	[REG_W1] = 1,
103	[REG_L] = 11,
104	[REG_15] = 15,
105	};
106
107	static inline u32 reg(u32 dst_reg, u32 src_reg)
108	{
109	return reg2hex[dst_reg] << 4 | reg2hex[src_reg];
110	}
111
112	static inline u32 reg_high(u32 reg)
113	{
114	return reg2hex[reg] << 4;
115	}
116
117	static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
118	{
119	u32 r1 = reg2hex[b1];
120
121	if (r1 >= 6 && r1 <= 15 && !jit->seen_reg[r1])
122	jit->seen_reg[r1] = 1;
123	}
124
125	#define REG_SET_SEEN(b1) \
126	({ \
127	reg_set_seen(jit, b1); \
128	})
129
130	#define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
131
132	/*
133	* EMIT macros for code generation
134	*/
135
136	#define _EMIT2(op) \
137	({ \
138	if (jit->prg_buf) \
139	*(u16 *) (jit->prg_buf + jit->prg) = (op); \
140	jit->prg += 2; \
141	})
142
143	#define EMIT2(op, b1, b2) \
144	({ \
145	_EMIT2((op) | reg(b1, b2)); \
146	REG_SET_SEEN(b1); \
147	REG_SET_SEEN(b2); \
148	})
149
150	#define _EMIT4(op) \
151	({ \
152	if (jit->prg_buf) \
153	*(u32 *) (jit->prg_buf + jit->prg) = (op); \
154	jit->prg += 4; \
155	})
156
157	#define EMIT4(op, b1, b2) \
158	({ \
159	_EMIT4((op) | reg(b1, b2)); \
160	REG_SET_SEEN(b1); \
161	REG_SET_SEEN(b2); \
162	})
163
164	#define EMIT4_RRF(op, b1, b2, b3) \
165	({ \
166	_EMIT4((op) | reg_high(b3) << 8 | reg(b1, b2)); \
167	REG_SET_SEEN(b1); \
168	REG_SET_SEEN(b2); \
169	REG_SET_SEEN(b3); \
170	})
171
172	#define _EMIT4_DISP(op, disp) \
173	({ \
174	unsigned int __disp = (disp) & 0xfff; \
175	_EMIT4((op) | __disp); \
176	})
177
178	#define EMIT4_DISP(op, b1, b2, disp) \
179	({ \
180	_EMIT4_DISP((op) | reg_high(b1) << 16 | \
181	reg_high(b2) << 8, (disp)); \
182	REG_SET_SEEN(b1); \
183	REG_SET_SEEN(b2); \
184	})
185
186	#define EMIT4_IMM(op, b1, imm) \
187	({ \
188	unsigned int __imm = (imm) & 0xffff; \
189	_EMIT4((op) | reg_high(b1) << 16 | __imm); \
190	REG_SET_SEEN(b1); \
191	})
192
193	#define EMIT4_PCREL(op, pcrel) \
194	({ \
195	long __pcrel = ((pcrel) >> 1) & 0xffff; \
196	_EMIT4((op) | __pcrel); \
197	})
198
199	#define EMIT4_PCREL_RIC(op, mask, target) \
200	({ \
201	int __rel = ((target) - jit->prg) / 2; \
202	_EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \
203	})
204
205	#define _EMIT6(op1, op2) \
206	({ \
207	if (jit->prg_buf) { \
208	*(u32 *) (jit->prg_buf + jit->prg) = (op1); \
209	*(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \
210	} \
211	jit->prg += 6; \
212	})
213
214	#define _EMIT6_DISP(op1, op2, disp) \
215	({ \
216	unsigned int __disp = (disp) & 0xfff; \
217	_EMIT6((op1) | __disp, op2); \
218	})
219
220	#define _EMIT6_DISP_LH(op1, op2, disp) \
221	({ \
222	u32 _disp = (u32) (disp); \
223	unsigned int __disp_h = _disp & 0xff000; \
224	unsigned int __disp_l = _disp & 0x00fff; \
225	_EMIT6((op1) | __disp_l, (op2) | __disp_h >> 4); \
226	})
227
228	#define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \
229	({ \
230	_EMIT6_DISP_LH((op1) | reg(b1, b2) << 16 | \
231	reg_high(b3) << 8, op2, disp); \
232	REG_SET_SEEN(b1); \
233	REG_SET_SEEN(b2); \
234	REG_SET_SEEN(b3); \
235	})
236
237	#define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \
238	({ \
239	unsigned int rel = (int)((target) - jit->prg) / 2; \
240	_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \
241	(op2) | (mask) << 12); \
242	REG_SET_SEEN(b1); \
243	REG_SET_SEEN(b2); \
244	})
245
246	#define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \
247	({ \
248	unsigned int rel = (int)((target) - jit->prg) / 2; \
249	_EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \
250	(rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \
251	REG_SET_SEEN(b1); \
252	BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \
253	})
254
255	#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \
256	({ \
257	int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
258	_EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\
259	REG_SET_SEEN(b1); \
260	REG_SET_SEEN(b2); \
261	})
262
263	#define EMIT6_PCREL_RILB(op, b, target) \
264	({ \
265	unsigned int rel = (int)((target) - jit->prg) / 2; \
266	_EMIT6((op) | reg_high(b) << 16 | rel >> 16, rel & 0xffff);\
267	REG_SET_SEEN(b); \
268	})
269
270	#define EMIT6_PCREL_RIL(op, target) \
271	({ \
272	unsigned int rel = (int)((target) - jit->prg) / 2; \
273	_EMIT6((op) | rel >> 16, rel & 0xffff); \
274	})
275
276	#define EMIT6_PCREL_RILC(op, mask, target) \
277	({ \
278	EMIT6_PCREL_RIL((op) | (mask) << 20, (target)); \
279	})
280
281	#define _EMIT6_IMM(op, imm) \
282	({ \
283	unsigned int __imm = (imm); \
284	_EMIT6((op) | (__imm >> 16), __imm & 0xffff); \
285	})
286
287	#define EMIT6_IMM(op, b1, imm) \
288	({ \
289	_EMIT6_IMM((op) | reg_high(b1) << 16, imm); \
290	REG_SET_SEEN(b1); \
291	})
292
293	#define _EMIT_CONST_U32(val) \
294	({ \
295	unsigned int ret; \
296	ret = jit->lit32; \
297	if (jit->prg_buf) \
298	*(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\
299	jit->lit32 += 4; \
300	ret; \
301	})
302
303	#define EMIT_CONST_U32(val) \
304	({ \
305	jit->seen |= SEEN_LITERAL; \
306	_EMIT_CONST_U32(val) - jit->base_ip; \
307	})
308
309	#define _EMIT_CONST_U64(val) \
310	({ \
311	unsigned int ret; \
312	ret = jit->lit64; \
313	if (jit->prg_buf) \
314	*(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\
315	jit->lit64 += 8; \
316	ret; \
317	})
318
319	#define EMIT_CONST_U64(val) \
320	({ \
321	jit->seen |= SEEN_LITERAL; \
322	_EMIT_CONST_U64(val) - jit->base_ip; \
323	})
324
325	#define EMIT_ZERO(b1) \
326	({ \
327	if (!fp->aux->verifier_zext) { \
328	/* llgfr %dst,%dst (zero extend to 64 bit) */ \
329	EMIT4(0xb9160000, b1, b1); \
330	REG_SET_SEEN(b1); \
331	} \
332	})
333
334	/*
335	* Return whether this is the first pass. The first pass is special, since we
336	* don't know any sizes yet, and thus must be conservative.
337	*/
338	static bool is_first_pass(struct bpf_jit *jit)
339	{
340	return jit->size == 0;
341	}
342
343	/*
344	* Return whether this is the code generation pass. The code generation pass is
345	* special, since we should change as little as possible.
346	*/
347	static bool is_codegen_pass(struct bpf_jit *jit)
348	{
349	return jit->prg_buf;
350	}
351
352	/*
353	* Return whether "rel" can be encoded as a short PC-relative offset
354	*/
355	static bool is_valid_rel(int rel)
356	{
357	return rel >= -65536 && rel <= 65534;
358	}
359
360	/*
361	* Return whether "off" can be reached using a short PC-relative offset
362	*/
363	static bool can_use_rel(struct bpf_jit *jit, int off)
364	{
365	return is_valid_rel(rel: off - jit->prg);
366	}
367
368	/*
369	* Return whether given displacement can be encoded using
370	* Long-Displacement Facility
371	*/
372	static bool is_valid_ldisp(int disp)
373	{
374	return disp >= -524288 && disp <= 524287;
375	}
376
377	/*
378	* Return whether the next 32-bit literal pool entry can be referenced using
379	* Long-Displacement Facility
380	*/
381	static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)
382	{
383	return is_valid_ldisp(disp: jit->lit32 - jit->base_ip);
384	}
385
386	/*
387	* Return whether the next 64-bit literal pool entry can be referenced using
388	* Long-Displacement Facility
389	*/
390	static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)
391	{
392	return is_valid_ldisp(disp: jit->lit64 - jit->base_ip);
393	}
394
395	/*
396	* Fill whole space with illegal instructions
397	*/
398	static void jit_fill_hole(void *area, unsigned int size)
399	{
400	memset(area, 0, size);
401	}
402
403	/*
404	* Save registers from "rs" (register start) to "re" (register end) on stack
405	*/
406	static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
407	{
408	u32 off = STK_OFF_R6 + (rs - 6) * 8;
409
410	if (rs == re)
411	/* stg %rs,off(%r15) */
412	_EMIT6(0xe300f000 | rs << 20 | off, 0x0024);
413	else
414	/* stmg %rs,%re,off(%r15) */
415	_EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off);
416	}
417
418	/*
419	* Restore registers from "rs" (register start) to "re" (register end) on stack
420	*/
421	static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth)
422	{
423	u32 off = STK_OFF_R6 + (rs - 6) * 8;
424
425	if (jit->seen & SEEN_STACK)
426	off += STK_OFF + stack_depth;
427
428	if (rs == re)
429	/* lg %rs,off(%r15) */
430	_EMIT6(0xe300f000 | rs << 20 | off, 0x0004);
431	else
432	/* lmg %rs,%re,off(%r15) */
433	_EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off);
434	}
435
436	/*
437	* Return first seen register (from start)
438	*/
439	static int get_start(struct bpf_jit *jit, int start)
440	{
441	int i;
442
443	for (i = start; i <= 15; i++) {
444	if (jit->seen_reg[i])
445	return i;
446	}
447	return 0;
448	}
449
450	/*
451	* Return last seen register (from start) (gap >= 2)
452	*/
453	static int get_end(struct bpf_jit *jit, int start)
454	{
455	int i;
456
457	for (i = start; i < 15; i++) {
458	if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])
459	return i - 1;
460	}
461	return jit->seen_reg[15] ? 15 : 14;
462	}
463
464	#define REGS_SAVE 1
465	#define REGS_RESTORE 0
466	/*
467	* Save and restore clobbered registers (6-15) on stack.
468	* We save/restore registers in chunks with gap >= 2 registers.
469	*/
470	static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)
471	{
472	const int last = 15, save_restore_size = 6;
473	int re = 6, rs;
474
475	if (is_first_pass(jit)) {
476	/*
477	* We don't know yet which registers are used. Reserve space
478	* conservatively.
479	*/
480	jit->prg += (last - re + 1) * save_restore_size;
481	return;
482	}
483
484	do {
485	rs = get_start(jit, start: re);
486	if (!rs)
487	break;
488	re = get_end(jit, start: rs + 1);
489	if (op == REGS_SAVE)
490	save_regs(jit, rs, re);
491	else
492	restore_regs(jit, rs, re, stack_depth);
493	re++;
494	} while (re <= last);
495	}
496
497	static void bpf_skip(struct bpf_jit *jit, int size)
498	{
499	if (size >= 6 && !is_valid_rel(rel: size)) {
500	/* brcl 0xf,size */
501	EMIT6_PCREL_RIL(0xc0f4000000, size);
502	size -= 6;
503	} else if (size >= 4 && is_valid_rel(rel: size)) {
504	/* brc 0xf,size */
505	EMIT4_PCREL(0xa7f40000, size);
506	size -= 4;
507	}
508	while (size >= 2) {
509	/* bcr 0,%0 */
510	_EMIT2(0x0700);
511	size -= 2;
512	}
513	}
514
515	/*
516	* PLT for hotpatchable calls. The calling convention is the same as for the
517	* ftrace hotpatch trampolines: %r0 is return address, %r1 is clobbered.
518	*/
519	struct bpf_plt {
520	char code[16];
521	void *ret;
522	void *target;
523	} __packed;
524	extern const struct bpf_plt bpf_plt;
525	asm(
526	".pushsection .rodata\n"
527	" .balign 8\n"
528	"bpf_plt:\n"
529	" lgrl %r0,bpf_plt_ret\n"
530	" lgrl %r1,bpf_plt_target\n"
531	" br %r1\n"
532	" .balign 8\n"
533	"bpf_plt_ret: .quad 0\n"
534	"bpf_plt_target: .quad 0\n"
535	" .popsection\n"
536	);
537
538	static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target)
539	{
540	memcpy(plt, &bpf_plt, sizeof(*plt));
541	plt->ret = ret;
542	plt->target = target;
543	}
544
545	/*
546	* Emit function prologue
547	*
548	* Save registers and create stack frame if necessary.
549	* See stack frame layout description in "bpf_jit.h"!
550	*/
551	static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
552	u32 stack_depth)
553	{
554	/* No-op for hotpatching */
555	/* brcl 0,prologue_plt */
556	EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt);
557	jit->prologue_plt_ret = jit->prg;
558
559	if (!bpf_is_subprog(prog: fp)) {
560	/* Initialize the tail call counter in the main program. */
561	/* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */
562	_EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT);
563	} else {
564	/*
565	* Skip the tail call counter initialization in subprograms.
566	* Insert nops in order to have tail_call_start at a
567	* predictable offset.
568	*/
569	bpf_skip(jit, size: 6);
570	}
571	/* Tail calls have to skip above initialization */
572	jit->tail_call_start = jit->prg;
573	/* Save registers */
574	save_restore_regs(jit, REGS_SAVE, stack_depth);
575	/* Setup literal pool */
576	if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) {
577	if (!is_first_pass(jit) &&
578	is_valid_ldisp(disp: jit->size - (jit->prg + 2))) {
579	/* basr %l,0 */
580	EMIT2(0x0d00, REG_L, REG_0);
581	jit->base_ip = jit->prg;
582	} else {
583	/* larl %l,lit32_start */
584	EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start);
585	jit->base_ip = jit->lit32_start;
586	}
587	}
588	/* Setup stack and backchain */
589	if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {
590	if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
591	/* lgr %w1,%r15 (backchain) */
592	EMIT4(0xb9040000, REG_W1, REG_15);
593	/* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
594	EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
595	/* aghi %r15,-STK_OFF */
596	EMIT4_IMM(0xa70b0000, REG_15, -(STK_OFF + stack_depth));
597	if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
598	/* stg %w1,152(%r15) (backchain) */
599	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
600	REG_15, 152);
601	}
602	}
603
604	/*
605	* Emit an expoline for a jump that follows
606	*/
607	static void emit_expoline(struct bpf_jit *jit)
608	{
609	/* exrl %r0,.+10 */
610	EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
611	/* j . */
612	EMIT4_PCREL(0xa7f40000, 0);
613	}
614
615	/*
616	* Emit __s390_indirect_jump_r1 thunk if necessary
617	*/
618	static void emit_r1_thunk(struct bpf_jit *jit)
619	{
620	if (nospec_uses_trampoline()) {
621	jit->r1_thunk_ip = jit->prg;
622	emit_expoline(jit);
623	/* br %r1 */
624	_EMIT2(0x07f1);
625	}
626	}
627
628	/*
629	* Call r1 either directly or via __s390_indirect_jump_r1 thunk
630	*/
631	static void call_r1(struct bpf_jit *jit)
632	{
633	if (nospec_uses_trampoline())
634	/* brasl %r14,__s390_indirect_jump_r1 */
635	EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
636	else
637	/* basr %r14,%r1 */
638	EMIT2(0x0d00, REG_14, REG_1);
639	}
640
641	/*
642	* Function epilogue
643	*/
644	static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
645	{
646	jit->exit_ip = jit->prg;
647	/* Load exit code: lgr %r2,%b0 */
648	EMIT4(0xb9040000, REG_2, BPF_REG_0);
649	/* Restore registers */
650	save_restore_regs(jit, REGS_RESTORE, stack_depth);
651	if (nospec_uses_trampoline()) {
652	jit->r14_thunk_ip = jit->prg;
653	/* Generate __s390_indirect_jump_r14 thunk */
654	emit_expoline(jit);
655	}
656	/* br %r14 */
657	_EMIT2(0x07fe);
658
659	if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
660	emit_r1_thunk(jit);
661
662	jit->prg = ALIGN(jit->prg, 8);
663	jit->prologue_plt = jit->prg;
664	if (jit->prg_buf)
665	bpf_jit_plt(plt: (struct bpf_plt *)(jit->prg_buf + jit->prg),
666	ret: jit->prg_buf + jit->prologue_plt_ret, NULL);
667	jit->prg += sizeof(struct bpf_plt);
668	}
669
670	static int get_probe_mem_regno(const u8 *insn)
671	{
672	/*
673	* insn must point to llgc, llgh, llgf, lg, lgb, lgh or lgf, which have
674	* destination register at the same position.
675	*/
676	if (insn[0] != 0xe3) /* common prefix */
677	return -1;
678	if (insn[5] != 0x90 && /* llgc */
679	insn[5] != 0x91 && /* llgh */
680	insn[5] != 0x16 && /* llgf */
681	insn[5] != 0x04 && /* lg */
682	insn[5] != 0x77 && /* lgb */
683	insn[5] != 0x15 && /* lgh */
684	insn[5] != 0x14) /* lgf */
685	return -1;
686	return insn[1] >> 4;
687	}
688
689	bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs)
690	{
691	regs->psw.addr = extable_fixup(x);
692	regs->gprs[x->data] = 0;
693	return true;
694	}
695
696	static int bpf_jit_probe_mem(struct bpf_jit *jit, struct bpf_prog *fp,
697	int probe_prg, int nop_prg)
698	{
699	struct exception_table_entry *ex;
700	int reg, prg;
701	s64 delta;
702	u8 *insn;
703	int i;
704
705	if (!fp->aux->extable)
706	/* Do nothing during early JIT passes. */
707	return 0;
708	insn = jit->prg_buf + probe_prg;
709	reg = get_probe_mem_regno(insn);
710	if (WARN_ON_ONCE(reg < 0))
711	/* JIT bug - unexpected probe instruction. */
712	return -1;
713	if (WARN_ON_ONCE(probe_prg + insn_length(*insn) != nop_prg))
714	/* JIT bug - gap between probe and nop instructions. */
715	return -1;
716	for (i = 0; i < 2; i++) {
717	if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))
718	/* Verifier bug - not enough entries. */
719	return -1;
720	ex = &fp->aux->extable[jit->excnt];
721	/* Add extable entries for probe and nop instructions. */
722	prg = i == 0 ? probe_prg : nop_prg;
723	delta = jit->prg_buf + prg - (u8 *)&ex->insn;
724	if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX))
725	/* JIT bug - code and extable must be close. */
726	return -1;
727	ex->insn = delta;
728	/*
729	* Always land on the nop. Note that extable infrastructure
730	* ignores fixup field, it is handled by ex_handler_bpf().
731	*/
732	delta = jit->prg_buf + nop_prg - (u8 *)&ex->fixup;
733	if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX))
734	/* JIT bug - landing pad and extable must be close. */
735	return -1;
736	ex->fixup = delta;
737	ex->type = EX_TYPE_BPF;
738	ex->data = reg;
739	jit->excnt++;
740	}
741	return 0;
742	}
743
744	/*
745	* Sign-extend the register if necessary
746	*/
747	static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)
748	{
749	if (!(flags & BTF_FMODEL_SIGNED_ARG))
750	return 0;
751
752	switch (size) {
753	case 1:
754	/* lgbr %r,%r */
755	EMIT4(0xb9060000, r, r);
756	return 0;
757	case 2:
758	/* lghr %r,%r */
759	EMIT4(0xb9070000, r, r);
760	return 0;
761	case 4:
762	/* lgfr %r,%r */
763	EMIT4(0xb9140000, r, r);
764	return 0;
765	case 8:
766	return 0;
767	default:
768	return -1;
769	}
770	}
771
772	/*
773	* Compile one eBPF instruction into s390x code
774	*
775	* NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of
776	* stack space for the large switch statement.
777	*/
778	static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
779	int i, bool extra_pass, u32 stack_depth)
780	{
781	struct bpf_insn *insn = &fp->insnsi[i];
782	s32 branch_oc_off = insn->off;
783	u32 dst_reg = insn->dst_reg;
784	u32 src_reg = insn->src_reg;
785	int last, insn_count = 1;
786	u32 *addrs = jit->addrs;
787	s32 imm = insn->imm;
788	s16 off = insn->off;
789	int probe_prg = -1;
790	unsigned int mask;
791	int nop_prg;
792	int err;
793
794	if (BPF_CLASS(insn->code) == BPF_LDX &&
795	(BPF_MODE(insn->code) == BPF_PROBE_MEM ||
796	BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
797	probe_prg = jit->prg;
798
799	switch (insn->code) {
800	/*
801	* BPF_MOV
802	*/
803	case BPF_ALU | BPF_MOV | BPF_X:
804	switch (insn->off) {
805	case 0: /* DST = (u32) SRC */
806	/* llgfr %dst,%src */
807	EMIT4(0xb9160000, dst_reg, src_reg);
808	if (insn_is_zext(insn: &insn[1]))
809	insn_count = 2;
810	break;
811	case 8: /* DST = (u32)(s8) SRC */
812	/* lbr %dst,%src */
813	EMIT4(0xb9260000, dst_reg, src_reg);
814	/* llgfr %dst,%dst */
815	EMIT4(0xb9160000, dst_reg, dst_reg);
816	break;
817	case 16: /* DST = (u32)(s16) SRC */
818	/* lhr %dst,%src */
819	EMIT4(0xb9270000, dst_reg, src_reg);
820	/* llgfr %dst,%dst */
821	EMIT4(0xb9160000, dst_reg, dst_reg);
822	break;
823	}
824	break;
825	case BPF_ALU64 | BPF_MOV | BPF_X:
826	switch (insn->off) {
827	case 0: /* DST = SRC */
828	/* lgr %dst,%src */
829	EMIT4(0xb9040000, dst_reg, src_reg);
830	break;
831	case 8: /* DST = (s8) SRC */
832	/* lgbr %dst,%src */
833	EMIT4(0xb9060000, dst_reg, src_reg);
834	break;
835	case 16: /* DST = (s16) SRC */
836	/* lghr %dst,%src */
837	EMIT4(0xb9070000, dst_reg, src_reg);
838	break;
839	case 32: /* DST = (s32) SRC */
840	/* lgfr %dst,%src */
841	EMIT4(0xb9140000, dst_reg, src_reg);
842	break;
843	}
844	break;
845	case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */
846	/* llilf %dst,imm */
847	EMIT6_IMM(0xc00f0000, dst_reg, imm);
848	if (insn_is_zext(insn: &insn[1]))
849	insn_count = 2;
850	break;
851	case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */
852	/* lgfi %dst,imm */
853	EMIT6_IMM(0xc0010000, dst_reg, imm);
854	break;
855	/*
856	* BPF_LD 64
857	*/
858	case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
859	{
860	/* 16 byte instruction that uses two 'struct bpf_insn' */
861	u64 imm64;
862
863	imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32;
864	/* lgrl %dst,imm */
865	EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64));
866	insn_count = 2;
867	break;
868	}
869	/*
870	* BPF_ADD
871	*/
872	case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */
873	/* ar %dst,%src */
874	EMIT2(0x1a00, dst_reg, src_reg);
875	EMIT_ZERO(dst_reg);
876	break;
877	case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */
878	/* agr %dst,%src */
879	EMIT4(0xb9080000, dst_reg, src_reg);
880	break;
881	case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */
882	if (imm != 0) {
883	/* alfi %dst,imm */
884	EMIT6_IMM(0xc20b0000, dst_reg, imm);
885	}
886	EMIT_ZERO(dst_reg);
887	break;
888	case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */
889	if (!imm)
890	break;
891	/* agfi %dst,imm */
892	EMIT6_IMM(0xc2080000, dst_reg, imm);
893	break;
894	/*
895	* BPF_SUB
896	*/
897	case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */
898	/* sr %dst,%src */
899	EMIT2(0x1b00, dst_reg, src_reg);
900	EMIT_ZERO(dst_reg);
901	break;
902	case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */
903	/* sgr %dst,%src */
904	EMIT4(0xb9090000, dst_reg, src_reg);
905	break;
906	case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */
907	if (imm != 0) {
908	/* alfi %dst,-imm */
909	EMIT6_IMM(0xc20b0000, dst_reg, -imm);
910	}
911	EMIT_ZERO(dst_reg);
912	break;
913	case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */
914	if (!imm)
915	break;
916	if (imm == -0x80000000) {
917	/* algfi %dst,0x80000000 */
918	EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000);
919	} else {
920	/* agfi %dst,-imm */
921	EMIT6_IMM(0xc2080000, dst_reg, -imm);
922	}
923	break;
924	/*
925	* BPF_MUL
926	*/
927	case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */
928	/* msr %dst,%src */
929	EMIT4(0xb2520000, dst_reg, src_reg);
930	EMIT_ZERO(dst_reg);
931	break;
932	case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */
933	/* msgr %dst,%src */
934	EMIT4(0xb90c0000, dst_reg, src_reg);
935	break;
936	case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */
937	if (imm != 1) {
938	/* msfi %r5,imm */
939	EMIT6_IMM(0xc2010000, dst_reg, imm);
940	}
941	EMIT_ZERO(dst_reg);
942	break;
943	case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */
944	if (imm == 1)
945	break;
946	/* msgfi %dst,imm */
947	EMIT6_IMM(0xc2000000, dst_reg, imm);
948	break;
949	/*
950	* BPF_DIV / BPF_MOD
951	*/
952	case BPF_ALU | BPF_DIV | BPF_X:
953	case BPF_ALU | BPF_MOD | BPF_X:
954	{
955	int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
956
957	switch (off) {
958	case 0: /* dst = (u32) dst {/,%} (u32) src */
959	/* xr %w0,%w0 */
960	EMIT2(0x1700, REG_W0, REG_W0);
961	/* lr %w1,%dst */
962	EMIT2(0x1800, REG_W1, dst_reg);
963	/* dlr %w0,%src */
964	EMIT4(0xb9970000, REG_W0, src_reg);
965	break;
966	case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */
967	/* lgfr %r1,%dst */
968	EMIT4(0xb9140000, REG_W1, dst_reg);
969	/* dsgfr %r0,%src */
970	EMIT4(0xb91d0000, REG_W0, src_reg);
971	break;
972	}
973	/* llgfr %dst,%rc */
974	EMIT4(0xb9160000, dst_reg, rc_reg);
975	if (insn_is_zext(insn: &insn[1]))
976	insn_count = 2;
977	break;
978	}
979	case BPF_ALU64 | BPF_DIV | BPF_X:
980	case BPF_ALU64 | BPF_MOD | BPF_X:
981	{
982	int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
983
984	switch (off) {
985	case 0: /* dst = dst {/,%} src */
986	/* lghi %w0,0 */
987	EMIT4_IMM(0xa7090000, REG_W0, 0);
988	/* lgr %w1,%dst */
989	EMIT4(0xb9040000, REG_W1, dst_reg);
990	/* dlgr %w0,%src */
991	EMIT4(0xb9870000, REG_W0, src_reg);
992	break;
993	case 1: /* dst = (s64) dst {/,%} (s64) src */
994	/* lgr %w1,%dst */
995	EMIT4(0xb9040000, REG_W1, dst_reg);
996	/* dsgr %w0,%src */
997	EMIT4(0xb90d0000, REG_W0, src_reg);
998	break;
999	}
1000	/* lgr %dst,%rc */
1001	EMIT4(0xb9040000, dst_reg, rc_reg);
1002	break;
1003	}
1004	case BPF_ALU | BPF_DIV | BPF_K:
1005	case BPF_ALU | BPF_MOD | BPF_K:
1006	{
1007	int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
1008
1009	if (imm == 1) {
1010	if (BPF_OP(insn->code) == BPF_MOD)
1011	/* lghi %dst,0 */
1012	EMIT4_IMM(0xa7090000, dst_reg, 0);
1013	else
1014	EMIT_ZERO(dst_reg);
1015	break;
1016	}
1017	if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {
1018	switch (off) {
1019	case 0: /* dst = (u32) dst {/,%} (u32) imm */
1020	/* xr %w0,%w0 */
1021	EMIT2(0x1700, REG_W0, REG_W0);
1022	/* lr %w1,%dst */
1023	EMIT2(0x1800, REG_W1, dst_reg);
1024	/* dl %w0,<d(imm)>(%l) */
1025	EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0,
1026	REG_L, EMIT_CONST_U32(imm));
1027	break;
1028	case 1: /* dst = (s32) dst {/,%} (s32) imm */
1029	/* lgfr %r1,%dst */
1030	EMIT4(0xb9140000, REG_W1, dst_reg);
1031	/* dsgf %r0,<d(imm)>(%l) */
1032	EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0,
1033	REG_L, EMIT_CONST_U32(imm));
1034	break;
1035	}
1036	} else {
1037	switch (off) {
1038	case 0: /* dst = (u32) dst {/,%} (u32) imm */
1039	/* xr %w0,%w0 */
1040	EMIT2(0x1700, REG_W0, REG_W0);
1041	/* lr %w1,%dst */
1042	EMIT2(0x1800, REG_W1, dst_reg);
1043	/* lrl %dst,imm */
1044	EMIT6_PCREL_RILB(0xc40d0000, dst_reg,
1045	_EMIT_CONST_U32(imm));
1046	jit->seen |= SEEN_LITERAL;
1047	/* dlr %w0,%dst */
1048	EMIT4(0xb9970000, REG_W0, dst_reg);
1049	break;
1050	case 1: /* dst = (s32) dst {/,%} (s32) imm */
1051	/* lgfr %w1,%dst */
1052	EMIT4(0xb9140000, REG_W1, dst_reg);
1053	/* lgfrl %dst,imm */
1054	EMIT6_PCREL_RILB(0xc40c0000, dst_reg,
1055	_EMIT_CONST_U32(imm));
1056	jit->seen |= SEEN_LITERAL;
1057	/* dsgr %w0,%dst */
1058	EMIT4(0xb90d0000, REG_W0, dst_reg);
1059	break;
1060	}
1061	}
1062	/* llgfr %dst,%rc */
1063	EMIT4(0xb9160000, dst_reg, rc_reg);
1064	if (insn_is_zext(insn: &insn[1]))
1065	insn_count = 2;
1066	break;
1067	}
1068	case BPF_ALU64 | BPF_DIV | BPF_K:
1069	case BPF_ALU64 | BPF_MOD | BPF_K:
1070	{
1071	int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
1072
1073	if (imm == 1) {
1074	if (BPF_OP(insn->code) == BPF_MOD)
1075	/* lhgi %dst,0 */
1076	EMIT4_IMM(0xa7090000, dst_reg, 0);
1077	break;
1078	}
1079	if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1080	switch (off) {
1081	case 0: /* dst = dst {/,%} imm */
1082	/* lghi %w0,0 */
1083	EMIT4_IMM(0xa7090000, REG_W0, 0);
1084	/* lgr %w1,%dst */
1085	EMIT4(0xb9040000, REG_W1, dst_reg);
1086	/* dlg %w0,<d(imm)>(%l) */
1087	EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0,
1088	REG_L, EMIT_CONST_U64(imm));
1089	break;
1090	case 1: /* dst = (s64) dst {/,%} (s64) imm */
1091	/* lgr %w1,%dst */
1092	EMIT4(0xb9040000, REG_W1, dst_reg);
1093	/* dsg %w0,<d(imm)>(%l) */
1094	EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0,
1095	REG_L, EMIT_CONST_U64(imm));
1096	break;
1097	}
1098	} else {
1099	switch (off) {
1100	case 0: /* dst = dst {/,%} imm */
1101	/* lghi %w0,0 */
1102	EMIT4_IMM(0xa7090000, REG_W0, 0);
1103	/* lgr %w1,%dst */
1104	EMIT4(0xb9040000, REG_W1, dst_reg);
1105	/* lgrl %dst,imm */
1106	EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1107	_EMIT_CONST_U64(imm));
1108	jit->seen |= SEEN_LITERAL;
1109	/* dlgr %w0,%dst */
1110	EMIT4(0xb9870000, REG_W0, dst_reg);
1111	break;
1112	case 1: /* dst = (s64) dst {/,%} (s64) imm */
1113	/* lgr %w1,%dst */
1114	EMIT4(0xb9040000, REG_W1, dst_reg);
1115	/* lgrl %dst,imm */
1116	EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1117	_EMIT_CONST_U64(imm));
1118	jit->seen |= SEEN_LITERAL;
1119	/* dsgr %w0,%dst */
1120	EMIT4(0xb90d0000, REG_W0, dst_reg);
1121	break;
1122	}
1123	}
1124	/* lgr %dst,%rc */
1125	EMIT4(0xb9040000, dst_reg, rc_reg);
1126	break;
1127	}
1128	/*
1129	* BPF_AND
1130	*/
1131	case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */
1132	/* nr %dst,%src */
1133	EMIT2(0x1400, dst_reg, src_reg);
1134	EMIT_ZERO(dst_reg);
1135	break;
1136	case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
1137	/* ngr %dst,%src */
1138	EMIT4(0xb9800000, dst_reg, src_reg);
1139	break;
1140	case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */
1141	/* nilf %dst,imm */
1142	EMIT6_IMM(0xc00b0000, dst_reg, imm);
1143	EMIT_ZERO(dst_reg);
1144	break;
1145	case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
1146	if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1147	/* ng %dst,<d(imm)>(%l) */
1148	EMIT6_DISP_LH(0xe3000000, 0x0080,
1149	dst_reg, REG_0, REG_L,
1150	EMIT_CONST_U64(imm));
1151	} else {
1152	/* lgrl %w0,imm */
1153	EMIT6_PCREL_RILB(0xc4080000, REG_W0,
1154	_EMIT_CONST_U64(imm));
1155	jit->seen |= SEEN_LITERAL;
1156	/* ngr %dst,%w0 */
1157	EMIT4(0xb9800000, dst_reg, REG_W0);
1158	}
1159	break;
1160	/*
1161	* BPF_OR
1162	*/
1163	case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
1164	/* or %dst,%src */
1165	EMIT2(0x1600, dst_reg, src_reg);
1166	EMIT_ZERO(dst_reg);
1167	break;
1168	case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
1169	/* ogr %dst,%src */
1170	EMIT4(0xb9810000, dst_reg, src_reg);
1171	break;
1172	case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */
1173	/* oilf %dst,imm */
1174	EMIT6_IMM(0xc00d0000, dst_reg, imm);
1175	EMIT_ZERO(dst_reg);
1176	break;
1177	case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */
1178	if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1179	/* og %dst,<d(imm)>(%l) */
1180	EMIT6_DISP_LH(0xe3000000, 0x0081,
1181	dst_reg, REG_0, REG_L,
1182	EMIT_CONST_U64(imm));
1183	} else {
1184	/* lgrl %w0,imm */
1185	EMIT6_PCREL_RILB(0xc4080000, REG_W0,
1186	_EMIT_CONST_U64(imm));
1187	jit->seen |= SEEN_LITERAL;
1188	/* ogr %dst,%w0 */
1189	EMIT4(0xb9810000, dst_reg, REG_W0);
1190	}
1191	break;
1192	/*
1193	* BPF_XOR
1194	*/
1195	case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */
1196	/* xr %dst,%src */
1197	EMIT2(0x1700, dst_reg, src_reg);
1198	EMIT_ZERO(dst_reg);
1199	break;
1200	case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */
1201	/* xgr %dst,%src */
1202	EMIT4(0xb9820000, dst_reg, src_reg);
1203	break;
1204	case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */
1205	if (imm != 0) {
1206	/* xilf %dst,imm */
1207	EMIT6_IMM(0xc0070000, dst_reg, imm);
1208	}
1209	EMIT_ZERO(dst_reg);
1210	break;
1211	case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */
1212	if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1213	/* xg %dst,<d(imm)>(%l) */
1214	EMIT6_DISP_LH(0xe3000000, 0x0082,
1215	dst_reg, REG_0, REG_L,
1216	EMIT_CONST_U64(imm));
1217	} else {
1218	/* lgrl %w0,imm */
1219	EMIT6_PCREL_RILB(0xc4080000, REG_W0,
1220	_EMIT_CONST_U64(imm));
1221	jit->seen |= SEEN_LITERAL;
1222	/* xgr %dst,%w0 */
1223	EMIT4(0xb9820000, dst_reg, REG_W0);
1224	}
1225	break;
1226	/*
1227	* BPF_LSH
1228	*/
1229	case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */
1230	/* sll %dst,0(%src) */
1231	EMIT4_DISP(0x89000000, dst_reg, src_reg, 0);
1232	EMIT_ZERO(dst_reg);
1233	break;
1234	case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */
1235	/* sllg %dst,%dst,0(%src) */
1236	EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0);
1237	break;
1238	case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */
1239	if (imm != 0) {
1240	/* sll %dst,imm(%r0) */
1241	EMIT4_DISP(0x89000000, dst_reg, REG_0, imm);
1242	}
1243	EMIT_ZERO(dst_reg);
1244	break;
1245	case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */
1246	if (imm == 0)
1247	break;
1248	/* sllg %dst,%dst,imm(%r0) */
1249	EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm);
1250	break;
1251	/*
1252	* BPF_RSH
1253	*/
1254	case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */
1255	/* srl %dst,0(%src) */
1256	EMIT4_DISP(0x88000000, dst_reg, src_reg, 0);
1257	EMIT_ZERO(dst_reg);
1258	break;
1259	case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */
1260	/* srlg %dst,%dst,0(%src) */
1261	EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0);
1262	break;
1263	case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */
1264	if (imm != 0) {
1265	/* srl %dst,imm(%r0) */
1266	EMIT4_DISP(0x88000000, dst_reg, REG_0, imm);
1267	}
1268	EMIT_ZERO(dst_reg);
1269	break;
1270	case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */
1271	if (imm == 0)
1272	break;
1273	/* srlg %dst,%dst,imm(%r0) */
1274	EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm);
1275	break;
1276	/*
1277	* BPF_ARSH
1278	*/
1279	case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */
1280	/* sra %dst,%dst,0(%src) */
1281	EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0);
1282	EMIT_ZERO(dst_reg);
1283	break;
1284	case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */
1285	/* srag %dst,%dst,0(%src) */
1286	EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
1287	break;
1288	case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */
1289	if (imm != 0) {
1290	/* sra %dst,imm(%r0) */
1291	EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm);
1292	}
1293	EMIT_ZERO(dst_reg);
1294	break;
1295	case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */
1296	if (imm == 0)
1297	break;
1298	/* srag %dst,%dst,imm(%r0) */
1299	EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm);
1300	break;
1301	/*
1302	* BPF_NEG
1303	*/
1304	case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */
1305	/* lcr %dst,%dst */
1306	EMIT2(0x1300, dst_reg, dst_reg);
1307	EMIT_ZERO(dst_reg);
1308	break;
1309	case BPF_ALU64 | BPF_NEG: /* dst = -dst */
1310	/* lcgr %dst,%dst */
1311	EMIT4(0xb9030000, dst_reg, dst_reg);
1312	break;
1313	/*
1314	* BPF_FROM_BE/LE
1315	*/
1316	case BPF_ALU | BPF_END | BPF_FROM_BE:
1317	/* s390 is big endian, therefore only clear high order bytes */
1318	switch (imm) {
1319	case 16: /* dst = (u16) cpu_to_be16(dst) */
1320	/* llghr %dst,%dst */
1321	EMIT4(0xb9850000, dst_reg, dst_reg);
1322	if (insn_is_zext(insn: &insn[1]))
1323	insn_count = 2;
1324	break;
1325	case 32: /* dst = (u32) cpu_to_be32(dst) */
1326	if (!fp->aux->verifier_zext)
1327	/* llgfr %dst,%dst */
1328	EMIT4(0xb9160000, dst_reg, dst_reg);
1329	break;
1330	case 64: /* dst = (u64) cpu_to_be64(dst) */
1331	break;
1332	}
1333	break;
1334	case BPF_ALU | BPF_END | BPF_FROM_LE:
1335	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
1336	switch (imm) {
1337	case 16: /* dst = (u16) cpu_to_le16(dst) */
1338	/* lrvr %dst,%dst */
1339	EMIT4(0xb91f0000, dst_reg, dst_reg);
1340	/* srl %dst,16(%r0) */
1341	EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);
1342	/* llghr %dst,%dst */
1343	EMIT4(0xb9850000, dst_reg, dst_reg);
1344	if (insn_is_zext(insn: &insn[1]))
1345	insn_count = 2;
1346	break;
1347	case 32: /* dst = (u32) cpu_to_le32(dst) */
1348	/* lrvr %dst,%dst */
1349	EMIT4(0xb91f0000, dst_reg, dst_reg);
1350	if (!fp->aux->verifier_zext)
1351	/* llgfr %dst,%dst */
1352	EMIT4(0xb9160000, dst_reg, dst_reg);
1353	break;
1354	case 64: /* dst = (u64) cpu_to_le64(dst) */
1355	/* lrvgr %dst,%dst */
1356	EMIT4(0xb90f0000, dst_reg, dst_reg);
1357	break;
1358	}
1359	break;
1360	/*
1361	* BPF_NOSPEC (speculation barrier)
1362	*/
1363	case BPF_ST | BPF_NOSPEC:
1364	break;
1365	/*
1366	* BPF_ST(X)
1367	*/
1368	case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */
1369	/* stcy %src,off(%dst) */
1370	EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, REG_0, off);
1371	jit->seen |= SEEN_MEM;
1372	break;
1373	case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
1374	/* sthy %src,off(%dst) */
1375	EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, REG_0, off);
1376	jit->seen |= SEEN_MEM;
1377	break;
1378	case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
1379	/* sty %src,off(%dst) */
1380	EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, REG_0, off);
1381	jit->seen |= SEEN_MEM;
1382	break;
1383	case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
1384	/* stg %src,off(%dst) */
1385	EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, REG_0, off);
1386	jit->seen |= SEEN_MEM;
1387	break;
1388	case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
1389	/* lhi %w0,imm */
1390	EMIT4_IMM(0xa7080000, REG_W0, (u8) imm);
1391	/* stcy %w0,off(dst) */
1392	EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, REG_0, off);
1393	jit->seen |= SEEN_MEM;
1394	break;
1395	case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
1396	/* lhi %w0,imm */
1397	EMIT4_IMM(0xa7080000, REG_W0, (u16) imm);
1398	/* sthy %w0,off(dst) */
1399	EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, REG_0, off);
1400	jit->seen |= SEEN_MEM;
1401	break;
1402	case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
1403	/* llilf %w0,imm */
1404	EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm);
1405	/* sty %w0,off(%dst) */
1406	EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, REG_0, off);
1407	jit->seen |= SEEN_MEM;
1408	break;
1409	case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
1410	/* lgfi %w0,imm */
1411	EMIT6_IMM(0xc0010000, REG_W0, imm);
1412	/* stg %w0,off(%dst) */
1413	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, REG_0, off);
1414	jit->seen |= SEEN_MEM;
1415	break;
1416	/*
1417	* BPF_ATOMIC
1418	*/
1419	case BPF_STX | BPF_ATOMIC | BPF_DW:
1420	case BPF_STX | BPF_ATOMIC | BPF_W:
1421	{
1422	bool is32 = BPF_SIZE(insn->code) == BPF_W;
1423
1424	switch (insn->imm) {
1425	/* {op32|op64} {%w0|%src},%src,off(%dst) */
1426	#define EMIT_ATOMIC(op32, op64) do { \
1427	EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \
1428	(insn->imm & BPF_FETCH) ? src_reg : REG_W0, \
1429	src_reg, dst_reg, off); \
1430	if (is32 && (insn->imm & BPF_FETCH)) \
1431	EMIT_ZERO(src_reg); \
1432	} while (0)
1433	case BPF_ADD:
1434	case BPF_ADD | BPF_FETCH:
1435	/* {laal|laalg} */
1436	EMIT_ATOMIC(0x00fa, 0x00ea);
1437	break;
1438	case BPF_AND:
1439	case BPF_AND | BPF_FETCH:
1440	/* {lan|lang} */
1441	EMIT_ATOMIC(0x00f4, 0x00e4);
1442	break;
1443	case BPF_OR:
1444	case BPF_OR | BPF_FETCH:
1445	/* {lao|laog} */
1446	EMIT_ATOMIC(0x00f6, 0x00e6);
1447	break;
1448	case BPF_XOR:
1449	case BPF_XOR | BPF_FETCH:
1450	/* {lax|laxg} */
1451	EMIT_ATOMIC(0x00f7, 0x00e7);
1452	break;
1453	#undef EMIT_ATOMIC
1454	case BPF_XCHG:
1455	/* {ly|lg} %w0,off(%dst) */
1456	EMIT6_DISP_LH(0xe3000000,
1457	is32 ? 0x0058 : 0x0004, REG_W0, REG_0,
1458	dst_reg, off);
1459	/* 0: {csy|csg} %w0,%src,off(%dst) */
1460	EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030,
1461	REG_W0, src_reg, dst_reg, off);
1462	/* brc 4,0b */
1463	EMIT4_PCREL_RIC(0xa7040000, 4, jit->prg - 6);
1464	/* {llgfr|lgr} %src,%w0 */
1465	EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0);
1466	if (is32 && insn_is_zext(insn: &insn[1]))
1467	insn_count = 2;
1468	break;
1469	case BPF_CMPXCHG:
1470	/* 0: {csy|csg} %b0,%src,off(%dst) */
1471	EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030,
1472	BPF_REG_0, src_reg, dst_reg, off);
1473	break;
1474	default:
1475	pr_err("Unknown atomic operation %02x\n", insn->imm);
1476	return -1;
1477	}
1478
1479	jit->seen |= SEEN_MEM;
1480	break;
1481	}
1482	/*
1483	* BPF_LDX
1484	*/
1485	case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
1486	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1487	/* llgc %dst,0(off,%src) */
1488	EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);
1489	jit->seen |= SEEN_MEM;
1490	if (insn_is_zext(insn: &insn[1]))
1491	insn_count = 2;
1492	break;
1493	case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */
1494	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1495	/* lgb %dst,0(off,%src) */
1496	EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off);
1497	jit->seen |= SEEN_MEM;
1498	break;
1499	case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
1500	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1501	/* llgh %dst,0(off,%src) */
1502	EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);
1503	jit->seen |= SEEN_MEM;
1504	if (insn_is_zext(insn: &insn[1]))
1505	insn_count = 2;
1506	break;
1507	case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */
1508	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1509	/* lgh %dst,0(off,%src) */
1510	EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off);
1511	jit->seen |= SEEN_MEM;
1512	break;
1513	case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
1514	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1515	/* llgf %dst,off(%src) */
1516	jit->seen |= SEEN_MEM;
1517	EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
1518	if (insn_is_zext(insn: &insn[1]))
1519	insn_count = 2;
1520	break;
1521	case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */
1522	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1523	/* lgf %dst,off(%src) */
1524	jit->seen |= SEEN_MEM;
1525	EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off);
1526	break;
1527	case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
1528	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1529	/* lg %dst,0(off,%src) */
1530	jit->seen |= SEEN_MEM;
1531	EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
1532	break;
1533	/*
1534	* BPF_JMP / CALL
1535	*/
1536	case BPF_JMP | BPF_CALL:
1537	{
1538	const struct btf_func_model *m;
1539	bool func_addr_fixed;
1540	int j, ret;
1541	u64 func;
1542
1543	ret = bpf_jit_get_func_addr(prog: fp, insn, extra_pass,
1544	func_addr: &func, func_addr_fixed: &func_addr_fixed);
1545	if (ret < 0)
1546	return -1;
1547
1548	REG_SET_SEEN(BPF_REG_5);
1549	jit->seen |= SEEN_FUNC;
1550	/*
1551	* Copy the tail call counter to where the callee expects it.
1552	*
1553	* Note 1: The callee can increment the tail call counter, but
1554	* we do not load it back, since the x86 JIT does not do this
1555	* either.
1556	*
1557	* Note 2: We assume that the verifier does not let us call the
1558	* main program, which clears the tail call counter on entry.
1559	*/
1560	/* mvc STK_OFF_TCCNT(4,%r15),N(%r15) */
1561	_EMIT6(0xd203f000 | STK_OFF_TCCNT,
1562	0xf000 | (STK_OFF_TCCNT + STK_OFF + stack_depth));
1563
1564	/* Sign-extend the kfunc arguments. */
1565	if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
1566	m = bpf_jit_find_kfunc_model(prog: fp, insn);
1567	if (!m)
1568	return -1;
1569
1570	for (j = 0; j < m->nr_args; j++) {
1571	if (sign_extend(jit, r: BPF_REG_1 + j,
1572	size: m->arg_size[j],
1573	flags: m->arg_flags[j]))
1574	return -1;
1575	}
1576	}
1577
1578	/* lgrl %w1,func */
1579	EMIT6_PCREL_RILB(0xc4080000, REG_W1, _EMIT_CONST_U64(func));
1580	/* %r1() */
1581	call_r1(jit);
1582	/* lgr %b0,%r2: load return value into %b0 */
1583	EMIT4(0xb9040000, BPF_REG_0, REG_2);
1584	break;
1585	}
1586	case BPF_JMP | BPF_TAIL_CALL: {
1587	int patch_1_clrj, patch_2_clij, patch_3_brc;
1588
1589	/*
1590	* Implicit input:
1591	* B1: pointer to ctx
1592	* B2: pointer to bpf_array
1593	* B3: index in bpf_array
1594	*
1595	* if (index >= array->map.max_entries)
1596	* goto out;
1597	*/
1598
1599	/* llgf %w1,map.max_entries(%b2) */
1600	EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2,
1601	offsetof(struct bpf_array, map.max_entries));
1602	/* if ((u32)%b3 >= (u32)%w1) goto out; */
1603	/* clrj %b3,%w1,0xa,out */
1604	patch_1_clrj = jit->prg;
1605	EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa,
1606	jit->prg);
1607
1608	/*
1609	* if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT)
1610	* goto out;
1611	*/
1612
1613	if (jit->seen & SEEN_STACK)
1614	off = STK_OFF_TCCNT + STK_OFF + stack_depth;
1615	else
1616	off = STK_OFF_TCCNT;
1617	/* lhi %w0,1 */
1618	EMIT4_IMM(0xa7080000, REG_W0, 1);
1619	/* laal %w1,%w0,off(%r15) */
1620	EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off);
1621	/* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */
1622	patch_2_clij = jit->prg;
1623	EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1,
1624	2, jit->prg);
1625
1626	/*
1627	* prog = array->ptrs[index];
1628	* if (prog == NULL)
1629	* goto out;
1630	*/
1631
1632	/* llgfr %r1,%b3: %r1 = (u32) index */
1633	EMIT4(0xb9160000, REG_1, BPF_REG_3);
1634	/* sllg %r1,%r1,3: %r1 *= 8 */
1635	EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3);
1636	/* ltg %r1,prog(%b2,%r1) */
1637	EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2,
1638	REG_1, offsetof(struct bpf_array, ptrs));
1639	/* brc 0x8,out */
1640	patch_3_brc = jit->prg;
1641	EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);
1642
1643	/*
1644	* Restore registers before calling function
1645	*/
1646	save_restore_regs(jit, REGS_RESTORE, stack_depth);
1647
1648	/*
1649	* goto *(prog->bpf_func + tail_call_start);
1650	*/
1651
1652	/* lg %r1,bpf_func(%r1) */
1653	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0,
1654	offsetof(struct bpf_prog, bpf_func));
1655	if (nospec_uses_trampoline()) {
1656	jit->seen |= SEEN_FUNC;
1657	/* aghi %r1,tail_call_start */
1658	EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start);
1659	/* brcl 0xf,__s390_indirect_jump_r1 */
1660	EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip);
1661	} else {
1662	/* bc 0xf,tail_call_start(%r1) */
1663	_EMIT4(0x47f01000 + jit->tail_call_start);
1664	}
1665	/* out: */
1666	if (jit->prg_buf) {
1667	*(u16 *)(jit->prg_buf + patch_1_clrj + 2) =
1668	(jit->prg - patch_1_clrj) >> 1;
1669	*(u16 *)(jit->prg_buf + patch_2_clij + 2) =
1670	(jit->prg - patch_2_clij) >> 1;
1671	*(u16 *)(jit->prg_buf + patch_3_brc + 2) =
1672	(jit->prg - patch_3_brc) >> 1;
1673	}
1674	break;
1675	}
1676	case BPF_JMP | BPF_EXIT: /* return b0 */
1677	last = (i == fp->len - 1) ? 1 : 0;
1678	if (last)
1679	break;
1680	if (!is_first_pass(jit) && can_use_rel(jit, off: jit->exit_ip))
1681	/* brc 0xf, <exit> */
1682	EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip);
1683	else
1684	/* brcl 0xf, <exit> */
1685	EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip);
1686	break;
1687	/*
1688	* Branch relative (number of skipped instructions) to offset on
1689	* condition.
1690	*
1691	* Condition code to mask mapping:
1692	*
1693	* CC | Description | Mask
1694	* ------------------------------
1695	* 0 | Operands equal | 8
1696	* 1 | First operand low | 4
1697	* 2 | First operand high | 2
1698	* 3 | Unused | 1
1699	*
1700	* For s390x relative branches: ip = ip + off_bytes
1701	* For BPF relative branches: insn = insn + off_insns + 1
1702	*
1703	* For example for s390x with offset 0 we jump to the branch
1704	* instruction itself (loop) and for BPF with offset 0 we
1705	* branch to the instruction behind the branch.
1706	*/
1707	case BPF_JMP32 | BPF_JA: /* if (true) */
1708	branch_oc_off = imm;
1709	fallthrough;
1710	case BPF_JMP | BPF_JA: /* if (true) */
1711	mask = 0xf000; /* j */
1712	goto branch_oc;
1713	case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */
1714	case BPF_JMP32 | BPF_JSGT | BPF_K: /* ((s32) dst > (s32) imm) */
1715	mask = 0x2000; /* jh */
1716	goto branch_ks;
1717	case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */
1718	case BPF_JMP32 | BPF_JSLT | BPF_K: /* ((s32) dst < (s32) imm) */
1719	mask = 0x4000; /* jl */
1720	goto branch_ks;
1721	case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */
1722	case BPF_JMP32 | BPF_JSGE | BPF_K: /* ((s32) dst >= (s32) imm) */
1723	mask = 0xa000; /* jhe */
1724	goto branch_ks;
1725	case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */
1726	case BPF_JMP32 | BPF_JSLE | BPF_K: /* ((s32) dst <= (s32) imm) */
1727	mask = 0xc000; /* jle */
1728	goto branch_ks;
1729	case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */
1730	case BPF_JMP32 | BPF_JGT | BPF_K: /* ((u32) dst_reg > (u32) imm) */
1731	mask = 0x2000; /* jh */
1732	goto branch_ku;
1733	case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */
1734	case BPF_JMP32 | BPF_JLT | BPF_K: /* ((u32) dst_reg < (u32) imm) */
1735	mask = 0x4000; /* jl */
1736	goto branch_ku;
1737	case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */
1738	case BPF_JMP32 | BPF_JGE | BPF_K: /* ((u32) dst_reg >= (u32) imm) */
1739	mask = 0xa000; /* jhe */
1740	goto branch_ku;
1741	case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */
1742	case BPF_JMP32 | BPF_JLE | BPF_K: /* ((u32) dst_reg <= (u32) imm) */
1743	mask = 0xc000; /* jle */
1744	goto branch_ku;
1745	case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */
1746	case BPF_JMP32 | BPF_JNE | BPF_K: /* ((u32) dst_reg != (u32) imm) */
1747	mask = 0x7000; /* jne */
1748	goto branch_ku;
1749	case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */
1750	case BPF_JMP32 | BPF_JEQ | BPF_K: /* ((u32) dst_reg == (u32) imm) */
1751	mask = 0x8000; /* je */
1752	goto branch_ku;
1753	case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */
1754	case BPF_JMP32 | BPF_JSET | BPF_K: /* ((u32) dst_reg & (u32) imm) */
1755	mask = 0x7000; /* jnz */
1756	if (BPF_CLASS(insn->code) == BPF_JMP32) {
1757	/* llilf %w1,imm (load zero extend imm) */
1758	EMIT6_IMM(0xc00f0000, REG_W1, imm);
1759	/* nr %w1,%dst */
1760	EMIT2(0x1400, REG_W1, dst_reg);
1761	} else {
1762	/* lgfi %w1,imm (load sign extend imm) */
1763	EMIT6_IMM(0xc0010000, REG_W1, imm);
1764	/* ngr %w1,%dst */
1765	EMIT4(0xb9800000, REG_W1, dst_reg);
1766	}
1767	goto branch_oc;
1768
1769	case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */
1770	case BPF_JMP32 | BPF_JSGT | BPF_X: /* ((s32) dst > (s32) src) */
1771	mask = 0x2000; /* jh */
1772	goto branch_xs;
1773	case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */
1774	case BPF_JMP32 | BPF_JSLT | BPF_X: /* ((s32) dst < (s32) src) */
1775	mask = 0x4000; /* jl */
1776	goto branch_xs;
1777	case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */
1778	case BPF_JMP32 | BPF_JSGE | BPF_X: /* ((s32) dst >= (s32) src) */
1779	mask = 0xa000; /* jhe */
1780	goto branch_xs;
1781	case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */
1782	case BPF_JMP32 | BPF_JSLE | BPF_X: /* ((s32) dst <= (s32) src) */
1783	mask = 0xc000; /* jle */
1784	goto branch_xs;
1785	case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */
1786	case BPF_JMP32 | BPF_JGT | BPF_X: /* ((u32) dst > (u32) src) */
1787	mask = 0x2000; /* jh */
1788	goto branch_xu;
1789	case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */
1790	case BPF_JMP32 | BPF_JLT | BPF_X: /* ((u32) dst < (u32) src) */
1791	mask = 0x4000; /* jl */
1792	goto branch_xu;
1793	case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */
1794	case BPF_JMP32 | BPF_JGE | BPF_X: /* ((u32) dst >= (u32) src) */
1795	mask = 0xa000; /* jhe */
1796	goto branch_xu;
1797	case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */
1798	case BPF_JMP32 | BPF_JLE | BPF_X: /* ((u32) dst <= (u32) src) */
1799	mask = 0xc000; /* jle */
1800	goto branch_xu;
1801	case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */
1802	case BPF_JMP32 | BPF_JNE | BPF_X: /* ((u32) dst != (u32) src) */
1803	mask = 0x7000; /* jne */
1804	goto branch_xu;
1805	case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */
1806	case BPF_JMP32 | BPF_JEQ | BPF_X: /* ((u32) dst == (u32) src) */
1807	mask = 0x8000; /* je */
1808	goto branch_xu;
1809	case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */
1810	case BPF_JMP32 | BPF_JSET | BPF_X: /* ((u32) dst & (u32) src) */
1811	{
1812	bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
1813
1814	mask = 0x7000; /* jnz */
1815	/* nrk or ngrk %w1,%dst,%src */
1816	EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000),
1817	REG_W1, dst_reg, src_reg);
1818	goto branch_oc;
1819	branch_ks:
1820	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
1821	/* cfi or cgfi %dst,imm */
1822	EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000,
1823	dst_reg, imm);
1824	if (!is_first_pass(jit) &&
1825	can_use_rel(jit, off: addrs[i + off + 1])) {
1826	/* brc mask,off */
1827	EMIT4_PCREL_RIC(0xa7040000,
1828	mask >> 12, addrs[i + off + 1]);
1829	} else {
1830	/* brcl mask,off */
1831	EMIT6_PCREL_RILC(0xc0040000,
1832	mask >> 12, addrs[i + off + 1]);
1833	}
1834	break;
1835	branch_ku:
1836	/* lgfi %w1,imm (load sign extend imm) */
1837	src_reg = REG_1;
1838	EMIT6_IMM(0xc0010000, src_reg, imm);
1839	goto branch_xu;
1840	branch_xs:
1841	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
1842	if (!is_first_pass(jit) &&
1843	can_use_rel(jit, off: addrs[i + off + 1])) {
1844	/* crj or cgrj %dst,%src,mask,off */
1845	EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064),
1846	dst_reg, src_reg, i, off, mask);
1847	} else {
1848	/* cr or cgr %dst,%src */
1849	if (is_jmp32)
1850	EMIT2(0x1900, dst_reg, src_reg);
1851	else
1852	EMIT4(0xb9200000, dst_reg, src_reg);
1853	/* brcl mask,off */
1854	EMIT6_PCREL_RILC(0xc0040000,
1855	mask >> 12, addrs[i + off + 1]);
1856	}
1857	break;
1858	branch_xu:
1859	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
1860	if (!is_first_pass(jit) &&
1861	can_use_rel(jit, off: addrs[i + off + 1])) {
1862	/* clrj or clgrj %dst,%src,mask,off */
1863	EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065),
1864	dst_reg, src_reg, i, off, mask);
1865	} else {
1866	/* clr or clgr %dst,%src */
1867	if (is_jmp32)
1868	EMIT2(0x1500, dst_reg, src_reg);
1869	else
1870	EMIT4(0xb9210000, dst_reg, src_reg);
1871	/* brcl mask,off */
1872	EMIT6_PCREL_RILC(0xc0040000,
1873	mask >> 12, addrs[i + off + 1]);
1874	}
1875	break;
1876	branch_oc:
1877	if (!is_first_pass(jit) &&
1878	can_use_rel(jit, off: addrs[i + branch_oc_off + 1])) {
1879	/* brc mask,off */
1880	EMIT4_PCREL_RIC(0xa7040000,
1881	mask >> 12,
1882	addrs[i + branch_oc_off + 1]);
1883	} else {
1884	/* brcl mask,off */
1885	EMIT6_PCREL_RILC(0xc0040000,
1886	mask >> 12,
1887	addrs[i + branch_oc_off + 1]);
1888	}
1889	break;
1890	}
1891	default: /* too complex, give up */
1892	pr_err("Unknown opcode %02x\n", insn->code);
1893	return -1;
1894	}
1895
1896	if (probe_prg != -1) {
1897	/*
1898	* Handlers of certain exceptions leave psw.addr pointing to
1899	* the instruction directly after the failing one. Therefore,
1900	* create two exception table entries and also add a nop in
1901	* case two probing instructions come directly after each
1902	* other.
1903	*/
1904	nop_prg = jit->prg;
1905	/* bcr 0,%0 */
1906	_EMIT2(0x0700);
1907	err = bpf_jit_probe_mem(jit, fp, probe_prg, nop_prg);
1908	if (err < 0)
1909	return err;
1910	}
1911
1912	return insn_count;
1913	}
1914
1915	/*
1916	* Return whether new i-th instruction address does not violate any invariant
1917	*/
1918	static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i)
1919	{
1920	/* On the first pass anything goes */
1921	if (is_first_pass(jit))
1922	return true;
1923
1924	/* The codegen pass must not change anything */
1925	if (is_codegen_pass(jit))
1926	return jit->addrs[i] == jit->prg;
1927
1928	/* Passes in between must not increase code size */
1929	return jit->addrs[i] >= jit->prg;
1930	}
1931
1932	/*
1933	* Update the address of i-th instruction
1934	*/
1935	static int bpf_set_addr(struct bpf_jit *jit, int i)
1936	{
1937	int delta;
1938
1939	if (is_codegen_pass(jit)) {
1940	delta = jit->prg - jit->addrs[i];
1941	if (delta < 0)
1942	bpf_skip(jit, size: -delta);
1943	}
1944	if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))
1945	return -1;
1946	jit->addrs[i] = jit->prg;
1947	return 0;
1948	}
1949
1950	/*
1951	* Compile eBPF program into s390x code
1952	*/
1953	static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,
1954	bool extra_pass, u32 stack_depth)
1955	{
1956	int i, insn_count, lit32_size, lit64_size;
1957
1958	jit->lit32 = jit->lit32_start;
1959	jit->lit64 = jit->lit64_start;
1960	jit->prg = 0;
1961	jit->excnt = 0;
1962
1963	bpf_jit_prologue(jit, fp, stack_depth);
1964	if (bpf_set_addr(jit, i: 0) < 0)
1965	return -1;
1966	for (i = 0; i < fp->len; i += insn_count) {
1967	insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth);
1968	if (insn_count < 0)
1969	return -1;
1970	/* Next instruction address */
1971	if (bpf_set_addr(jit, i: i + insn_count) < 0)
1972	return -1;
1973	}
1974	bpf_jit_epilogue(jit, stack_depth);
1975
1976	lit32_size = jit->lit32 - jit->lit32_start;
1977	lit64_size = jit->lit64 - jit->lit64_start;
1978	jit->lit32_start = jit->prg;
1979	if (lit32_size)
1980	jit->lit32_start = ALIGN(jit->lit32_start, 4);
1981	jit->lit64_start = jit->lit32_start + lit32_size;
1982	if (lit64_size)
1983	jit->lit64_start = ALIGN(jit->lit64_start, 8);
1984	jit->size = jit->lit64_start + lit64_size;
1985	jit->size_prg = jit->prg;
1986
1987	if (WARN_ON_ONCE(fp->aux->extable &&
1988	jit->excnt != fp->aux->num_exentries))
1989	/* Verifier bug - too many entries. */
1990	return -1;
1991
1992	return 0;
1993	}
1994
1995	bool bpf_jit_needs_zext(void)
1996	{
1997	return true;
1998	}
1999
2000	struct s390_jit_data {
2001	struct bpf_binary_header *header;
2002	struct bpf_jit ctx;
2003	int pass;
2004	};
2005
2006	static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
2007	struct bpf_prog *fp)
2008	{
2009	struct bpf_binary_header *header;
2010	u32 extable_size;
2011	u32 code_size;
2012
2013	/* We need two entries per insn. */
2014	fp->aux->num_exentries *= 2;
2015
2016	code_size = roundup(jit->size,
2017	__alignof__(struct exception_table_entry));
2018	extable_size = fp->aux->num_exentries *
2019	sizeof(struct exception_table_entry);
2020	header = bpf_jit_binary_alloc(proglen: code_size + extable_size, image_ptr: &jit->prg_buf,
2021	alignment: 8, bpf_fill_ill_insns: jit_fill_hole);
2022	if (!header)
2023	return NULL;
2024	fp->aux->extable = (struct exception_table_entry *)
2025	(jit->prg_buf + code_size);
2026	return header;
2027	}
2028
2029	/*
2030	* Compile eBPF program "fp"
2031	*/
2032	struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
2033	{
2034	u32 stack_depth = round_up(fp->aux->stack_depth, 8);
2035	struct bpf_prog *tmp, *orig_fp = fp;
2036	struct bpf_binary_header *header;
2037	struct s390_jit_data *jit_data;
2038	bool tmp_blinded = false;
2039	bool extra_pass = false;
2040	struct bpf_jit jit;
2041	int pass;
2042
2043	if (!fp->jit_requested)
2044	return orig_fp;
2045
2046	tmp = bpf_jit_blind_constants(fp);
2047	/*
2048	* If blinding was requested and we failed during blinding,
2049	* we must fall back to the interpreter.
2050	*/
2051	if (IS_ERR(ptr: tmp))
2052	return orig_fp;
2053	if (tmp != fp) {
2054	tmp_blinded = true;
2055	fp = tmp;
2056	}
2057
2058	jit_data = fp->aux->jit_data;
2059	if (!jit_data) {
2060	jit_data = kzalloc(size: sizeof(*jit_data), GFP_KERNEL);
2061	if (!jit_data) {
2062	fp = orig_fp;
2063	goto out;
2064	}
2065	fp->aux->jit_data = jit_data;
2066	}
2067	if (jit_data->ctx.addrs) {
2068	jit = jit_data->ctx;
2069	header = jit_data->header;
2070	extra_pass = true;
2071	pass = jit_data->pass + 1;
2072	goto skip_init_ctx;
2073	}
2074
2075	memset(&jit, 0, sizeof(jit));
2076	jit.addrs = kvcalloc(n: fp->len + 1, size: sizeof(*jit.addrs), GFP_KERNEL);
2077	if (jit.addrs == NULL) {
2078	fp = orig_fp;
2079	goto free_addrs;
2080	}
2081	/*
2082	* Three initial passes:
2083	* - 1/2: Determine clobbered registers
2084	* - 3: Calculate program size and addrs array
2085	*/
2086	for (pass = 1; pass <= 3; pass++) {
2087	if (bpf_jit_prog(jit: &jit, fp, extra_pass, stack_depth)) {
2088	fp = orig_fp;
2089	goto free_addrs;
2090	}
2091	}
2092	/*
2093	* Final pass: Allocate and generate program
2094	*/
2095	header = bpf_jit_alloc(jit: &jit, fp);
2096	if (!header) {
2097	fp = orig_fp;
2098	goto free_addrs;
2099	}
2100	skip_init_ctx:
2101	if (bpf_jit_prog(jit: &jit, fp, extra_pass, stack_depth)) {
2102	bpf_jit_binary_free(hdr: header);
2103	fp = orig_fp;
2104	goto free_addrs;
2105	}
2106	if (bpf_jit_enable > 1) {
2107	bpf_jit_dump(flen: fp->len, proglen: jit.size, pass, image: jit.prg_buf);
2108	print_fn_code(jit.prg_buf, jit.size_prg);
2109	}
2110	if (!fp->is_func || extra_pass) {
2111	bpf_jit_binary_lock_ro(hdr: header);
2112	} else {
2113	jit_data->header = header;
2114	jit_data->ctx = jit;
2115	jit_data->pass = pass;
2116	}
2117	fp->bpf_func = (void *) jit.prg_buf;
2118	fp->jited = 1;
2119	fp->jited_len = jit.size;
2120
2121	if (!fp->is_func || extra_pass) {
2122	bpf_prog_fill_jited_linfo(prog: fp, insn_to_jit_off: jit.addrs + 1);
2123	free_addrs:
2124	kvfree(addr: jit.addrs);
2125	kfree(objp: jit_data);
2126	fp->aux->jit_data = NULL;
2127	}
2128	out:
2129	if (tmp_blinded)
2130	bpf_jit_prog_release_other(fp, fp_other: fp == orig_fp ?
2131	tmp : orig_fp);
2132	return fp;
2133	}
2134
2135	bool bpf_jit_supports_kfunc_call(void)
2136	{
2137	return true;
2138	}
2139
2140	bool bpf_jit_supports_far_kfunc_call(void)
2141	{
2142	return true;
2143	}
2144
2145	int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
2146	void *old_addr, void *new_addr)
2147	{
2148	struct bpf_plt expected_plt, current_plt, new_plt, *plt;
2149	struct {
2150	u16 opc;
2151	s32 disp;
2152	} __packed insn;
2153	char *ret;
2154	int err;
2155
2156	/* Verify the branch to be patched. */
2157	err = copy_from_kernel_nofault(dst: &insn, src: ip, size: sizeof(insn));
2158	if (err < 0)
2159	return err;
2160	if (insn.opc != (0xc004 | (old_addr ? 0xf0 : 0)))
2161	return -EINVAL;
2162
2163	if (t == BPF_MOD_JUMP &&
2164	insn.disp == ((char *)new_addr - (char *)ip) >> 1) {
2165	/*
2166	* The branch already points to the destination,
2167	* there is no PLT.
2168	*/
2169	} else {
2170	/* Verify the PLT. */
2171	plt = ip + (insn.disp << 1);
2172	err = copy_from_kernel_nofault(dst: &current_plt, src: plt,
2173	size: sizeof(current_plt));
2174	if (err < 0)
2175	return err;
2176	ret = (char *)ip + 6;
2177	bpf_jit_plt(plt: &expected_plt, ret, target: old_addr);
2178	if (memcmp(p: &current_plt, q: &expected_plt, size: sizeof(current_plt)))
2179	return -EINVAL;
2180	/* Adjust the call address. */
2181	bpf_jit_plt(plt: &new_plt, ret, target: new_addr);
2182	s390_kernel_write(&plt->target, &new_plt.target,
2183	sizeof(void *));
2184	}
2185
2186	/* Adjust the mask of the branch. */
2187	insn.opc = 0xc004 | (new_addr ? 0xf0 : 0);
2188	s390_kernel_write((char *)ip + 1, (char *)&insn.opc + 1, 1);
2189
2190	/* Make the new code visible to the other CPUs. */
2191	text_poke_sync_lock();
2192
2193	return 0;
2194	}
2195
2196	struct bpf_tramp_jit {
2197	struct bpf_jit common;
2198	int orig_stack_args_off;/* Offset of arguments placed on stack by the
2199	* func_addr's original caller
2200	*/
2201	int stack_size; /* Trampoline stack size */
2202	int backchain_off; /* Offset of backchain */
2203	int stack_args_off; /* Offset of stack arguments for calling
2204	* func_addr, has to be at the top
2205	*/
2206	int reg_args_off; /* Offset of register arguments for calling
2207	* func_addr
2208	*/
2209	int ip_off; /* For bpf_get_func_ip(), has to be at
2210	* (ctx - 16)
2211	*/
2212	int arg_cnt_off; /* For bpf_get_func_arg_cnt(), has to be at
2213	* (ctx - 8)
2214	*/
2215	int bpf_args_off; /* Offset of BPF_PROG context, which consists
2216	* of BPF arguments followed by return value
2217	*/
2218	int retval_off; /* Offset of return value (see above) */
2219	int r7_r8_off; /* Offset of saved %r7 and %r8, which are used
2220	* for __bpf_prog_enter() return value and
2221	* func_addr respectively
2222	*/
2223	int run_ctx_off; /* Offset of struct bpf_tramp_run_ctx */
2224	int tccnt_off; /* Offset of saved tailcall counter */
2225	int r14_off; /* Offset of saved %r14, has to be at the
2226	* bottom */
2227	int do_fexit; /* do_fexit: label */
2228	};
2229
2230	static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val)
2231	{
2232	/* llihf %dst_reg,val_hi */
2233	EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32));
2234	/* oilf %rdst_reg,val_lo */
2235	EMIT6_IMM(0xc00d0000, dst_reg, val);
2236	}
2237
2238	static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
2239	const struct btf_func_model *m,
2240	struct bpf_tramp_link *tlink, bool save_ret)
2241	{
2242	struct bpf_jit *jit = &tjit->common;
2243	int cookie_off = tjit->run_ctx_off +
2244	offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
2245	struct bpf_prog *p = tlink->link.prog;
2246	int patch;
2247
2248	/*
2249	* run_ctx.cookie = tlink->cookie;
2250	*/
2251
2252	/* %r0 = tlink->cookie */
2253	load_imm64(jit, REG_W0, val: tlink->cookie);
2254	/* stg %r0,cookie_off(%r15) */
2255	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off);
2256
2257	/*
2258	* if ((start = __bpf_prog_enter(p, &run_ctx)) == 0)
2259	* goto skip;
2260	*/
2261
2262	/* %r1 = __bpf_prog_enter */
2263	load_imm64(jit, REG_1, val: (u64)bpf_trampoline_enter(prog: p));
2264	/* %r2 = p */
2265	load_imm64(jit, REG_2, val: (u64)p);
2266	/* la %r3,run_ctx_off(%r15) */
2267	EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off);
2268	/* %r1() */
2269	call_r1(jit);
2270	/* ltgr %r7,%r2 */
2271	EMIT4(0xb9020000, REG_7, REG_2);
2272	/* brcl 8,skip */
2273	patch = jit->prg;
2274	EMIT6_PCREL_RILC(0xc0040000, 8, 0);
2275
2276	/*
2277	* retval = bpf_func(args, p->insnsi);
2278	*/
2279
2280	/* %r1 = p->bpf_func */
2281	load_imm64(jit, REG_1, val: (u64)p->bpf_func);
2282	/* la %r2,bpf_args_off(%r15) */
2283	EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off);
2284	/* %r3 = p->insnsi */
2285	if (!p->jited)
2286	load_imm64(jit, REG_3, val: (u64)p->insnsi);
2287	/* %r1() */
2288	call_r1(jit);
2289	/* stg %r2,retval_off(%r15) */
2290	if (save_ret) {
2291	if (sign_extend(jit, REG_2, size: m->ret_size, flags: m->ret_flags))
2292	return -1;
2293	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15,
2294	tjit->retval_off);
2295	}
2296
2297	/* skip: */
2298	if (jit->prg_buf)
2299	*(u32 *)&jit->prg_buf[patch + 2] = (jit->prg - patch) >> 1;
2300
2301	/*
2302	* __bpf_prog_exit(p, start, &run_ctx);
2303	*/
2304
2305	/* %r1 = __bpf_prog_exit */
2306	load_imm64(jit, REG_1, val: (u64)bpf_trampoline_exit(prog: p));
2307	/* %r2 = p */
2308	load_imm64(jit, REG_2, val: (u64)p);
2309	/* lgr %r3,%r7 */
2310	EMIT4(0xb9040000, REG_3, REG_7);
2311	/* la %r4,run_ctx_off(%r15) */
2312	EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off);
2313	/* %r1() */
2314	call_r1(jit);
2315
2316	return 0;
2317	}
2318
2319	static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size)
2320	{
2321	int stack_offset = tjit->stack_size;
2322
2323	tjit->stack_size += size;
2324	return stack_offset;
2325	}
2326
2327	/* ABI uses %r2 - %r6 for parameter passing. */
2328	#define MAX_NR_REG_ARGS 5
2329
2330	/* The "L" field of the "mvc" instruction is 8 bits. */
2331	#define MAX_MVC_SIZE 256
2332	#define MAX_NR_STACK_ARGS (MAX_MVC_SIZE / sizeof(u64))
2333
2334	/* -mfentry generates a 6-byte nop on s390x. */
2335	#define S390X_PATCH_SIZE 6
2336
2337	static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
2338	struct bpf_tramp_jit *tjit,
2339	const struct btf_func_model *m,
2340	u32 flags,
2341	struct bpf_tramp_links *tlinks,
2342	void *func_addr)
2343	{
2344	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
2345	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
2346	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
2347	int nr_bpf_args, nr_reg_args, nr_stack_args;
2348	struct bpf_jit *jit = &tjit->common;
2349	int arg, bpf_arg_off;
2350	int i, j;
2351
2352	/* Support as many stack arguments as "mvc" instruction can handle. */
2353	nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS);
2354	nr_stack_args = m->nr_args - nr_reg_args;
2355	if (nr_stack_args > MAX_NR_STACK_ARGS)
2356	return -ENOTSUPP;
2357
2358	/* Return to %r14, since func_addr and %r0 are not available. */
2359	if ((!func_addr && !(flags & BPF_TRAMP_F_ORIG_STACK)) ||
2360	(flags & BPF_TRAMP_F_INDIRECT))
2361	flags |= BPF_TRAMP_F_SKIP_FRAME;
2362
2363	/*
2364	* Compute how many arguments we need to pass to BPF programs.
2365	* BPF ABI mirrors that of x86_64: arguments that are 16 bytes or
2366	* smaller are packed into 1 or 2 registers; larger arguments are
2367	* passed via pointers.
2368	* In s390x ABI, arguments that are 8 bytes or smaller are packed into
2369	* a register; larger arguments are passed via pointers.
2370	* We need to deal with this difference.
2371	*/
2372	nr_bpf_args = 0;
2373	for (i = 0; i < m->nr_args; i++) {
2374	if (m->arg_size[i] <= 8)
2375	nr_bpf_args += 1;
2376	else if (m->arg_size[i] <= 16)
2377	nr_bpf_args += 2;
2378	else
2379	return -ENOTSUPP;
2380	}
2381
2382	/*
2383	* Calculate the stack layout.
2384	*/
2385
2386	/*
2387	* Allocate STACK_FRAME_OVERHEAD bytes for the callees. As the s390x
2388	* ABI requires, put our backchain at the end of the allocated memory.
2389	*/
2390	tjit->stack_size = STACK_FRAME_OVERHEAD;
2391	tjit->backchain_off = tjit->stack_size - sizeof(u64);
2392	tjit->stack_args_off = alloc_stack(tjit, size: nr_stack_args * sizeof(u64));
2393	tjit->reg_args_off = alloc_stack(tjit, size: nr_reg_args * sizeof(u64));
2394	tjit->ip_off = alloc_stack(tjit, size: sizeof(u64));
2395	tjit->arg_cnt_off = alloc_stack(tjit, size: sizeof(u64));
2396	tjit->bpf_args_off = alloc_stack(tjit, size: nr_bpf_args * sizeof(u64));
2397	tjit->retval_off = alloc_stack(tjit, size: sizeof(u64));
2398	tjit->r7_r8_off = alloc_stack(tjit, size: 2 * sizeof(u64));
2399	tjit->run_ctx_off = alloc_stack(tjit,
2400	size: sizeof(struct bpf_tramp_run_ctx));
2401	tjit->tccnt_off = alloc_stack(tjit, size: sizeof(u64));
2402	tjit->r14_off = alloc_stack(tjit, size: sizeof(u64) * 2);
2403	/*
2404	* In accordance with the s390x ABI, the caller has allocated
2405	* STACK_FRAME_OVERHEAD bytes for us. 8 of them contain the caller's
2406	* backchain, and the rest we can use.
2407	*/
2408	tjit->stack_size -= STACK_FRAME_OVERHEAD - sizeof(u64);
2409	tjit->orig_stack_args_off = tjit->stack_size + STACK_FRAME_OVERHEAD;
2410
2411	/* lgr %r1,%r15 */
2412	EMIT4(0xb9040000, REG_1, REG_15);
2413	/* aghi %r15,-stack_size */
2414	EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size);
2415	/* stg %r1,backchain_off(%r15) */
2416	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15,
2417	tjit->backchain_off);
2418	/* mvc tccnt_off(4,%r15),stack_size+STK_OFF_TCCNT(%r15) */
2419	_EMIT6(0xd203f000 | tjit->tccnt_off,
2420	0xf000 | (tjit->stack_size + STK_OFF_TCCNT));
2421	/* stmg %r2,%rN,fwd_reg_args_off(%r15) */
2422	if (nr_reg_args)
2423	EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2,
2424	REG_2 + (nr_reg_args - 1), REG_15,
2425	tjit->reg_args_off);
2426	for (i = 0, j = 0; i < m->nr_args; i++) {
2427	if (i < MAX_NR_REG_ARGS)
2428	arg = REG_2 + i;
2429	else
2430	arg = tjit->orig_stack_args_off +
2431	(i - MAX_NR_REG_ARGS) * sizeof(u64);
2432	bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64);
2433	if (m->arg_size[i] <= 8) {
2434	if (i < MAX_NR_REG_ARGS)
2435	/* stg %arg,bpf_arg_off(%r15) */
2436	EMIT6_DISP_LH(0xe3000000, 0x0024, arg,
2437	REG_0, REG_15, bpf_arg_off);
2438	else
2439	/* mvc bpf_arg_off(8,%r15),arg(%r15) */
2440	_EMIT6(0xd207f000 | bpf_arg_off,
2441	0xf000 | arg);
2442	j += 1;
2443	} else {
2444	if (i < MAX_NR_REG_ARGS) {
2445	/* mvc bpf_arg_off(16,%r15),0(%arg) */
2446	_EMIT6(0xd20ff000 | bpf_arg_off,
2447	reg2hex[arg] << 12);
2448	} else {
2449	/* lg %r1,arg(%r15) */
2450	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_0,
2451	REG_15, arg);
2452	/* mvc bpf_arg_off(16,%r15),0(%r1) */
2453	_EMIT6(0xd20ff000 | bpf_arg_off, 0x1000);
2454	}
2455	j += 2;
2456	}
2457	}
2458	/* stmg %r7,%r8,r7_r8_off(%r15) */
2459	EMIT6_DISP_LH(0xeb000000, 0x0024, REG_7, REG_8, REG_15,
2460	tjit->r7_r8_off);
2461	/* stg %r14,r14_off(%r15) */
2462	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off);
2463
2464	if (flags & BPF_TRAMP_F_ORIG_STACK) {
2465	/*
2466	* The ftrace trampoline puts the return address (which is the
2467	* address of the original function + S390X_PATCH_SIZE) into
2468	* %r0; see ftrace_shared_hotpatch_trampoline_br and
2469	* ftrace_init_nop() for details.
2470	*/
2471
2472	/* lgr %r8,%r0 */
2473	EMIT4(0xb9040000, REG_8, REG_0);
2474	} else {
2475	/* %r8 = func_addr + S390X_PATCH_SIZE */
2476	load_imm64(jit, REG_8, val: (u64)func_addr + S390X_PATCH_SIZE);
2477	}
2478
2479	/*
2480	* ip = func_addr;
2481	* arg_cnt = m->nr_args;
2482	*/
2483
2484	if (flags & BPF_TRAMP_F_IP_ARG) {
2485	/* %r0 = func_addr */
2486	load_imm64(jit, REG_0, val: (u64)func_addr);
2487	/* stg %r0,ip_off(%r15) */
2488	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15,
2489	tjit->ip_off);
2490	}
2491	/* lghi %r0,nr_bpf_args */
2492	EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args);
2493	/* stg %r0,arg_cnt_off(%r15) */
2494	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15,
2495	tjit->arg_cnt_off);
2496
2497	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2498	/*
2499	* __bpf_tramp_enter(im);
2500	*/
2501
2502	/* %r1 = __bpf_tramp_enter */
2503	load_imm64(jit, REG_1, val: (u64)__bpf_tramp_enter);
2504	/* %r2 = im */
2505	load_imm64(jit, REG_2, val: (u64)im);
2506	/* %r1() */
2507	call_r1(jit);
2508	}
2509
2510	for (i = 0; i < fentry->nr_links; i++)
2511	if (invoke_bpf_prog(tjit, m, tlink: fentry->links[i],
2512	save_ret: flags & BPF_TRAMP_F_RET_FENTRY_RET))
2513	return -EINVAL;
2514
2515	if (fmod_ret->nr_links) {
2516	/*
2517	* retval = 0;
2518	*/
2519
2520	/* xc retval_off(8,%r15),retval_off(%r15) */
2521	_EMIT6(0xd707f000 | tjit->retval_off,
2522	0xf000 | tjit->retval_off);
2523
2524	for (i = 0; i < fmod_ret->nr_links; i++) {
2525	if (invoke_bpf_prog(tjit, m, tlink: fmod_ret->links[i], save_ret: true))
2526	return -EINVAL;
2527
2528	/*
2529	* if (retval)
2530	* goto do_fexit;
2531	*/
2532
2533	/* ltg %r0,retval_off(%r15) */
2534	EMIT6_DISP_LH(0xe3000000, 0x0002, REG_0, REG_0, REG_15,
2535	tjit->retval_off);
2536	/* brcl 7,do_fexit */
2537	EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit);
2538	}
2539	}
2540
2541	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2542	/*
2543	* retval = func_addr(args);
2544	*/
2545
2546	/* lmg %r2,%rN,reg_args_off(%r15) */
2547	if (nr_reg_args)
2548	EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2,
2549	REG_2 + (nr_reg_args - 1), REG_15,
2550	tjit->reg_args_off);
2551	/* mvc stack_args_off(N,%r15),orig_stack_args_off(%r15) */
2552	if (nr_stack_args)
2553	_EMIT6(0xd200f000 |
2554	(nr_stack_args * sizeof(u64) - 1) << 16 |
2555	tjit->stack_args_off,
2556	0xf000 | tjit->orig_stack_args_off);
2557	/* mvc STK_OFF_TCCNT(4,%r15),tccnt_off(%r15) */
2558	_EMIT6(0xd203f000 | STK_OFF_TCCNT, 0xf000 | tjit->tccnt_off);
2559	/* lgr %r1,%r8 */
2560	EMIT4(0xb9040000, REG_1, REG_8);
2561	/* %r1() */
2562	call_r1(jit);
2563	/* stg %r2,retval_off(%r15) */
2564	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15,
2565	tjit->retval_off);
2566
2567	im->ip_after_call = jit->prg_buf + jit->prg;
2568
2569	/*
2570	* The following nop will be patched by bpf_tramp_image_put().
2571	*/
2572
2573	/* brcl 0,im->ip_epilogue */
2574	EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue);
2575	}
2576
2577	/* do_fexit: */
2578	tjit->do_fexit = jit->prg;
2579	for (i = 0; i < fexit->nr_links; i++)
2580	if (invoke_bpf_prog(tjit, m, tlink: fexit->links[i], save_ret: false))
2581	return -EINVAL;
2582
2583	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2584	im->ip_epilogue = jit->prg_buf + jit->prg;
2585
2586	/*
2587	* __bpf_tramp_exit(im);
2588	*/
2589
2590	/* %r1 = __bpf_tramp_exit */
2591	load_imm64(jit, REG_1, val: (u64)__bpf_tramp_exit);
2592	/* %r2 = im */
2593	load_imm64(jit, REG_2, val: (u64)im);
2594	/* %r1() */
2595	call_r1(jit);
2596	}
2597
2598	/* lmg %r2,%rN,reg_args_off(%r15) */
2599	if ((flags & BPF_TRAMP_F_RESTORE_REGS) && nr_reg_args)
2600	EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2,
2601	REG_2 + (nr_reg_args - 1), REG_15,
2602	tjit->reg_args_off);
2603	/* lgr %r1,%r8 */
2604	if (!(flags & BPF_TRAMP_F_SKIP_FRAME))
2605	EMIT4(0xb9040000, REG_1, REG_8);
2606	/* lmg %r7,%r8,r7_r8_off(%r15) */
2607	EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15,
2608	tjit->r7_r8_off);
2609	/* lg %r14,r14_off(%r15) */
2610	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off);
2611	/* lg %r2,retval_off(%r15) */
2612	if (flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET))
2613	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15,
2614	tjit->retval_off);
2615	/* mvc stack_size+STK_OFF_TCCNT(4,%r15),tccnt_off(%r15) */
2616	_EMIT6(0xd203f000 | (tjit->stack_size + STK_OFF_TCCNT),
2617	0xf000 | tjit->tccnt_off);
2618	/* aghi %r15,stack_size */
2619	EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size);
2620	/* Emit an expoline for the following indirect jump. */
2621	if (nospec_uses_trampoline())
2622	emit_expoline(jit);
2623	if (flags & BPF_TRAMP_F_SKIP_FRAME)
2624	/* br %r14 */
2625	_EMIT2(0x07fe);
2626	else
2627	/* br %r1 */
2628	_EMIT2(0x07f1);
2629
2630	emit_r1_thunk(jit);
2631
2632	return 0;
2633	}
2634
2635	int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
2636	struct bpf_tramp_links *tlinks, void *orig_call)
2637	{
2638	struct bpf_tramp_image im;
2639	struct bpf_tramp_jit tjit;
2640	int ret;
2641
2642	memset(&tjit, 0, sizeof(tjit));
2643
2644	ret = __arch_prepare_bpf_trampoline(im: &im, tjit: &tjit, m, flags,
2645	tlinks, func_addr: orig_call);
2646
2647	return ret < 0 ? ret : tjit.common.prg;
2648	}
2649
2650	int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
2651	void *image_end, const struct btf_func_model *m,
2652	u32 flags, struct bpf_tramp_links *tlinks,
2653	void *func_addr)
2654	{
2655	struct bpf_tramp_jit tjit;
2656	int ret;
2657
2658	/* Compute offsets, check whether the code fits. */
2659	memset(&tjit, 0, sizeof(tjit));
2660	ret = __arch_prepare_bpf_trampoline(im, tjit: &tjit, m, flags,
2661	tlinks, func_addr);
2662
2663	if (ret < 0)
2664	return ret;
2665	if (tjit.common.prg > (char *)image_end - (char *)image)
2666	/*
2667	* Use the same error code as for exceeding
2668	* BPF_MAX_TRAMP_LINKS.
2669	*/
2670	return -E2BIG;
2671
2672	tjit.common.prg = 0;
2673	tjit.common.prg_buf = image;
2674	ret = __arch_prepare_bpf_trampoline(im, tjit: &tjit, m, flags,
2675	tlinks, func_addr);
2676
2677	return ret < 0 ? ret : tjit.common.prg;
2678	}
2679
2680	bool bpf_jit_supports_subprog_tailcalls(void)
2681	{
2682	return true;
2683	}
2684