1/* SPDX-License-Identifier: GPL-2.0 */
2#include <asm/ptrace.h>
3
4#include "bpf_jit_32.h"
5
6#define SAVE_SZ 96
7#define SCRATCH_OFF 72
8#define BE_PTR(label) be label
9#define SIGN_EXTEND(reg)
10
11#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
12
13 .text
14 .globl bpf_jit_load_word
15bpf_jit_load_word:
16 cmp r_OFF, 0
17 bl bpf_slow_path_word_neg
18 nop
19 .globl bpf_jit_load_word_positive_offset
20bpf_jit_load_word_positive_offset:
21 sub r_HEADLEN, r_OFF, r_TMP
22 cmp r_TMP, 3
23 ble bpf_slow_path_word
24 add r_SKB_DATA, r_OFF, r_TMP
25 andcc r_TMP, 3, %g0
26 bne load_word_unaligned
27 nop
28 retl
29 ld [r_TMP], r_A
30load_word_unaligned:
31 ldub [r_TMP + 0x0], r_OFF
32 ldub [r_TMP + 0x1], r_TMP2
33 sll r_OFF, 8, r_OFF
34 or r_OFF, r_TMP2, r_OFF
35 ldub [r_TMP + 0x2], r_TMP2
36 sll r_OFF, 8, r_OFF
37 or r_OFF, r_TMP2, r_OFF
38 ldub [r_TMP + 0x3], r_TMP2
39 sll r_OFF, 8, r_OFF
40 retl
41 or r_OFF, r_TMP2, r_A
42
43 .globl bpf_jit_load_half
44bpf_jit_load_half:
45 cmp r_OFF, 0
46 bl bpf_slow_path_half_neg
47 nop
48 .globl bpf_jit_load_half_positive_offset
49bpf_jit_load_half_positive_offset:
50 sub r_HEADLEN, r_OFF, r_TMP
51 cmp r_TMP, 1
52 ble bpf_slow_path_half
53 add r_SKB_DATA, r_OFF, r_TMP
54 andcc r_TMP, 1, %g0
55 bne load_half_unaligned
56 nop
57 retl
58 lduh [r_TMP], r_A
59load_half_unaligned:
60 ldub [r_TMP + 0x0], r_OFF
61 ldub [r_TMP + 0x1], r_TMP2
62 sll r_OFF, 8, r_OFF
63 retl
64 or r_OFF, r_TMP2, r_A
65
66 .globl bpf_jit_load_byte
67bpf_jit_load_byte:
68 cmp r_OFF, 0
69 bl bpf_slow_path_byte_neg
70 nop
71 .globl bpf_jit_load_byte_positive_offset
72bpf_jit_load_byte_positive_offset:
73 cmp r_OFF, r_HEADLEN
74 bge bpf_slow_path_byte
75 nop
76 retl
77 ldub [r_SKB_DATA + r_OFF], r_A
78
79 .globl bpf_jit_load_byte_msh
80bpf_jit_load_byte_msh:
81 cmp r_OFF, 0
82 bl bpf_slow_path_byte_msh_neg
83 nop
84 .globl bpf_jit_load_byte_msh_positive_offset
85bpf_jit_load_byte_msh_positive_offset:
86 cmp r_OFF, r_HEADLEN
87 bge bpf_slow_path_byte_msh
88 nop
89 ldub [r_SKB_DATA + r_OFF], r_OFF
90 and r_OFF, 0xf, r_OFF
91 retl
92 sll r_OFF, 2, r_X
93
94#define bpf_slow_path_common(LEN) \
95 save %sp, -SAVE_SZ, %sp; \
96 mov %i0, %o0; \
97 mov r_OFF, %o1; \
98 add %fp, SCRATCH_OFF, %o2; \
99 call skb_copy_bits; \
100 mov (LEN), %o3; \
101 cmp %o0, 0; \
102 restore;
103
104bpf_slow_path_word:
105 bpf_slow_path_common(4)
106 bl bpf_error
107 ld [%sp + SCRATCH_OFF], r_A
108 retl
109 nop
110bpf_slow_path_half:
111 bpf_slow_path_common(2)
112 bl bpf_error
113 lduh [%sp + SCRATCH_OFF], r_A
114 retl
115 nop
116bpf_slow_path_byte:
117 bpf_slow_path_common(1)
118 bl bpf_error
119 ldub [%sp + SCRATCH_OFF], r_A
120 retl
121 nop
122bpf_slow_path_byte_msh:
123 bpf_slow_path_common(1)
124 bl bpf_error
125 ldub [%sp + SCRATCH_OFF], r_A
126 and r_OFF, 0xf, r_OFF
127 retl
128 sll r_OFF, 2, r_X
129
130#define bpf_negative_common(LEN) \
131 save %sp, -SAVE_SZ, %sp; \
132 mov %i0, %o0; \
133 mov r_OFF, %o1; \
134 SIGN_EXTEND(%o1); \
135 call bpf_internal_load_pointer_neg_helper; \
136 mov (LEN), %o2; \
137 mov %o0, r_TMP; \
138 cmp %o0, 0; \
139 BE_PTR(bpf_error); \
140 restore;
141
142bpf_slow_path_word_neg:
143 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
144 cmp r_OFF, r_TMP
145 bl bpf_error
146 nop
147 .globl bpf_jit_load_word_negative_offset
148bpf_jit_load_word_negative_offset:
149 bpf_negative_common(4)
150 andcc r_TMP, 3, %g0
151 bne load_word_unaligned
152 nop
153 retl
154 ld [r_TMP], r_A
155
156bpf_slow_path_half_neg:
157 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
158 cmp r_OFF, r_TMP
159 bl bpf_error
160 nop
161 .globl bpf_jit_load_half_negative_offset
162bpf_jit_load_half_negative_offset:
163 bpf_negative_common(2)
164 andcc r_TMP, 1, %g0
165 bne load_half_unaligned
166 nop
167 retl
168 lduh [r_TMP], r_A
169
170bpf_slow_path_byte_neg:
171 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
172 cmp r_OFF, r_TMP
173 bl bpf_error
174 nop
175 .globl bpf_jit_load_byte_negative_offset
176bpf_jit_load_byte_negative_offset:
177 bpf_negative_common(1)
178 retl
179 ldub [r_TMP], r_A
180
181bpf_slow_path_byte_msh_neg:
182 sethi %hi(SKF_MAX_NEG_OFF), r_TMP
183 cmp r_OFF, r_TMP
184 bl bpf_error
185 nop
186 .globl bpf_jit_load_byte_msh_negative_offset
187bpf_jit_load_byte_msh_negative_offset:
188 bpf_negative_common(1)
189 ldub [r_TMP], r_OFF
190 and r_OFF, 0xf, r_OFF
191 retl
192 sll r_OFF, 2, r_X
193
194bpf_error:
195 /* Make the JIT program return zero. The JIT epilogue
196 * stores away the original %o7 into r_saved_O7. The
197 * normal leaf function return is to use "retl" which
198 * would evalute to "jmpl %o7 + 8, %g0" but we want to
199 * use the saved value thus the sequence you see here.
200 */
201 jmpl r_saved_O7 + 8, %g0
202 clr %o0
203

source code of linux/arch/sparc/net/bpf_jit_asm_32.S