1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited |
4 | */ |
5 | |
6 | #include <linux/export.h> |
7 | #include <asm/alternative-asm.h> |
8 | #include <asm/asm.h> |
9 | #include <asm/asmmacro.h> |
10 | #include <asm/asm-extable.h> |
11 | #include <asm/cpu.h> |
12 | #include <asm/regdef.h> |
13 | #include <asm/unwind_hints.h> |
14 | |
15 | SYM_FUNC_START(__copy_user) |
16 | /* |
17 | * Some CPUs support hardware unaligned access |
18 | */ |
19 | ALTERNATIVE "b __copy_user_generic" , \ |
20 | "b __copy_user_fast" , CPU_FEATURE_UAL |
21 | SYM_FUNC_END(__copy_user) |
22 | |
23 | EXPORT_SYMBOL(__copy_user) |
24 | |
25 | /* |
26 | * unsigned long __copy_user_generic(void *to, const void *from, size_t n) |
27 | * |
28 | * a0: to |
29 | * a1: from |
30 | * a2: n |
31 | */ |
32 | SYM_FUNC_START(__copy_user_generic) |
33 | beqz a2, 3f |
34 | |
35 | 1: ld.b t0, a1, 0 |
36 | 2: st.b t0, a0, 0 |
37 | addi.d a0, a0, 1 |
38 | addi.d a1, a1, 1 |
39 | addi.d a2, a2, -1 |
40 | bgtz a2, 1b |
41 | |
42 | 3: move a0, a2 |
43 | jr ra |
44 | |
45 | _asm_extable 1b, 3b |
46 | _asm_extable 2b, 3b |
47 | SYM_FUNC_END(__copy_user_generic) |
48 | |
49 | /* |
50 | * unsigned long __copy_user_fast(void *to, const void *from, unsigned long n) |
51 | * |
52 | * a0: to |
53 | * a1: from |
54 | * a2: n |
55 | */ |
56 | SYM_FUNC_START(__copy_user_fast) |
57 | sltui t0, a2, 9 |
58 | bnez t0, .Lsmall |
59 | |
60 | 0: ld.d t0, a1, 0 |
61 | 1: st.d t0, a0, 0 |
62 | add.d a3, a1, a2 |
63 | add.d a2, a0, a2 |
64 | |
65 | /* align up destination address */ |
66 | andi t1, a0, 7 |
67 | sub.d t0, zero, t1 |
68 | addi.d t0, t0, 8 |
69 | add.d a1, a1, t0 |
70 | add.d a0, a0, t0 |
71 | |
72 | addi.d a4, a3, -64 |
73 | bgeu a1, a4, .Llt64 |
74 | |
75 | /* copy 64 bytes at a time */ |
76 | .Lloop64: |
77 | 2: ld.d t0, a1, 0 |
78 | 3: ld.d t1, a1, 8 |
79 | 4: ld.d t2, a1, 16 |
80 | 5: ld.d t3, a1, 24 |
81 | 6: ld.d t4, a1, 32 |
82 | 7: ld.d t5, a1, 40 |
83 | 8: ld.d t6, a1, 48 |
84 | 9: ld.d t7, a1, 56 |
85 | 10: st.d t0, a0, 0 |
86 | 11: st.d t1, a0, 8 |
87 | 12: st.d t2, a0, 16 |
88 | 13: st.d t3, a0, 24 |
89 | 14: st.d t4, a0, 32 |
90 | 15: st.d t5, a0, 40 |
91 | 16: st.d t6, a0, 48 |
92 | 17: st.d t7, a0, 56 |
93 | addi.d a1, a1, 64 |
94 | addi.d a0, a0, 64 |
95 | bltu a1, a4, .Lloop64 |
96 | |
97 | /* copy the remaining bytes */ |
98 | .Llt64: |
99 | addi.d a4, a3, -32 |
100 | bgeu a1, a4, .Llt32 |
101 | 18: ld.d t0, a1, 0 |
102 | 19: ld.d t1, a1, 8 |
103 | 20: ld.d t2, a1, 16 |
104 | 21: ld.d t3, a1, 24 |
105 | 22: st.d t0, a0, 0 |
106 | 23: st.d t1, a0, 8 |
107 | 24: st.d t2, a0, 16 |
108 | 25: st.d t3, a0, 24 |
109 | addi.d a1, a1, 32 |
110 | addi.d a0, a0, 32 |
111 | |
112 | .Llt32: |
113 | addi.d a4, a3, -16 |
114 | bgeu a1, a4, .Llt16 |
115 | 26: ld.d t0, a1, 0 |
116 | 27: ld.d t1, a1, 8 |
117 | 28: st.d t0, a0, 0 |
118 | 29: st.d t1, a0, 8 |
119 | addi.d a1, a1, 16 |
120 | addi.d a0, a0, 16 |
121 | |
122 | .Llt16: |
123 | addi.d a4, a3, -8 |
124 | bgeu a1, a4, .Llt8 |
125 | 30: ld.d t0, a1, 0 |
126 | 31: st.d t0, a0, 0 |
127 | addi.d a1, a1, 8 |
128 | addi.d a0, a0, 8 |
129 | |
130 | .Llt8: |
131 | 32: ld.d t0, a3, -8 |
132 | 33: st.d t0, a2, -8 |
133 | |
134 | /* return */ |
135 | move a0, zero |
136 | jr ra |
137 | |
138 | .align 5 |
139 | .Lsmall: |
140 | pcaddi t0, 8 |
141 | slli.d a3, a2, 5 |
142 | add.d t0, t0, a3 |
143 | jr t0 |
144 | |
145 | .align 5 |
146 | move a0, zero |
147 | jr ra |
148 | |
149 | .align 5 |
150 | 34: ld.b t0, a1, 0 |
151 | 35: st.b t0, a0, 0 |
152 | move a0, zero |
153 | jr ra |
154 | |
155 | .align 5 |
156 | 36: ld.h t0, a1, 0 |
157 | 37: st.h t0, a0, 0 |
158 | move a0, zero |
159 | jr ra |
160 | |
161 | .align 5 |
162 | 38: ld.h t0, a1, 0 |
163 | 39: ld.b t1, a1, 2 |
164 | 40: st.h t0, a0, 0 |
165 | 41: st.b t1, a0, 2 |
166 | move a0, zero |
167 | jr ra |
168 | |
169 | .align 5 |
170 | 42: ld.w t0, a1, 0 |
171 | 43: st.w t0, a0, 0 |
172 | move a0, zero |
173 | jr ra |
174 | |
175 | .align 5 |
176 | 44: ld.w t0, a1, 0 |
177 | 45: ld.b t1, a1, 4 |
178 | 46: st.w t0, a0, 0 |
179 | 47: st.b t1, a0, 4 |
180 | move a0, zero |
181 | jr ra |
182 | |
183 | .align 5 |
184 | 48: ld.w t0, a1, 0 |
185 | 49: ld.h t1, a1, 4 |
186 | 50: st.w t0, a0, 0 |
187 | 51: st.h t1, a0, 4 |
188 | move a0, zero |
189 | jr ra |
190 | |
191 | .align 5 |
192 | 52: ld.w t0, a1, 0 |
193 | 53: ld.w t1, a1, 3 |
194 | 54: st.w t0, a0, 0 |
195 | 55: st.w t1, a0, 3 |
196 | move a0, zero |
197 | jr ra |
198 | |
199 | .align 5 |
200 | 56: ld.d t0, a1, 0 |
201 | 57: st.d t0, a0, 0 |
202 | move a0, zero |
203 | jr ra |
204 | |
205 | /* fixup and ex_table */ |
206 | .Llarge_fixup: |
207 | sub.d a2, a2, a0 |
208 | |
209 | .Lsmall_fixup: |
210 | 58: ld.b t0, a1, 0 |
211 | 59: st.b t0, a0, 0 |
212 | addi.d a0, a0, 1 |
213 | addi.d a1, a1, 1 |
214 | addi.d a2, a2, -1 |
215 | bgt a2, zero, 58b |
216 | |
217 | .Lexit: |
218 | move a0, a2 |
219 | jr ra |
220 | |
221 | _asm_extable 0b, .Lsmall_fixup |
222 | _asm_extable 1b, .Lsmall_fixup |
223 | _asm_extable 2b, .Llarge_fixup |
224 | _asm_extable 3b, .Llarge_fixup |
225 | _asm_extable 4b, .Llarge_fixup |
226 | _asm_extable 5b, .Llarge_fixup |
227 | _asm_extable 6b, .Llarge_fixup |
228 | _asm_extable 7b, .Llarge_fixup |
229 | _asm_extable 8b, .Llarge_fixup |
230 | _asm_extable 9b, .Llarge_fixup |
231 | _asm_extable 10b, .Llarge_fixup |
232 | _asm_extable 11b, .Llarge_fixup |
233 | _asm_extable 12b, .Llarge_fixup |
234 | _asm_extable 13b, .Llarge_fixup |
235 | _asm_extable 14b, .Llarge_fixup |
236 | _asm_extable 15b, .Llarge_fixup |
237 | _asm_extable 16b, .Llarge_fixup |
238 | _asm_extable 17b, .Llarge_fixup |
239 | _asm_extable 18b, .Llarge_fixup |
240 | _asm_extable 19b, .Llarge_fixup |
241 | _asm_extable 20b, .Llarge_fixup |
242 | _asm_extable 21b, .Llarge_fixup |
243 | _asm_extable 22b, .Llarge_fixup |
244 | _asm_extable 23b, .Llarge_fixup |
245 | _asm_extable 24b, .Llarge_fixup |
246 | _asm_extable 25b, .Llarge_fixup |
247 | _asm_extable 26b, .Llarge_fixup |
248 | _asm_extable 27b, .Llarge_fixup |
249 | _asm_extable 28b, .Llarge_fixup |
250 | _asm_extable 29b, .Llarge_fixup |
251 | _asm_extable 30b, .Llarge_fixup |
252 | _asm_extable 31b, .Llarge_fixup |
253 | _asm_extable 32b, .Llarge_fixup |
254 | _asm_extable 33b, .Llarge_fixup |
255 | _asm_extable 34b, .Lexit |
256 | _asm_extable 35b, .Lexit |
257 | _asm_extable 36b, .Lsmall_fixup |
258 | _asm_extable 37b, .Lsmall_fixup |
259 | _asm_extable 38b, .Lsmall_fixup |
260 | _asm_extable 39b, .Lsmall_fixup |
261 | _asm_extable 40b, .Lsmall_fixup |
262 | _asm_extable 41b, .Lsmall_fixup |
263 | _asm_extable 42b, .Lsmall_fixup |
264 | _asm_extable 43b, .Lsmall_fixup |
265 | _asm_extable 44b, .Lsmall_fixup |
266 | _asm_extable 45b, .Lsmall_fixup |
267 | _asm_extable 46b, .Lsmall_fixup |
268 | _asm_extable 47b, .Lsmall_fixup |
269 | _asm_extable 48b, .Lsmall_fixup |
270 | _asm_extable 49b, .Lsmall_fixup |
271 | _asm_extable 50b, .Lsmall_fixup |
272 | _asm_extable 51b, .Lsmall_fixup |
273 | _asm_extable 52b, .Lsmall_fixup |
274 | _asm_extable 53b, .Lsmall_fixup |
275 | _asm_extable 54b, .Lsmall_fixup |
276 | _asm_extable 55b, .Lsmall_fixup |
277 | _asm_extable 56b, .Lsmall_fixup |
278 | _asm_extable 57b, .Lsmall_fixup |
279 | _asm_extable 58b, .Lexit |
280 | _asm_extable 59b, .Lexit |
281 | SYM_FUNC_END(__copy_user_fast) |
282 | |
283 | STACK_FRAME_NON_STANDARD __copy_user_fast |
284 | |