1 | /* memset with unaligned store and rep stosb |
2 | Copyright (C) 2016-2024 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <https://www.gnu.org/licenses/>. */ |
18 | |
19 | /* memset is implemented as: |
20 | 1. Use overlapping store to avoid branch. |
21 | 2. If size is less than VEC, use integer register stores. |
22 | 3. If size is from VEC_SIZE to 2 * VEC_SIZE, use 2 VEC stores. |
23 | 4. If size is from 2 * VEC_SIZE to 4 * VEC_SIZE, use 4 VEC stores. |
24 | 5. If size is more to 4 * VEC_SIZE, align to 1 * VEC_SIZE with |
25 | 4 VEC stores and store 4 * VEC at a time until done. |
26 | 6. On machines ERMS feature, if size is range |
27 | [__x86_rep_stosb_threshold, __x86_memset_non_temporal_threshold) |
28 | then REP STOSB will be used. |
29 | 7. If size >= __x86_memset_non_temporal_threshold, use a |
30 | non-temporal stores. */ |
31 | |
32 | #include <sysdep.h> |
33 | |
34 | #ifndef MEMSET_CHK_SYMBOL |
35 | # define MEMSET_CHK_SYMBOL(p,s) MEMSET_SYMBOL(p, s) |
36 | #endif |
37 | |
38 | #ifndef WMEMSET_CHK_SYMBOL |
39 | # define WMEMSET_CHK_SYMBOL(p,s) WMEMSET_SYMBOL(p, s) |
40 | #endif |
41 | |
42 | #ifndef VZEROUPPER |
43 | # if VEC_SIZE > 16 |
44 | # define VZEROUPPER vzeroupper |
45 | # define VZEROUPPER_SHORT_RETURN vzeroupper; ret |
46 | # else |
47 | # define VZEROUPPER |
48 | # endif |
49 | #endif |
50 | |
51 | #ifndef VZEROUPPER_SHORT_RETURN |
52 | # define VZEROUPPER_SHORT_RETURN rep; ret |
53 | #endif |
54 | |
55 | #ifndef MOVQ |
56 | # if VEC_SIZE > 16 |
57 | # define MOVQ vmovq |
58 | # define MOVD vmovd |
59 | # else |
60 | # define MOVQ movq |
61 | # define MOVD movd |
62 | # endif |
63 | #endif |
64 | |
65 | #if VEC_SIZE == 64 |
66 | # define LOOP_4X_OFFSET (VEC_SIZE * 4) |
67 | #else |
68 | # define LOOP_4X_OFFSET (0) |
69 | #endif |
70 | |
71 | #if defined USE_WITH_EVEX || defined USE_WITH_AVX512 |
72 | # define END_REG rcx |
73 | # define LOOP_REG rdi |
74 | # define LESS_VEC_REG rax |
75 | #else |
76 | # define END_REG rdi |
77 | # define LOOP_REG rdx |
78 | # define LESS_VEC_REG rdi |
79 | #endif |
80 | |
81 | #ifdef USE_XMM_LESS_VEC |
82 | # define XMM_SMALL 1 |
83 | #else |
84 | # define XMM_SMALL 0 |
85 | #endif |
86 | |
87 | #ifdef USE_LESS_VEC_MASK_STORE |
88 | # define SET_REG64 rcx |
89 | # define SET_REG32 ecx |
90 | # define SET_REG16 cx |
91 | # define SET_REG8 cl |
92 | #else |
93 | # define SET_REG64 rsi |
94 | # define SET_REG32 esi |
95 | # define SET_REG16 si |
96 | # define SET_REG8 sil |
97 | #endif |
98 | |
99 | #define PAGE_SIZE 4096 |
100 | |
101 | /* Macro to calculate size of small memset block for aligning |
102 | purposes. */ |
103 | #define SMALL_MEMSET_ALIGN(mov_sz, ret_sz) (2 * (mov_sz) + (ret_sz) + 1) |
104 | |
105 | |
106 | #ifndef SECTION |
107 | # error SECTION is not defined! |
108 | #endif |
109 | |
110 | .section SECTION(.text), "ax" , @progbits |
111 | #if IS_IN (libc) |
112 | # if defined SHARED |
113 | ENTRY_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned)) |
114 | cmp %RDX_LP, %RCX_LP |
115 | jb HIDDEN_JUMPTARGET (__chk_fail) |
116 | END_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned)) |
117 | # endif |
118 | |
119 | ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned)) |
120 | shl $2, %RDX_LP |
121 | WMEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi) |
122 | WMEMSET_VDUP_TO_VEC0_LOW() |
123 | cmpq $VEC_SIZE, %rdx |
124 | jb L(less_vec_from_wmemset) |
125 | WMEMSET_VDUP_TO_VEC0_HIGH() |
126 | jmp L(entry_from_wmemset) |
127 | END (WMEMSET_SYMBOL (__wmemset, unaligned)) |
128 | #endif |
129 | |
130 | #if defined SHARED && IS_IN (libc) |
131 | ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) |
132 | cmp %RDX_LP, %RCX_LP |
133 | jb HIDDEN_JUMPTARGET (__chk_fail) |
134 | END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) |
135 | #endif |
136 | |
137 | ENTRY (MEMSET_SYMBOL (__memset, unaligned)) |
138 | MEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi) |
139 | # ifdef __ILP32__ |
140 | /* Clear the upper 32 bits. */ |
141 | mov %edx, %edx |
142 | # endif |
143 | cmpq $VEC_SIZE, %rdx |
144 | jb L(less_vec) |
145 | MEMSET_VDUP_TO_VEC0_HIGH() |
146 | L(entry_from_wmemset): |
147 | cmpq $(VEC_SIZE * 2), %rdx |
148 | ja L(more_2x_vec) |
149 | /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ |
150 | VMOVU %VMM(0), -VEC_SIZE(%rdi,%rdx) |
151 | VMOVU %VMM(0), (%rdi) |
152 | VZEROUPPER_RETURN |
153 | |
154 | /* If have AVX512 mask instructions put L(less_vec) close to |
155 | entry as it doesn't take much space and is likely a hot target. */ |
156 | #ifdef USE_LESS_VEC_MASK_STORE |
157 | /* Align to ensure the L(less_vec) logic all fits in 1x cache lines. */ |
158 | .p2align 6,, 47 |
159 | .p2align 4 |
160 | L(less_vec): |
161 | L(less_vec_from_wmemset): |
162 | /* Less than 1 VEC. */ |
163 | # if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 |
164 | # error Unsupported VEC_SIZE! |
165 | # endif |
166 | /* Clear high bits from edi. Only keeping bits relevant to page |
167 | cross check. Note that we are using rax which is set in |
168 | MEMSET_VDUP_TO_VEC0_AND_SET_RETURN as ptr from here on out. */ |
169 | andl $(PAGE_SIZE - 1), %edi |
170 | /* Check if VEC_SIZE store cross page. Mask stores suffer |
171 | serious performance degradation when it has to fault suppress. */ |
172 | cmpl $(PAGE_SIZE - VEC_SIZE), %edi |
173 | /* This is generally considered a cold target. */ |
174 | ja L(cross_page) |
175 | # if VEC_SIZE > 32 |
176 | movq $-1, %rcx |
177 | bzhiq %rdx, %rcx, %rcx |
178 | kmovq %rcx, %k1 |
179 | # else |
180 | movl $-1, %ecx |
181 | bzhil %edx, %ecx, %ecx |
182 | kmovd %ecx, %k1 |
183 | # endif |
184 | vmovdqu8 %VMM(0), (%rax){%k1} |
185 | VZEROUPPER_RETURN |
186 | #endif |
187 | |
188 | #if defined USE_MULTIARCH && IS_IN (libc) |
189 | END (MEMSET_SYMBOL (__memset, unaligned)) |
190 | |
191 | # if defined SHARED && IS_IN (libc) |
192 | ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) |
193 | cmp %RDX_LP, %RCX_LP |
194 | jb HIDDEN_JUMPTARGET (__chk_fail) |
195 | END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) |
196 | # endif |
197 | |
198 | ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6) |
199 | MEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi) |
200 | # ifdef __ILP32__ |
201 | /* Clear the upper 32 bits. */ |
202 | mov %edx, %edx |
203 | # endif |
204 | cmp $VEC_SIZE, %RDX_LP |
205 | jb L(less_vec) |
206 | MEMSET_VDUP_TO_VEC0_HIGH () |
207 | cmp $(VEC_SIZE * 2), %RDX_LP |
208 | ja L(stosb_more_2x_vec) |
209 | /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ |
210 | VMOVU %VMM(0), (%rdi) |
211 | VMOVU %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx) |
212 | VZEROUPPER_RETURN |
213 | #endif |
214 | |
215 | .p2align 4,, 4 |
216 | L(last_2x_vec): |
217 | #ifdef USE_LESS_VEC_MASK_STORE |
218 | VMOVU %VMM(0), (VEC_SIZE * -2)(%rdi, %rdx) |
219 | VMOVU %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx) |
220 | #else |
221 | VMOVU %VMM(0), (VEC_SIZE * -2)(%rdi) |
222 | VMOVU %VMM(0), (VEC_SIZE * -1)(%rdi) |
223 | #endif |
224 | VZEROUPPER_RETURN |
225 | |
226 | #if defined USE_MULTIARCH && IS_IN (libc) |
227 | .p2align 4 |
228 | L(stosb_more_2x_vec): |
229 | cmp __x86_rep_stosb_threshold(%rip), %RDX_LP |
230 | ja L(stosb_local) |
231 | #endif |
232 | /* Fallthrough goes to L(loop_4x_vec). Tests for memset (2x, 4x] |
233 | and (4x, 8x] jump to target. */ |
234 | L(more_2x_vec): |
235 | /* Store next 2x vec regardless. */ |
236 | VMOVU %VMM(0), (%rdi) |
237 | VMOVU %VMM(0), (VEC_SIZE * 1)(%rdi) |
238 | |
239 | |
240 | /* Two different methods of setting up pointers / compare. The two |
241 | methods are based on the fact that EVEX/AVX512 mov instructions take |
242 | more bytes then AVX2/SSE2 mov instructions. As well that EVEX/AVX512 |
243 | machines also have fast LEA_BID. Both setup and END_REG to avoid complex |
244 | address mode. For EVEX/AVX512 this saves code size and keeps a few |
245 | targets in one fetch block. For AVX2/SSE2 this helps prevent AGU |
246 | bottlenecks. */ |
247 | #if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512) |
248 | /* If AVX2/SSE2 compute END_REG (rdi) with ALU. */ |
249 | addq %rdx, %END_REG |
250 | #endif |
251 | |
252 | cmpq $(VEC_SIZE * 4), %rdx |
253 | jbe L(last_2x_vec) |
254 | |
255 | |
256 | #if defined USE_WITH_EVEX || defined USE_WITH_AVX512 |
257 | /* If EVEX/AVX512 compute END_REG - (VEC_SIZE * 4 + LOOP_4X_OFFSET) with |
258 | LEA_BID. */ |
259 | |
260 | /* END_REG is rcx for EVEX/AVX512. */ |
261 | leaq -(VEC_SIZE * 4 + LOOP_4X_OFFSET)(%rdi, %rdx), %END_REG |
262 | #endif |
263 | |
264 | /* Store next 2x vec regardless. */ |
265 | VMOVU %VMM(0), (VEC_SIZE * 2)(%rax) |
266 | VMOVU %VMM(0), (VEC_SIZE * 3)(%rax) |
267 | |
268 | |
269 | #if defined USE_WITH_EVEX || defined USE_WITH_AVX512 |
270 | /* If LOOP_4X_OFFSET don't readjust LOOP_REG (rdi), just add |
271 | extra offset to addresses in loop. Used for AVX512 to save space |
272 | as no way to get (VEC_SIZE * 4) in imm8. */ |
273 | # if LOOP_4X_OFFSET == 0 |
274 | subq $-(VEC_SIZE * 4), %LOOP_REG |
275 | # endif |
276 | /* Avoid imm32 compare here to save code size. */ |
277 | cmpq %rdi, %rcx |
278 | #else |
279 | addq $-(VEC_SIZE * 4), %END_REG |
280 | cmpq $(VEC_SIZE * 8), %rdx |
281 | #endif |
282 | jbe L(last_4x_vec) |
283 | #if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512) |
284 | /* Set LOOP_REG (rdx). */ |
285 | leaq (VEC_SIZE * 4)(%rax), %LOOP_REG |
286 | #endif |
287 | /* Align dst for loop. */ |
288 | andq $(VEC_SIZE * -1), %LOOP_REG |
289 | .p2align 4 |
290 | L(loop): |
291 | VMOVA %VMM(0), LOOP_4X_OFFSET(%LOOP_REG) |
292 | VMOVA %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG) |
293 | VMOVA %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG) |
294 | VMOVA %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG) |
295 | subq $-(VEC_SIZE * 4), %LOOP_REG |
296 | cmpq %END_REG, %LOOP_REG |
297 | jb L(loop) |
298 | .p2align 4,, MOV_SIZE |
299 | L(last_4x_vec): |
300 | VMOVU %VMM(0), LOOP_4X_OFFSET(%END_REG) |
301 | VMOVU %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG) |
302 | VMOVU %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG) |
303 | VMOVU %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG) |
304 | L(return_vzeroupper): |
305 | #if VEC_SIZE > 16 |
306 | ZERO_UPPER_VEC_REGISTERS_RETURN |
307 | #else |
308 | ret |
309 | #endif |
310 | |
311 | #ifdef USE_WITH_AVX2 |
312 | .p2align 4 |
313 | #else |
314 | .p2align 4,, 4 |
315 | #endif |
316 | |
317 | #if defined USE_MULTIARCH && IS_IN (libc) |
318 | /* If no USE_LESS_VEC_MASK put L(stosb_local) here. Will be in |
319 | range for 2-byte jump encoding. */ |
320 | L(stosb_local): |
321 | cmp __x86_memset_non_temporal_threshold(%rip), %RDX_LP |
322 | jae L(nt_memset) |
323 | movzbl %sil, %eax |
324 | mov %RDX_LP, %RCX_LP |
325 | mov %RDI_LP, %RDX_LP |
326 | rep stosb |
327 | # if (defined USE_WITH_SSE2) || (defined USE_WITH_AVX512) |
328 | /* Use xchg to save 1-byte (this helps align targets below). */ |
329 | xchg %RDX_LP, %RAX_LP |
330 | # else |
331 | mov %RDX_LP, %RAX_LP |
332 | # endif |
333 | VZEROUPPER_RETURN |
334 | #endif |
335 | #ifndef USE_LESS_VEC_MASK_STORE |
336 | /* Define L(less_vec) only if not otherwise defined. */ |
337 | .p2align 4,, 12 |
338 | L(less_vec): |
339 | /* Broadcast esi to partial register (i.e VEC_SIZE == 32 broadcast to |
340 | xmm). This is only does anything for AVX2. */ |
341 | MEMSET_VDUP_TO_VEC0_LOW () |
342 | L(less_vec_from_wmemset): |
343 | #endif |
344 | L(cross_page): |
345 | #if VEC_SIZE > 32 |
346 | cmpl $32, %edx |
347 | jge L(between_32_63) |
348 | #endif |
349 | #if VEC_SIZE > 16 |
350 | cmpl $16, %edx |
351 | jge L(between_16_31) |
352 | #endif |
353 | #ifndef USE_XMM_LESS_VEC |
354 | MOVQ %VMM_128(0), %SET_REG64 |
355 | #endif |
356 | cmpl $8, %edx |
357 | jge L(between_8_15) |
358 | cmpl $4, %edx |
359 | jge L(between_4_7) |
360 | cmpl $1, %edx |
361 | jg L(between_2_3) |
362 | jl L(between_0_0) |
363 | movb %SET_REG8, (%LESS_VEC_REG) |
364 | L(between_0_0): |
365 | ret |
366 | |
367 | /* Align small targets only if not doing so would cross a fetch line. |
368 | */ |
369 | #if VEC_SIZE > 32 |
370 | .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE) |
371 | /* From 32 to 63. No branch when size == 32. */ |
372 | L(between_32_63): |
373 | VMOVU %VMM_256(0), (%LESS_VEC_REG) |
374 | VMOVU %VMM_256(0), -32(%LESS_VEC_REG, %rdx) |
375 | VZEROUPPER_RETURN |
376 | #endif |
377 | |
378 | #if VEC_SIZE >= 32 |
379 | .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, 1) |
380 | L(between_16_31): |
381 | /* From 16 to 31. No branch when size == 16. */ |
382 | VMOVU %VMM_128(0), (%LESS_VEC_REG) |
383 | VMOVU %VMM_128(0), -16(%LESS_VEC_REG, %rdx) |
384 | ret |
385 | #endif |
386 | |
387 | /* Move size is 3 for SSE2, EVEX, and AVX512. Move size is 4 for AVX2. |
388 | */ |
389 | .p2align 4,, SMALL_MEMSET_ALIGN(3 + XMM_SMALL, 1) |
390 | L(between_8_15): |
391 | /* From 8 to 15. No branch when size == 8. */ |
392 | #ifdef USE_XMM_LESS_VEC |
393 | MOVQ %VMM_128(0), (%rdi) |
394 | MOVQ %VMM_128(0), -8(%rdi, %rdx) |
395 | #else |
396 | movq %SET_REG64, (%LESS_VEC_REG) |
397 | movq %SET_REG64, -8(%LESS_VEC_REG, %rdx) |
398 | #endif |
399 | ret |
400 | |
401 | /* Move size is 2 for SSE2, EVEX, and AVX512. Move size is 4 for AVX2. |
402 | */ |
403 | .p2align 4,, SMALL_MEMSET_ALIGN(2 << XMM_SMALL, 1) |
404 | L(between_4_7): |
405 | /* From 4 to 7. No branch when size == 4. */ |
406 | #ifdef USE_XMM_LESS_VEC |
407 | MOVD %VMM_128(0), (%rdi) |
408 | MOVD %VMM_128(0), -4(%rdi, %rdx) |
409 | #else |
410 | movl %SET_REG32, (%LESS_VEC_REG) |
411 | movl %SET_REG32, -4(%LESS_VEC_REG, %rdx) |
412 | #endif |
413 | ret |
414 | |
415 | /* 4 * XMM_SMALL for the third mov for AVX2. */ |
416 | .p2align 4,, 4 * XMM_SMALL + SMALL_MEMSET_ALIGN(3, 1) |
417 | L(between_2_3): |
418 | /* From 2 to 3. No branch when size == 2. */ |
419 | #ifdef USE_XMM_LESS_VEC |
420 | movb %SET_REG8, (%rdi) |
421 | movb %SET_REG8, 1(%rdi) |
422 | movb %SET_REG8, -1(%rdi, %rdx) |
423 | #else |
424 | movw %SET_REG16, (%LESS_VEC_REG) |
425 | movb %SET_REG8, -1(%LESS_VEC_REG, %rdx) |
426 | #endif |
427 | ret |
428 | |
429 | #if defined USE_MULTIARCH && IS_IN (libc) |
430 | # ifdef USE_WITH_AVX512 |
431 | /* Force align so the loop doesn't cross a cache-line. */ |
432 | .p2align 4 |
433 | # endif |
434 | .p2align 4,, 7 |
435 | /* Memset using non-temporal stores. */ |
436 | L(nt_memset): |
437 | VMOVU %VMM(0), (VEC_SIZE * 0)(%rdi) |
438 | leaq (VEC_SIZE * -4)(%rdi, %rdx), %rdx |
439 | /* Align DST. */ |
440 | orq $(VEC_SIZE * 1 - 1), %rdi |
441 | incq %rdi |
442 | .p2align 4,, 7 |
443 | L(nt_loop): |
444 | VMOVNT %VMM(0), (VEC_SIZE * 0)(%rdi) |
445 | VMOVNT %VMM(0), (VEC_SIZE * 1)(%rdi) |
446 | VMOVNT %VMM(0), (VEC_SIZE * 2)(%rdi) |
447 | VMOVNT %VMM(0), (VEC_SIZE * 3)(%rdi) |
448 | subq $(VEC_SIZE * -4), %rdi |
449 | cmpq %rdx, %rdi |
450 | jb L(nt_loop) |
451 | sfence |
452 | VMOVU %VMM(0), (VEC_SIZE * 0)(%rdx) |
453 | VMOVU %VMM(0), (VEC_SIZE * 1)(%rdx) |
454 | VMOVU %VMM(0), (VEC_SIZE * 2)(%rdx) |
455 | VMOVU %VMM(0), (VEC_SIZE * 3)(%rdx) |
456 | VZEROUPPER_RETURN |
457 | #endif |
458 | |
459 | END(MEMSET_SYMBOL(__memset, unaligned_erms)) |
460 | |