1/* memset with unaligned store and rep stosb
2 Copyright (C) 2016-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19/* memset is implemented as:
20 1. Use overlapping store to avoid branch.
21 2. If size is less than VEC, use integer register stores.
22 3. If size is from VEC_SIZE to 2 * VEC_SIZE, use 2 VEC stores.
23 4. If size is from 2 * VEC_SIZE to 4 * VEC_SIZE, use 4 VEC stores.
24 5. If size is more to 4 * VEC_SIZE, align to 1 * VEC_SIZE with
25 4 VEC stores and store 4 * VEC at a time until done.
26 6. On machines ERMS feature, if size is range
27 [__x86_rep_stosb_threshold, __x86_memset_non_temporal_threshold)
28 then REP STOSB will be used.
29 7. If size >= __x86_memset_non_temporal_threshold, use a
30 non-temporal stores. */
31
32#include <sysdep.h>
33
34#ifndef MEMSET_CHK_SYMBOL
35# define MEMSET_CHK_SYMBOL(p,s) MEMSET_SYMBOL(p, s)
36#endif
37
38#ifndef WMEMSET_CHK_SYMBOL
39# define WMEMSET_CHK_SYMBOL(p,s) WMEMSET_SYMBOL(p, s)
40#endif
41
42#ifndef VZEROUPPER
43# if VEC_SIZE > 16
44# define VZEROUPPER vzeroupper
45# define VZEROUPPER_SHORT_RETURN vzeroupper; ret
46# else
47# define VZEROUPPER
48# endif
49#endif
50
51#ifndef VZEROUPPER_SHORT_RETURN
52# define VZEROUPPER_SHORT_RETURN rep; ret
53#endif
54
55#ifndef MOVQ
56# if VEC_SIZE > 16
57# define MOVQ vmovq
58# define MOVD vmovd
59# else
60# define MOVQ movq
61# define MOVD movd
62# endif
63#endif
64
65#if VEC_SIZE == 64
66# define LOOP_4X_OFFSET (VEC_SIZE * 4)
67#else
68# define LOOP_4X_OFFSET (0)
69#endif
70
71#if defined USE_WITH_EVEX || defined USE_WITH_AVX512
72# define END_REG rcx
73# define LOOP_REG rdi
74# define LESS_VEC_REG rax
75#else
76# define END_REG rdi
77# define LOOP_REG rdx
78# define LESS_VEC_REG rdi
79#endif
80
81#ifdef USE_XMM_LESS_VEC
82# define XMM_SMALL 1
83#else
84# define XMM_SMALL 0
85#endif
86
87#ifdef USE_LESS_VEC_MASK_STORE
88# define SET_REG64 rcx
89# define SET_REG32 ecx
90# define SET_REG16 cx
91# define SET_REG8 cl
92#else
93# define SET_REG64 rsi
94# define SET_REG32 esi
95# define SET_REG16 si
96# define SET_REG8 sil
97#endif
98
99#define PAGE_SIZE 4096
100
101/* Macro to calculate size of small memset block for aligning
102 purposes. */
103#define SMALL_MEMSET_ALIGN(mov_sz, ret_sz) (2 * (mov_sz) + (ret_sz) + 1)
104
105
106#ifndef SECTION
107# error SECTION is not defined!
108#endif
109
110 .section SECTION(.text), "ax", @progbits
111#if IS_IN (libc)
112# if defined SHARED
113ENTRY_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned))
114 cmp %RDX_LP, %RCX_LP
115 jb HIDDEN_JUMPTARGET (__chk_fail)
116END_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned))
117# endif
118
119ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned))
120 shl $2, %RDX_LP
121 WMEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi)
122 WMEMSET_VDUP_TO_VEC0_LOW()
123 cmpq $VEC_SIZE, %rdx
124 jb L(less_vec_from_wmemset)
125 WMEMSET_VDUP_TO_VEC0_HIGH()
126 jmp L(entry_from_wmemset)
127END (WMEMSET_SYMBOL (__wmemset, unaligned))
128#endif
129
130#if defined SHARED && IS_IN (libc)
131ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned))
132 cmp %RDX_LP, %RCX_LP
133 jb HIDDEN_JUMPTARGET (__chk_fail)
134END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned))
135#endif
136
137ENTRY (MEMSET_SYMBOL (__memset, unaligned))
138 MEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi)
139# ifdef __ILP32__
140 /* Clear the upper 32 bits. */
141 mov %edx, %edx
142# endif
143 cmpq $VEC_SIZE, %rdx
144 jb L(less_vec)
145 MEMSET_VDUP_TO_VEC0_HIGH()
146L(entry_from_wmemset):
147 cmpq $(VEC_SIZE * 2), %rdx
148 ja L(more_2x_vec)
149 /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */
150 VMOVU %VMM(0), -VEC_SIZE(%rdi,%rdx)
151 VMOVU %VMM(0), (%rdi)
152 VZEROUPPER_RETURN
153
154 /* If have AVX512 mask instructions put L(less_vec) close to
155 entry as it doesn't take much space and is likely a hot target. */
156#ifdef USE_LESS_VEC_MASK_STORE
157 /* Align to ensure the L(less_vec) logic all fits in 1x cache lines. */
158 .p2align 6,, 47
159 .p2align 4
160L(less_vec):
161L(less_vec_from_wmemset):
162 /* Less than 1 VEC. */
163# if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64
164# error Unsupported VEC_SIZE!
165# endif
166 /* Clear high bits from edi. Only keeping bits relevant to page
167 cross check. Note that we are using rax which is set in
168 MEMSET_VDUP_TO_VEC0_AND_SET_RETURN as ptr from here on out. */
169 andl $(PAGE_SIZE - 1), %edi
170 /* Check if VEC_SIZE store cross page. Mask stores suffer
171 serious performance degradation when it has to fault suppress. */
172 cmpl $(PAGE_SIZE - VEC_SIZE), %edi
173 /* This is generally considered a cold target. */
174 ja L(cross_page)
175# if VEC_SIZE > 32
176 movq $-1, %rcx
177 bzhiq %rdx, %rcx, %rcx
178 kmovq %rcx, %k1
179# else
180 movl $-1, %ecx
181 bzhil %edx, %ecx, %ecx
182 kmovd %ecx, %k1
183# endif
184 vmovdqu8 %VMM(0), (%rax){%k1}
185 VZEROUPPER_RETURN
186#endif
187
188#if defined USE_MULTIARCH && IS_IN (libc)
189END (MEMSET_SYMBOL (__memset, unaligned))
190
191# if defined SHARED && IS_IN (libc)
192ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms))
193 cmp %RDX_LP, %RCX_LP
194 jb HIDDEN_JUMPTARGET (__chk_fail)
195END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms))
196# endif
197
198ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6)
199 MEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi)
200# ifdef __ILP32__
201 /* Clear the upper 32 bits. */
202 mov %edx, %edx
203# endif
204 cmp $VEC_SIZE, %RDX_LP
205 jb L(less_vec)
206 MEMSET_VDUP_TO_VEC0_HIGH ()
207 cmp $(VEC_SIZE * 2), %RDX_LP
208 ja L(stosb_more_2x_vec)
209 /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */
210 VMOVU %VMM(0), (%rdi)
211 VMOVU %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
212 VZEROUPPER_RETURN
213#endif
214
215 .p2align 4,, 4
216L(last_2x_vec):
217#ifdef USE_LESS_VEC_MASK_STORE
218 VMOVU %VMM(0), (VEC_SIZE * -2)(%rdi, %rdx)
219 VMOVU %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
220#else
221 VMOVU %VMM(0), (VEC_SIZE * -2)(%rdi)
222 VMOVU %VMM(0), (VEC_SIZE * -1)(%rdi)
223#endif
224 VZEROUPPER_RETURN
225
226#if defined USE_MULTIARCH && IS_IN (libc)
227 .p2align 4
228L(stosb_more_2x_vec):
229 cmp __x86_rep_stosb_threshold(%rip), %RDX_LP
230 ja L(stosb_local)
231#endif
232 /* Fallthrough goes to L(loop_4x_vec). Tests for memset (2x, 4x]
233 and (4x, 8x] jump to target. */
234L(more_2x_vec):
235 /* Store next 2x vec regardless. */
236 VMOVU %VMM(0), (%rdi)
237 VMOVU %VMM(0), (VEC_SIZE * 1)(%rdi)
238
239
240 /* Two different methods of setting up pointers / compare. The two
241 methods are based on the fact that EVEX/AVX512 mov instructions take
242 more bytes then AVX2/SSE2 mov instructions. As well that EVEX/AVX512
243 machines also have fast LEA_BID. Both setup and END_REG to avoid complex
244 address mode. For EVEX/AVX512 this saves code size and keeps a few
245 targets in one fetch block. For AVX2/SSE2 this helps prevent AGU
246 bottlenecks. */
247#if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512)
248 /* If AVX2/SSE2 compute END_REG (rdi) with ALU. */
249 addq %rdx, %END_REG
250#endif
251
252 cmpq $(VEC_SIZE * 4), %rdx
253 jbe L(last_2x_vec)
254
255
256#if defined USE_WITH_EVEX || defined USE_WITH_AVX512
257 /* If EVEX/AVX512 compute END_REG - (VEC_SIZE * 4 + LOOP_4X_OFFSET) with
258 LEA_BID. */
259
260 /* END_REG is rcx for EVEX/AVX512. */
261 leaq -(VEC_SIZE * 4 + LOOP_4X_OFFSET)(%rdi, %rdx), %END_REG
262#endif
263
264 /* Store next 2x vec regardless. */
265 VMOVU %VMM(0), (VEC_SIZE * 2)(%rax)
266 VMOVU %VMM(0), (VEC_SIZE * 3)(%rax)
267
268
269#if defined USE_WITH_EVEX || defined USE_WITH_AVX512
270 /* If LOOP_4X_OFFSET don't readjust LOOP_REG (rdi), just add
271 extra offset to addresses in loop. Used for AVX512 to save space
272 as no way to get (VEC_SIZE * 4) in imm8. */
273# if LOOP_4X_OFFSET == 0
274 subq $-(VEC_SIZE * 4), %LOOP_REG
275# endif
276 /* Avoid imm32 compare here to save code size. */
277 cmpq %rdi, %rcx
278#else
279 addq $-(VEC_SIZE * 4), %END_REG
280 cmpq $(VEC_SIZE * 8), %rdx
281#endif
282 jbe L(last_4x_vec)
283#if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512)
284 /* Set LOOP_REG (rdx). */
285 leaq (VEC_SIZE * 4)(%rax), %LOOP_REG
286#endif
287 /* Align dst for loop. */
288 andq $(VEC_SIZE * -1), %LOOP_REG
289 .p2align 4
290L(loop):
291 VMOVA %VMM(0), LOOP_4X_OFFSET(%LOOP_REG)
292 VMOVA %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
293 VMOVA %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
294 VMOVA %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
295 subq $-(VEC_SIZE * 4), %LOOP_REG
296 cmpq %END_REG, %LOOP_REG
297 jb L(loop)
298 .p2align 4,, MOV_SIZE
299L(last_4x_vec):
300 VMOVU %VMM(0), LOOP_4X_OFFSET(%END_REG)
301 VMOVU %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
302 VMOVU %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
303 VMOVU %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
304L(return_vzeroupper):
305#if VEC_SIZE > 16
306 ZERO_UPPER_VEC_REGISTERS_RETURN
307#else
308 ret
309#endif
310
311#ifdef USE_WITH_AVX2
312 .p2align 4
313#else
314 .p2align 4,, 4
315#endif
316
317#if defined USE_MULTIARCH && IS_IN (libc)
318 /* If no USE_LESS_VEC_MASK put L(stosb_local) here. Will be in
319 range for 2-byte jump encoding. */
320L(stosb_local):
321 cmp __x86_memset_non_temporal_threshold(%rip), %RDX_LP
322 jae L(nt_memset)
323 movzbl %sil, %eax
324 mov %RDX_LP, %RCX_LP
325 mov %RDI_LP, %RDX_LP
326 rep stosb
327# if (defined USE_WITH_SSE2) || (defined USE_WITH_AVX512)
328 /* Use xchg to save 1-byte (this helps align targets below). */
329 xchg %RDX_LP, %RAX_LP
330# else
331 mov %RDX_LP, %RAX_LP
332# endif
333 VZEROUPPER_RETURN
334#endif
335#ifndef USE_LESS_VEC_MASK_STORE
336 /* Define L(less_vec) only if not otherwise defined. */
337 .p2align 4,, 12
338L(less_vec):
339 /* Broadcast esi to partial register (i.e VEC_SIZE == 32 broadcast to
340 xmm). This is only does anything for AVX2. */
341 MEMSET_VDUP_TO_VEC0_LOW ()
342L(less_vec_from_wmemset):
343#endif
344L(cross_page):
345#if VEC_SIZE > 32
346 cmpl $32, %edx
347 jge L(between_32_63)
348#endif
349#if VEC_SIZE > 16
350 cmpl $16, %edx
351 jge L(between_16_31)
352#endif
353#ifndef USE_XMM_LESS_VEC
354 MOVQ %VMM_128(0), %SET_REG64
355#endif
356 cmpl $8, %edx
357 jge L(between_8_15)
358 cmpl $4, %edx
359 jge L(between_4_7)
360 cmpl $1, %edx
361 jg L(between_2_3)
362 jl L(between_0_0)
363 movb %SET_REG8, (%LESS_VEC_REG)
364L(between_0_0):
365 ret
366
367 /* Align small targets only if not doing so would cross a fetch line.
368 */
369#if VEC_SIZE > 32
370 .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE)
371 /* From 32 to 63. No branch when size == 32. */
372L(between_32_63):
373 VMOVU %VMM_256(0), (%LESS_VEC_REG)
374 VMOVU %VMM_256(0), -32(%LESS_VEC_REG, %rdx)
375 VZEROUPPER_RETURN
376#endif
377
378#if VEC_SIZE >= 32
379 .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, 1)
380L(between_16_31):
381 /* From 16 to 31. No branch when size == 16. */
382 VMOVU %VMM_128(0), (%LESS_VEC_REG)
383 VMOVU %VMM_128(0), -16(%LESS_VEC_REG, %rdx)
384 ret
385#endif
386
387 /* Move size is 3 for SSE2, EVEX, and AVX512. Move size is 4 for AVX2.
388 */
389 .p2align 4,, SMALL_MEMSET_ALIGN(3 + XMM_SMALL, 1)
390L(between_8_15):
391 /* From 8 to 15. No branch when size == 8. */
392#ifdef USE_XMM_LESS_VEC
393 MOVQ %VMM_128(0), (%rdi)
394 MOVQ %VMM_128(0), -8(%rdi, %rdx)
395#else
396 movq %SET_REG64, (%LESS_VEC_REG)
397 movq %SET_REG64, -8(%LESS_VEC_REG, %rdx)
398#endif
399 ret
400
401 /* Move size is 2 for SSE2, EVEX, and AVX512. Move size is 4 for AVX2.
402 */
403 .p2align 4,, SMALL_MEMSET_ALIGN(2 << XMM_SMALL, 1)
404L(between_4_7):
405 /* From 4 to 7. No branch when size == 4. */
406#ifdef USE_XMM_LESS_VEC
407 MOVD %VMM_128(0), (%rdi)
408 MOVD %VMM_128(0), -4(%rdi, %rdx)
409#else
410 movl %SET_REG32, (%LESS_VEC_REG)
411 movl %SET_REG32, -4(%LESS_VEC_REG, %rdx)
412#endif
413 ret
414
415 /* 4 * XMM_SMALL for the third mov for AVX2. */
416 .p2align 4,, 4 * XMM_SMALL + SMALL_MEMSET_ALIGN(3, 1)
417L(between_2_3):
418 /* From 2 to 3. No branch when size == 2. */
419#ifdef USE_XMM_LESS_VEC
420 movb %SET_REG8, (%rdi)
421 movb %SET_REG8, 1(%rdi)
422 movb %SET_REG8, -1(%rdi, %rdx)
423#else
424 movw %SET_REG16, (%LESS_VEC_REG)
425 movb %SET_REG8, -1(%LESS_VEC_REG, %rdx)
426#endif
427 ret
428
429#if defined USE_MULTIARCH && IS_IN (libc)
430# ifdef USE_WITH_AVX512
431 /* Force align so the loop doesn't cross a cache-line. */
432 .p2align 4
433# endif
434 .p2align 4,, 7
435 /* Memset using non-temporal stores. */
436L(nt_memset):
437 VMOVU %VMM(0), (VEC_SIZE * 0)(%rdi)
438 leaq (VEC_SIZE * -4)(%rdi, %rdx), %rdx
439 /* Align DST. */
440 orq $(VEC_SIZE * 1 - 1), %rdi
441 incq %rdi
442 .p2align 4,, 7
443L(nt_loop):
444 VMOVNT %VMM(0), (VEC_SIZE * 0)(%rdi)
445 VMOVNT %VMM(0), (VEC_SIZE * 1)(%rdi)
446 VMOVNT %VMM(0), (VEC_SIZE * 2)(%rdi)
447 VMOVNT %VMM(0), (VEC_SIZE * 3)(%rdi)
448 subq $(VEC_SIZE * -4), %rdi
449 cmpq %rdx, %rdi
450 jb L(nt_loop)
451 sfence
452 VMOVU %VMM(0), (VEC_SIZE * 0)(%rdx)
453 VMOVU %VMM(0), (VEC_SIZE * 1)(%rdx)
454 VMOVU %VMM(0), (VEC_SIZE * 2)(%rdx)
455 VMOVU %VMM(0), (VEC_SIZE * 3)(%rdx)
456 VZEROUPPER_RETURN
457#endif
458
459END(MEMSET_SYMBOL(__memset, unaligned_erms))
460

Provided by KDAB

Privacy Policy
Update your C++ knowledge – Modern C++11/14/17 Training
Find out more

source code of glibc/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S