1//===-- Unittests for op_ files -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "memory_check_utils.h"
10#include "src/__support/macros/config.h"
11#include "src/__support/macros/properties/os.h"
12#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT64
13#include "src/string/memory_utils/op_aarch64.h"
14#include "src/string/memory_utils/op_builtin.h"
15#include "src/string/memory_utils/op_generic.h"
16#include "src/string/memory_utils/op_riscv.h"
17#include "src/string/memory_utils/op_x86.h"
18#include "test/UnitTest/Test.h"
19
20namespace LIBC_NAMESPACE_DECL {
21
22template <typename T> struct has_head_tail {
23 template <typename C> static char sfinae(decltype(&C::head_tail));
24 template <typename C> static uint16_t sfinae(...);
25 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
26};
27
28template <typename T> struct has_loop_and_tail {
29 template <typename C> static char sfinae(decltype(&C::loop_and_tail));
30 template <typename C> static uint16_t sfinae(...);
31 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
32};
33
34// Allocates two Buffer and extracts two spans out of them, one
35// aligned and one misaligned. Tests are run on both spans.
36struct Buffers {
37 Buffers(size_t size)
38 : aligned_buffer(size, Aligned::YES),
39 misaligned_buffer(size, Aligned::NO) {}
40
41 // Returns two spans of 'size' bytes. The first is aligned on
42 // Buffer::kAlign and the second one is unaligned.
43 cpp::array<cpp::span<char>, 2> spans() {
44 return {aligned_buffer.span(), misaligned_buffer.span()};
45 }
46
47 Buffer aligned_buffer;
48 Buffer misaligned_buffer;
49};
50
51using MemcpyImplementations = testing::TypeList<
52#ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
53 builtin::Memcpy<1>, //
54 builtin::Memcpy<2>, //
55 builtin::Memcpy<3>, //
56 builtin::Memcpy<4>, //
57 builtin::Memcpy<8>, //
58 builtin::Memcpy<16>, //
59 builtin::Memcpy<32>, //
60 builtin::Memcpy<64>
61#endif // LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
62 >;
63
64// Convenient helper to turn a span into cpp::byte *.
65static inline cpp::byte *as_byte(cpp::span<char> span) {
66 return reinterpret_cast<cpp::byte *>(span.data());
67}
68
69// Adapt CheckMemcpy signature to op implementation signatures.
70template <auto FnImpl>
71void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
72 FnImpl(as_byte(dst), as_byte(src), size);
73}
74template <size_t Size, auto FnImpl>
75void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
76 FnImpl(as_byte(dst), as_byte(src));
77}
78
79TYPED_TEST(LlvmLibcOpTest, Memcpy, MemcpyImplementations) {
80 using Impl = ParamType;
81 constexpr size_t kSize = Impl::SIZE;
82 { // Test block operation
83 static constexpr auto BlockImpl = CopyBlockAdaptor<kSize, Impl::block>;
84 Buffers SrcBuffer(kSize);
85 Buffers DstBuffer(kSize);
86 for (auto src : SrcBuffer.spans()) {
87 Randomize(src);
88 for (auto dst : DstBuffer.spans()) {
89 ASSERT_TRUE(CheckMemcpy<BlockImpl>(dst, src, kSize));
90 }
91 }
92 }
93 { // Test head tail operations from kSize to 2 * kSize.
94 static constexpr auto HeadTailImpl = CopyAdaptor<Impl::head_tail>;
95 Buffer SrcBuffer(2 * kSize);
96 Buffer DstBuffer(2 * kSize);
97 Randomize(SrcBuffer.span());
98 for (size_t size = kSize; size < 2 * kSize; ++size) {
99 auto src = SrcBuffer.span().subspan(0, size);
100 auto dst = DstBuffer.span().subspan(0, size);
101 ASSERT_TRUE(CheckMemcpy<HeadTailImpl>(dst, src, size));
102 }
103 }
104 { // Test loop operations from kSize to 3 * kSize.
105 if constexpr (kSize > 1) {
106 static constexpr auto LoopImpl = CopyAdaptor<Impl::loop_and_tail>;
107 Buffer SrcBuffer(3 * kSize);
108 Buffer DstBuffer(3 * kSize);
109 Randomize(SrcBuffer.span());
110 for (size_t size = kSize; size < 3 * kSize; ++size) {
111 auto src = SrcBuffer.span().subspan(0, size);
112 auto dst = DstBuffer.span().subspan(0, size);
113 ASSERT_TRUE(CheckMemcpy<LoopImpl>(dst, src, size));
114 }
115 }
116 }
117}
118
119using MemsetImplementations = testing::TypeList<
120#ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE
121 builtin::Memset<1>, //
122 builtin::Memset<2>, //
123 builtin::Memset<3>, //
124 builtin::Memset<4>, //
125 builtin::Memset<8>, //
126 builtin::Memset<16>, //
127 builtin::Memset<32>, //
128 builtin::Memset<64>,
129#endif
130#ifdef LIBC_TYPES_HAS_INT64
131 generic::Memset<uint64_t>, generic::Memset<cpp::array<uint64_t, 2>>,
132#endif // LIBC_TYPES_HAS_INT64
133#ifdef __AVX512F__
134 generic::Memset<generic_v512>, generic::Memset<cpp::array<generic_v512, 2>>,
135#endif
136#ifdef __AVX__
137 generic::Memset<generic_v256>, generic::Memset<cpp::array<generic_v256, 2>>,
138#endif
139#ifdef __SSE2__
140 generic::Memset<generic_v128>, generic::Memset<cpp::array<generic_v128, 2>>,
141#endif
142 generic::Memset<uint32_t>, generic::Memset<cpp::array<uint32_t, 2>>, //
143 generic::Memset<uint16_t>, generic::Memset<cpp::array<uint16_t, 2>>, //
144 generic::Memset<uint8_t>, generic::Memset<cpp::array<uint8_t, 2>>, //
145 generic::MemsetSequence<uint8_t, uint8_t>, //
146 generic::MemsetSequence<uint16_t, uint8_t>, //
147 generic::MemsetSequence<uint32_t, uint16_t, uint8_t> //
148 >;
149
150// Adapt CheckMemset signature to op implementation signatures.
151template <auto FnImpl>
152void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
153 FnImpl(as_byte(dst), value, size);
154}
155template <size_t Size, auto FnImpl>
156void SetBlockAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
157 FnImpl(as_byte(dst), value);
158}
159
160TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {
161 using Impl = ParamType;
162 constexpr size_t kSize = Impl::SIZE;
163 { // Test block operation
164 static constexpr auto BlockImpl = SetBlockAdaptor<kSize, Impl::block>;
165 Buffers DstBuffer(kSize);
166 for (uint8_t value : cpp::array<uint8_t, 3>{0, 1, 255}) {
167 for (auto dst : DstBuffer.spans()) {
168 ASSERT_TRUE(CheckMemset<BlockImpl>(dst, value, kSize));
169 }
170 }
171 }
172 if constexpr (has_head_tail<Impl>::value) {
173 // Test head tail operations from kSize to 2 * kSize.
174 static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>;
175 Buffer DstBuffer(2 * kSize);
176 for (size_t size = kSize; size < 2 * kSize; ++size) {
177 const uint8_t value = size % 10;
178 auto dst = DstBuffer.span().subspan(0, size);
179 ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size));
180 }
181 }
182 if constexpr (has_loop_and_tail<Impl>::value) {
183 // Test loop operations from kSize to 3 * kSize.
184 if constexpr (kSize > 1) {
185 static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>;
186 Buffer DstBuffer(3 * kSize);
187 for (size_t size = kSize; size < 3 * kSize; ++size) {
188 const uint8_t value = size % 10;
189 auto dst = DstBuffer.span().subspan(0, size);
190 ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size)));
191 }
192 }
193 }
194}
195
196#ifdef LIBC_TARGET_ARCH_IS_X86_64
197// Prevent GCC warning due to ignored __aligned__ attributes when passing x86
198// SIMD types as template arguments.
199#pragma GCC diagnostic push
200#pragma GCC diagnostic ignored "-Wignored-attributes"
201#endif // LIBC_TARGET_ARCH_IS_X86_64
202
203using BcmpImplementations = testing::TypeList<
204#ifdef LIBC_TARGET_ARCH_IS_X86_64
205#ifdef __SSE4_1__
206 generic::Bcmp<__m128i>,
207#endif // __SSE4_1__
208#ifdef __AVX2__
209 generic::Bcmp<__m256i>,
210#endif // __AVX2__
211#ifdef __AVX512BW__
212 generic::Bcmp<__m512i>,
213#endif // __AVX512BW__
214
215#endif // LIBC_TARGET_ARCH_IS_X86_64
216#ifdef LIBC_TARGET_ARCH_IS_AARCH64
217 aarch64::Bcmp<16>, //
218 aarch64::Bcmp<32>,
219#endif
220#ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
221 generic::Bcmp<uint16_t>,
222 generic::Bcmp<uint32_t>, //
223#ifdef LIBC_TYPES_HAS_INT64
224 generic::Bcmp<uint64_t>,
225#endif // LIBC_TYPES_HAS_INT64
226 generic::BcmpSequence<uint16_t, uint8_t>,
227 generic::BcmpSequence<uint32_t, uint8_t>, //
228 generic::BcmpSequence<uint32_t, uint16_t>, //
229 generic::BcmpSequence<uint32_t, uint16_t, uint8_t>,
230#endif // LIBC_TARGET_ARCH_IS_ARM
231 generic::BcmpSequence<uint8_t, uint8_t>,
232 generic::BcmpSequence<uint8_t, uint8_t, uint8_t>, //
233 generic::Bcmp<uint8_t>>;
234
235#ifdef LIBC_TARGET_ARCH_IS_X86_64
236#pragma GCC diagnostic pop
237#endif // LIBC_TARGET_ARCH_IS_X86_64
238
239// Adapt CheckBcmp signature to op implementation signatures.
240template <auto FnImpl>
241int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
242 return (int)FnImpl(as_byte(p1), as_byte(p2), size);
243}
244template <size_t Size, auto FnImpl>
245int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
246 return (int)FnImpl(as_byte(p1), as_byte(p2));
247}
248
249TYPED_TEST(LlvmLibcOpTest, Bcmp, BcmpImplementations) {
250 using Impl = ParamType;
251 constexpr size_t kSize = Impl::SIZE;
252 { // Test block operation
253 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
254 Buffers Buffer1(kSize);
255 Buffers Buffer2(kSize);
256 for (auto span1 : Buffer1.spans()) {
257 Randomize(span1);
258 for (auto span2 : Buffer2.spans())
259 ASSERT_TRUE((CheckBcmp<BlockImpl>(span1, span2, kSize)));
260 }
261 }
262 if constexpr (has_head_tail<Impl>::value) {
263 // Test head tail operations from kSize to 2 * kSize.
264 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
265 Buffer Buffer1(2 * kSize);
266 Buffer Buffer2(2 * kSize);
267 Randomize(Buffer1.span());
268 for (size_t size = kSize; size < 2 * kSize; ++size) {
269 auto span1 = Buffer1.span().subspan(0, size);
270 auto span2 = Buffer2.span().subspan(0, size);
271 ASSERT_TRUE((CheckBcmp<HeadTailImpl>(span1, span2, size)));
272 }
273 }
274 if constexpr (has_loop_and_tail<Impl>::value) {
275 // Test loop operations from kSize to 3 * kSize.
276 if constexpr (kSize > 1) {
277 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
278 Buffer Buffer1(3 * kSize);
279 Buffer Buffer2(3 * kSize);
280 Randomize(Buffer1.span());
281 for (size_t size = kSize; size < 3 * kSize; ++size) {
282 auto span1 = Buffer1.span().subspan(0, size);
283 auto span2 = Buffer2.span().subspan(0, size);
284 ASSERT_TRUE((CheckBcmp<LoopImpl>(span1, span2, size)));
285 }
286 }
287 }
288}
289
290#ifdef LIBC_TARGET_ARCH_IS_X86_64
291// Prevent GCC warning due to ignored __aligned__ attributes when passing x86
292// SIMD types as template arguments.
293#pragma GCC diagnostic push
294#pragma GCC diagnostic ignored "-Wignored-attributes"
295#endif // LIBC_TARGET_ARCH_IS_X86_64
296
297using MemcmpImplementations = testing::TypeList<
298#if defined(LIBC_TARGET_ARCH_IS_X86_64) && !defined(LIBC_TARGET_OS_IS_WINDOWS)
299#ifdef __SSE2__
300 generic::Memcmp<__m128i>, //
301#endif
302#ifdef __AVX2__
303 generic::Memcmp<__m256i>, //
304#endif
305#ifdef __AVX512BW__
306 generic::Memcmp<__m512i>, //
307#endif
308#endif // LIBC_TARGET_ARCH_IS_X86_64
309#ifdef LIBC_TARGET_ARCH_IS_AARCH64
310 generic::Memcmp<uint8x16_t>, //
311 generic::Memcmp<uint8x16x2_t>,
312#endif
313#ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
314 generic::Memcmp<uint16_t>,
315 generic::Memcmp<uint32_t>, //
316#ifdef LIBC_TYPES_HAS_INT64
317 generic::Memcmp<uint64_t>,
318#endif // LIBC_TYPES_HAS_INT64
319 generic::MemcmpSequence<uint16_t, uint8_t>,
320 generic::MemcmpSequence<uint32_t, uint16_t, uint8_t>, //
321#endif // LIBC_TARGET_ARCH_IS_ARM
322 generic::MemcmpSequence<uint8_t, uint8_t>,
323 generic::MemcmpSequence<uint8_t, uint8_t, uint8_t>,
324 generic::Memcmp<uint8_t>>;
325
326#ifdef LIBC_TARGET_ARCH_IS_X86_64
327#pragma GCC diagnostic pop
328#endif // LIBC_TARGET_ARCH_IS_X86_64
329
330TYPED_TEST(LlvmLibcOpTest, Memcmp, MemcmpImplementations) {
331 using Impl = ParamType;
332 constexpr size_t kSize = Impl::SIZE;
333 { // Test block operation
334 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
335 Buffers Buffer1(kSize);
336 Buffers Buffer2(kSize);
337 for (auto span1 : Buffer1.spans()) {
338 Randomize(span1);
339 for (auto span2 : Buffer2.spans())
340 ASSERT_TRUE((CheckMemcmp<BlockImpl>(span1, span2, kSize)));
341 }
342 }
343 if constexpr (has_head_tail<Impl>::value) {
344 // Test head tail operations from kSize to 2 * kSize.
345 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
346 Buffer Buffer1(2 * kSize);
347 Buffer Buffer2(2 * kSize);
348 Randomize(Buffer1.span());
349 for (size_t size = kSize; size < 2 * kSize; ++size) {
350 auto span1 = Buffer1.span().subspan(0, size);
351 auto span2 = Buffer2.span().subspan(0, size);
352 ASSERT_TRUE((CheckMemcmp<HeadTailImpl>(span1, span2, size)));
353 }
354 }
355 if constexpr (has_loop_and_tail<Impl>::value) {
356 // Test loop operations from kSize to 3 * kSize.
357 if constexpr (kSize > 1) {
358 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
359 Buffer Buffer1(3 * kSize);
360 Buffer Buffer2(3 * kSize);
361 Randomize(Buffer1.span());
362 for (size_t size = kSize; size < 3 * kSize; ++size) {
363 auto span1 = Buffer1.span().subspan(0, size);
364 auto span2 = Buffer2.span().subspan(0, size);
365 ASSERT_TRUE((CheckMemcmp<LoopImpl>(span1, span2, size)));
366 }
367 }
368 }
369}
370
371} // namespace LIBC_NAMESPACE_DECL
372

source code of libc/test/src/string/memory_utils/op_tests.cpp