1//===-- Unittests for op_ files -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "memory_check_utils.h"
10#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT64
11#include "src/string/memory_utils/op_aarch64.h"
12#include "src/string/memory_utils/op_builtin.h"
13#include "src/string/memory_utils/op_generic.h"
14#include "src/string/memory_utils/op_riscv.h"
15#include "src/string/memory_utils/op_x86.h"
16#include "test/UnitTest/Test.h"
17
18namespace LIBC_NAMESPACE {
19
20template <typename T> struct has_head_tail {
21 template <typename C> static char sfinae(decltype(&C::head_tail));
22 template <typename C> static uint16_t sfinae(...);
23 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
24};
25
26template <typename T> struct has_loop_and_tail {
27 template <typename C> static char sfinae(decltype(&C::loop_and_tail));
28 template <typename C> static uint16_t sfinae(...);
29 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
30};
31
32// Allocates two Buffer and extracts two spans out of them, one
33// aligned and one misaligned. Tests are run on both spans.
34struct Buffers {
35 Buffers(size_t size)
36 : aligned_buffer(size, Aligned::YES),
37 misaligned_buffer(size, Aligned::NO) {}
38
39 // Returns two spans of 'size' bytes. The first is aligned on
40 // Buffer::kAlign and the second one is unaligned.
41 cpp::array<cpp::span<char>, 2> spans() {
42 return {aligned_buffer.span(), misaligned_buffer.span()};
43 }
44
45 Buffer aligned_buffer;
46 Buffer misaligned_buffer;
47};
48
49using MemcpyImplementations = testing::TypeList<
50#ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
51 builtin::Memcpy<1>, //
52 builtin::Memcpy<2>, //
53 builtin::Memcpy<3>, //
54 builtin::Memcpy<4>, //
55 builtin::Memcpy<8>, //
56 builtin::Memcpy<16>, //
57 builtin::Memcpy<32>, //
58 builtin::Memcpy<64>
59#endif // LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
60 >;
61
62// Convenient helper to turn a span into cpp::byte *.
63static inline cpp::byte *as_byte(cpp::span<char> span) {
64 return reinterpret_cast<cpp::byte *>(span.data());
65}
66
67// Adapt CheckMemcpy signature to op implementation signatures.
68template <auto FnImpl>
69void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
70 FnImpl(as_byte(span: dst), as_byte(span: src), size);
71}
72template <size_t Size, auto FnImpl>
73void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
74 FnImpl(as_byte(span: dst), as_byte(span: src));
75}
76
77TYPED_TEST(LlvmLibcOpTest, Memcpy, MemcpyImplementations) {
78 using Impl = ParamType;
79 constexpr size_t kSize = Impl::SIZE;
80 { // Test block operation
81 static constexpr auto BlockImpl = CopyBlockAdaptor<kSize, Impl::block>;
82 Buffers SrcBuffer(kSize);
83 Buffers DstBuffer(kSize);
84 for (auto src : SrcBuffer.spans()) {
85 Randomize(buffer: src);
86 for (auto dst : DstBuffer.spans()) {
87 ASSERT_TRUE(CheckMemcpy<BlockImpl>(dst, src, kSize));
88 }
89 }
90 }
91 { // Test head tail operations from kSize to 2 * kSize.
92 static constexpr auto HeadTailImpl = CopyAdaptor<Impl::head_tail>;
93 Buffer SrcBuffer(2 * kSize);
94 Buffer DstBuffer(2 * kSize);
95 Randomize(buffer: SrcBuffer.span());
96 for (size_t size = kSize; size < 2 * kSize; ++size) {
97 auto src = SrcBuffer.span().subspan(offset: 0, count: size);
98 auto dst = DstBuffer.span().subspan(offset: 0, count: size);
99 ASSERT_TRUE(CheckMemcpy<HeadTailImpl>(dst, src, size));
100 }
101 }
102 { // Test loop operations from kSize to 3 * kSize.
103 if constexpr (kSize > 1) {
104 static constexpr auto LoopImpl = CopyAdaptor<Impl::loop_and_tail>;
105 Buffer SrcBuffer(3 * kSize);
106 Buffer DstBuffer(3 * kSize);
107 Randomize(buffer: SrcBuffer.span());
108 for (size_t size = kSize; size < 3 * kSize; ++size) {
109 auto src = SrcBuffer.span().subspan(offset: 0, count: size);
110 auto dst = DstBuffer.span().subspan(offset: 0, count: size);
111 ASSERT_TRUE(CheckMemcpy<LoopImpl>(dst, src, size));
112 }
113 }
114 }
115}
116
117using MemsetImplementations = testing::TypeList<
118#ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE
119 builtin::Memset<1>, //
120 builtin::Memset<2>, //
121 builtin::Memset<3>, //
122 builtin::Memset<4>, //
123 builtin::Memset<8>, //
124 builtin::Memset<16>, //
125 builtin::Memset<32>, //
126 builtin::Memset<64>,
127#endif
128#ifdef LIBC_TYPES_HAS_INT64
129 generic::Memset<uint64_t>, generic::Memset<cpp::array<uint64_t, 2>>,
130#endif // LIBC_TYPES_HAS_INT64
131#ifdef __AVX512F__
132 generic::Memset<generic_v512>, generic::Memset<cpp::array<generic_v512, 2>>,
133#endif
134#ifdef __AVX__
135 generic::Memset<generic_v256>, generic::Memset<cpp::array<generic_v256, 2>>,
136#endif
137#ifdef __SSE2__
138 generic::Memset<generic_v128>, generic::Memset<cpp::array<generic_v128, 2>>,
139#endif
140 generic::Memset<uint32_t>, generic::Memset<cpp::array<uint32_t, 2>>, //
141 generic::Memset<uint16_t>, generic::Memset<cpp::array<uint16_t, 2>>, //
142 generic::Memset<uint8_t>, generic::Memset<cpp::array<uint8_t, 2>>, //
143 generic::MemsetSequence<uint8_t, uint8_t>, //
144 generic::MemsetSequence<uint16_t, uint8_t>, //
145 generic::MemsetSequence<uint32_t, uint16_t, uint8_t> //
146 >;
147
148// Adapt CheckMemset signature to op implementation signatures.
149template <auto FnImpl>
150void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
151 FnImpl(as_byte(span: dst), value, size);
152}
153template <size_t Size, auto FnImpl>
154void SetBlockAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
155 FnImpl(as_byte(span: dst), value);
156}
157
158TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {
159 using Impl = ParamType;
160 constexpr size_t kSize = Impl::SIZE;
161 { // Test block operation
162 static constexpr auto BlockImpl = SetBlockAdaptor<kSize, Impl::block>;
163 Buffers DstBuffer(kSize);
164 for (uint8_t value : cpp::array<uint8_t, 3>{0, 1, 255}) {
165 for (auto dst : DstBuffer.spans()) {
166 ASSERT_TRUE(CheckMemset<BlockImpl>(dst, value, kSize));
167 }
168 }
169 }
170 if constexpr (has_head_tail<Impl>::value) {
171 // Test head tail operations from kSize to 2 * kSize.
172 static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>;
173 Buffer DstBuffer(2 * kSize);
174 for (size_t size = kSize; size < 2 * kSize; ++size) {
175 const char value = size % 10;
176 auto dst = DstBuffer.span().subspan(offset: 0, count: size);
177 ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size));
178 }
179 }
180 if constexpr (has_loop_and_tail<Impl>::value) {
181 // Test loop operations from kSize to 3 * kSize.
182 if constexpr (kSize > 1) {
183 static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>;
184 Buffer DstBuffer(3 * kSize);
185 for (size_t size = kSize; size < 3 * kSize; ++size) {
186 const char value = size % 10;
187 auto dst = DstBuffer.span().subspan(offset: 0, count: size);
188 ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size)));
189 }
190 }
191 }
192}
193
194using BcmpImplementations = testing::TypeList<
195#ifdef LIBC_TARGET_ARCH_IS_X86_64
196#ifdef __SSE4_1__
197 generic::Bcmp<__m128i>,
198#endif // __SSE4_1__
199#ifdef __AVX2__
200 generic::Bcmp<__m256i>,
201#endif // __AVX2__
202#ifdef __AVX512BW__
203 generic::Bcmp<__m512i>,
204#endif // __AVX512BW__
205
206#endif // LIBC_TARGET_ARCH_IS_X86_64
207#ifdef LIBC_TARGET_ARCH_IS_AARCH64
208 aarch64::Bcmp<16>, //
209 aarch64::Bcmp<32>,
210#endif
211#ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
212 generic::Bcmp<uint16_t>,
213 generic::Bcmp<uint32_t>, //
214#ifdef LIBC_TYPES_HAS_INT64
215 generic::Bcmp<uint64_t>,
216#endif // LIBC_TYPES_HAS_INT64
217 generic::BcmpSequence<uint16_t, uint8_t>,
218 generic::BcmpSequence<uint32_t, uint8_t>, //
219 generic::BcmpSequence<uint32_t, uint16_t>, //
220 generic::BcmpSequence<uint32_t, uint16_t, uint8_t>,
221#endif // LIBC_TARGET_ARCH_IS_ARM
222 generic::BcmpSequence<uint8_t, uint8_t>,
223 generic::BcmpSequence<uint8_t, uint8_t, uint8_t>, //
224 generic::Bcmp<uint8_t>>;
225
226// Adapt CheckBcmp signature to op implementation signatures.
227template <auto FnImpl>
228int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
229 return (int)FnImpl(as_byte(span: p1), as_byte(span: p2), size);
230}
231template <size_t Size, auto FnImpl>
232int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
233 return (int)FnImpl(as_byte(span: p1), as_byte(span: p2));
234}
235
236TYPED_TEST(LlvmLibcOpTest, Bcmp, BcmpImplementations) {
237 using Impl = ParamType;
238 constexpr size_t kSize = Impl::SIZE;
239 { // Test block operation
240 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
241 Buffers Buffer1(kSize);
242 Buffers Buffer2(kSize);
243 for (auto span1 : Buffer1.spans()) {
244 Randomize(buffer: span1);
245 for (auto span2 : Buffer2.spans())
246 ASSERT_TRUE((CheckBcmp<BlockImpl>(span1, span2, kSize)));
247 }
248 }
249 if constexpr (has_head_tail<Impl>::value) {
250 // Test head tail operations from kSize to 2 * kSize.
251 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
252 Buffer Buffer1(2 * kSize);
253 Buffer Buffer2(2 * kSize);
254 Randomize(buffer: Buffer1.span());
255 for (size_t size = kSize; size < 2 * kSize; ++size) {
256 auto span1 = Buffer1.span().subspan(offset: 0, count: size);
257 auto span2 = Buffer2.span().subspan(offset: 0, count: size);
258 ASSERT_TRUE((CheckBcmp<HeadTailImpl>(span1, span2, size)));
259 }
260 }
261 if constexpr (has_loop_and_tail<Impl>::value) {
262 // Test loop operations from kSize to 3 * kSize.
263 if constexpr (kSize > 1) {
264 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
265 Buffer Buffer1(3 * kSize);
266 Buffer Buffer2(3 * kSize);
267 Randomize(buffer: Buffer1.span());
268 for (size_t size = kSize; size < 3 * kSize; ++size) {
269 auto span1 = Buffer1.span().subspan(offset: 0, count: size);
270 auto span2 = Buffer2.span().subspan(offset: 0, count: size);
271 ASSERT_TRUE((CheckBcmp<LoopImpl>(span1, span2, size)));
272 }
273 }
274 }
275}
276
277using MemcmpImplementations = testing::TypeList<
278#ifdef LIBC_TARGET_ARCH_IS_X86_64
279#ifdef __SSE2__
280 generic::Memcmp<__m128i>, //
281#endif
282#ifdef __AVX2__
283 generic::Memcmp<__m256i>, //
284#endif
285#ifdef __AVX512BW__
286 generic::Memcmp<__m512i>, //
287#endif
288#endif // LIBC_TARGET_ARCH_IS_X86_64
289#ifdef LIBC_TARGET_ARCH_IS_AARCH64
290 generic::Memcmp<uint8x16_t>, //
291 generic::Memcmp<uint8x16x2_t>,
292#endif
293#ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
294 generic::Memcmp<uint16_t>,
295 generic::Memcmp<uint32_t>, //
296#ifdef LIBC_TYPES_HAS_INT64
297 generic::Memcmp<uint64_t>,
298#endif // LIBC_TYPES_HAS_INT64
299 generic::MemcmpSequence<uint16_t, uint8_t>,
300 generic::MemcmpSequence<uint32_t, uint16_t, uint8_t>, //
301#endif // LIBC_TARGET_ARCH_IS_ARM
302 generic::MemcmpSequence<uint8_t, uint8_t>,
303 generic::MemcmpSequence<uint8_t, uint8_t, uint8_t>,
304 generic::Memcmp<uint8_t>>;
305
306TYPED_TEST(LlvmLibcOpTest, Memcmp, MemcmpImplementations) {
307 using Impl = ParamType;
308 constexpr size_t kSize = Impl::SIZE;
309 { // Test block operation
310 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
311 Buffers Buffer1(kSize);
312 Buffers Buffer2(kSize);
313 for (auto span1 : Buffer1.spans()) {
314 Randomize(buffer: span1);
315 for (auto span2 : Buffer2.spans())
316 ASSERT_TRUE((CheckMemcmp<BlockImpl>(span1, span2, kSize)));
317 }
318 }
319 if constexpr (has_head_tail<Impl>::value) {
320 // Test head tail operations from kSize to 2 * kSize.
321 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
322 Buffer Buffer1(2 * kSize);
323 Buffer Buffer2(2 * kSize);
324 Randomize(buffer: Buffer1.span());
325 for (size_t size = kSize; size < 2 * kSize; ++size) {
326 auto span1 = Buffer1.span().subspan(offset: 0, count: size);
327 auto span2 = Buffer2.span().subspan(offset: 0, count: size);
328 ASSERT_TRUE((CheckMemcmp<HeadTailImpl>(span1, span2, size)));
329 }
330 }
331 if constexpr (has_loop_and_tail<Impl>::value) {
332 // Test loop operations from kSize to 3 * kSize.
333 if constexpr (kSize > 1) {
334 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
335 Buffer Buffer1(3 * kSize);
336 Buffer Buffer2(3 * kSize);
337 Randomize(buffer: Buffer1.span());
338 for (size_t size = kSize; size < 3 * kSize; ++size) {
339 auto span1 = Buffer1.span().subspan(offset: 0, count: size);
340 auto span2 = Buffer2.span().subspan(offset: 0, count: size);
341 ASSERT_TRUE((CheckMemcmp<LoopImpl>(span1, span2, size)));
342 }
343 }
344 }
345}
346
347} // namespace LIBC_NAMESPACE
348

source code of libc/test/src/string/memory_utils/op_tests.cpp