Warning: This file is not a C or C++ file. It does not have highlighting.

1//===-- Memset implementation for aarch64 -----------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H
9#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H
10
11#include "src/__support/macros/attributes.h" // LIBC_INLINE
12#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL
13#include "src/string/memory_utils/op_aarch64.h"
14#include "src/string/memory_utils/op_generic.h"
15#include "src/string/memory_utils/utils.h" // Ptr, CPtr
16
17#include <stddef.h> // size_t
18
19namespace LIBC_NAMESPACE_DECL {
20
21using uint128_t = generic_v128;
22using uint256_t = generic_v256;
23using uint512_t = generic_v512;
24
25[[maybe_unused]] LIBC_INLINE static void
26inline_memset_aarch64_no_fp(Ptr dst, uint8_t value, size_t count) {
27 if (count == 0)
28 return;
29 if (count <= 3) {
30 generic::Memset<uint8_t>::block(dst, value);
31 if (count > 1)
32 generic::Memset<uint16_t>::tail(dst, value, count);
33 return;
34 }
35 if (count <= 8)
36 return generic::Memset<uint32_t>::head_tail(dst, value, count);
37 if (count <= 16)
38 return generic::Memset<uint64_t>::head_tail(dst, value, count);
39 if (count <= 32)
40 return generic::Memset<uint128_t>::head_tail(dst, value, count);
41 if (count <= (32 + 64)) {
42 generic::Memset<uint256_t>::block(dst, value);
43 if (count <= 64)
44 return generic::Memset<uint256_t>::tail(dst, value, count);
45 generic::Memset<uint256_t>::block(dst + 32, value);
46 generic::Memset<uint256_t>::tail(dst, value, count);
47 return;
48 }
49
50 generic::Memset<uint128_t>::block(dst, value);
51 align_to_next_boundary<16>(dst, count);
52 return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
53}
54
55#if defined(__ARM_NEON)
56[[maybe_unused]] LIBC_INLINE static void
57inline_memset_aarch64_with_fp(Ptr dst, uint8_t value, size_t count) {
58 if (count == 0)
59 return;
60 if (count <= 3) {
61 generic::Memset<uint8_t>::block(dst, value);
62 if (count > 1)
63 generic::Memset<uint16_t>::tail(dst, value, count);
64 return;
65 }
66 if (count <= 8)
67 return generic::Memset<uint32_t>::head_tail(dst, value, count);
68 if (count <= 16)
69 return generic::Memset<uint64_t>::head_tail(dst, value, count);
70 if (count <= 32)
71 return generic::Memset<uint128_t>::head_tail(dst, value, count);
72 if (count <= (32 + 64)) {
73 generic::Memset<uint256_t>::block(dst, value);
74 if (count <= 64)
75 return generic::Memset<uint256_t>::tail(dst, value, count);
76 generic::Memset<uint256_t>::block(dst + 32, value);
77 generic::Memset<uint256_t>::tail(dst, value, count);
78 return;
79 }
80
81 if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
82 generic::Memset<uint512_t>::block(dst, 0);
83 align_to_next_boundary<64>(dst, count);
84 return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);
85 }
86
87 generic::Memset<uint128_t>::block(dst, value);
88 align_to_next_boundary<16>(dst, count);
89 return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
90}
91#endif
92
93[[gnu::flatten]] [[maybe_unused]] LIBC_INLINE static void
94inline_memset_aarch64_dispatch(Ptr dst, uint8_t value, size_t count) {
95#if defined(__ARM_NEON)
96 return inline_memset_aarch64_with_fp(dst, value, count);
97#else
98 return inline_memset_aarch64_no_fp(dst, value, count);
99#endif
100}
101
102} // namespace LIBC_NAMESPACE_DECL
103
104#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H
105

Warning: This file is not a C or C++ file. It does not have highlighting.

source code of libc/src/string/memory_utils/aarch64/inline_memset.h