1//===-- memtag.h ------------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_MEMTAG_H_
10#define SCUDO_MEMTAG_H_
11
12#include "internal_defs.h"
13
14#if SCUDO_CAN_USE_MTE
15#include <sys/auxv.h>
16#include <sys/prctl.h>
17#endif
18
19namespace scudo {
20
21#if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) || \
22 defined(SCUDO_FUZZ)
23
24// We assume that Top-Byte Ignore is enabled if the architecture supports memory
25// tagging. Not all operating systems enable TBI, so we only claim architectural
26// support for memory tagging if the operating system enables TBI.
27// HWASan uses the top byte for its own purpose and Scudo should not touch it.
28#if SCUDO_CAN_USE_MTE && !defined(SCUDO_DISABLE_TBI) && \
29 !__has_feature(hwaddress_sanitizer)
30inline constexpr bool archSupportsMemoryTagging() { return true; }
31#else
32inline constexpr bool archSupportsMemoryTagging() { return false; }
33#endif
34
35inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
36
37inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
38
39inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
40
41#else
42
43inline constexpr bool archSupportsMemoryTagging() { return false; }
44
45inline NORETURN uptr archMemoryTagGranuleSize() {
46 UNREACHABLE("memory tagging not supported");
47}
48
49inline NORETURN uptr untagPointer(uptr Ptr) {
50 (void)Ptr;
51 UNREACHABLE("memory tagging not supported");
52}
53
54inline NORETURN uint8_t extractTag(uptr Ptr) {
55 (void)Ptr;
56 UNREACHABLE("memory tagging not supported");
57}
58
59#endif
60
61#if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)
62
63#if SCUDO_CAN_USE_MTE
64
65inline bool systemSupportsMemoryTagging() {
66#ifndef HWCAP2_MTE
67#define HWCAP2_MTE (1 << 18)
68#endif
69 return getauxval(AT_HWCAP2) & HWCAP2_MTE;
70}
71
72inline bool systemDetectsMemoryTagFaultsTestOnly() {
73#ifndef PR_SET_TAGGED_ADDR_CTRL
74#define PR_SET_TAGGED_ADDR_CTRL 54
75#endif
76#ifndef PR_GET_TAGGED_ADDR_CTRL
77#define PR_GET_TAGGED_ADDR_CTRL 56
78#endif
79#ifndef PR_TAGGED_ADDR_ENABLE
80#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
81#endif
82#ifndef PR_MTE_TCF_SHIFT
83#define PR_MTE_TCF_SHIFT 1
84#endif
85#ifndef PR_MTE_TAG_SHIFT
86#define PR_MTE_TAG_SHIFT 3
87#endif
88#ifndef PR_MTE_TCF_NONE
89#define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
90#endif
91#ifndef PR_MTE_TCF_SYNC
92#define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
93#endif
94#ifndef PR_MTE_TCF_MASK
95#define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
96#endif
97 int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
98 if (res == -1)
99 return false;
100 return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
101}
102
103inline void enableSystemMemoryTaggingTestOnly() {
104 prctl(PR_SET_TAGGED_ADDR_CTRL,
105 PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
106 0, 0, 0);
107}
108
109#else // !SCUDO_CAN_USE_MTE
110
111inline bool systemSupportsMemoryTagging() { return false; }
112
113inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
114 UNREACHABLE("memory tagging not supported");
115}
116
117inline NORETURN void enableSystemMemoryTaggingTestOnly() {
118 UNREACHABLE("memory tagging not supported");
119}
120
121#endif // SCUDO_CAN_USE_MTE
122
123class ScopedDisableMemoryTagChecks {
124 uptr PrevTCO;
125
126public:
127 ScopedDisableMemoryTagChecks() {
128 __asm__ __volatile__(
129 R"(
130 .arch_extension memtag
131 mrs %0, tco
132 msr tco, #1
133 )"
134 : "=r"(PrevTCO));
135 }
136
137 ~ScopedDisableMemoryTagChecks() {
138 __asm__ __volatile__(
139 R"(
140 .arch_extension memtag
141 msr tco, %0
142 )"
143 :
144 : "r"(PrevTCO));
145 }
146};
147
148inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
149 ExcludeMask |= 1; // Always exclude Tag 0.
150 uptr TaggedPtr;
151 __asm__ __volatile__(
152 R"(
153 .arch_extension memtag
154 irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
155 )"
156 : [TaggedPtr] "=r"(TaggedPtr)
157 : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
158 return TaggedPtr;
159}
160
161inline uptr addFixedTag(uptr Ptr, uptr Tag) {
162 DCHECK_LT(Tag, 16);
163 DCHECK_EQ(untagPointer(Ptr), Ptr);
164 return Ptr | (Tag << 56);
165}
166
167inline uptr storeTags(uptr Begin, uptr End) {
168 DCHECK_EQ(0, Begin % 16);
169 uptr LineSize, Next, Tmp;
170 __asm__ __volatile__(
171 R"(
172 .arch_extension memtag
173
174 // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
175 // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
176 // indicates that the DC instructions are unavailable.
177 DCZID .req %[Tmp]
178 mrs DCZID, dczid_el0
179 tbnz DCZID, #4, 3f
180 and DCZID, DCZID, #15
181 mov %[LineSize], #4
182 lsl %[LineSize], %[LineSize], DCZID
183 .unreq DCZID
184
185 // Our main loop doesn't handle the case where we don't need to perform any
186 // DC GZVA operations. If the size of our tagged region is less than
187 // twice the cache line size, bail out to the slow path since it's not
188 // guaranteed that we'll be able to do a DC GZVA.
189 Size .req %[Tmp]
190 sub Size, %[End], %[Cur]
191 cmp Size, %[LineSize], lsl #1
192 b.lt 3f
193 .unreq Size
194
195 LineMask .req %[Tmp]
196 sub LineMask, %[LineSize], #1
197
198 // STZG until the start of the next cache line.
199 orr %[Next], %[Cur], LineMask
200 1:
201 stzg %[Cur], [%[Cur]], #16
202 cmp %[Cur], %[Next]
203 b.lt 1b
204
205 // DC GZVA cache lines until we have no more full cache lines.
206 bic %[Next], %[End], LineMask
207 .unreq LineMask
208 2:
209 dc gzva, %[Cur]
210 add %[Cur], %[Cur], %[LineSize]
211 cmp %[Cur], %[Next]
212 b.lt 2b
213
214 // STZG until the end of the tagged region. This loop is also used to handle
215 // slow path cases.
216 3:
217 cmp %[Cur], %[End]
218 b.ge 4f
219 stzg %[Cur], [%[Cur]], #16
220 b 3b
221
222 4:
223 )"
224 : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
225 [Tmp] "=&r"(Tmp)
226 : [End] "r"(End)
227 : "memory");
228 DCHECK_EQ(0, Begin % 16);
229 return Begin;
230}
231
232inline void storeTag(uptr Ptr) {
233 DCHECK_EQ(0, Ptr % 16);
234 __asm__ __volatile__(R"(
235 .arch_extension memtag
236 stg %0, [%0]
237 )"
238 :
239 : "r"(Ptr)
240 : "memory");
241}
242
243inline uptr loadTag(uptr Ptr) {
244 DCHECK_EQ(0, Ptr % 16);
245 uptr TaggedPtr = Ptr;
246 __asm__ __volatile__(
247 R"(
248 .arch_extension memtag
249 ldg %0, [%0]
250 )"
251 : "+r"(TaggedPtr)
252 :
253 : "memory");
254 return TaggedPtr;
255}
256
257#else
258
259inline NORETURN bool systemSupportsMemoryTagging() {
260 UNREACHABLE("memory tagging not supported");
261}
262
263inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
264 UNREACHABLE("memory tagging not supported");
265}
266
267inline NORETURN void enableSystemMemoryTaggingTestOnly() {
268 UNREACHABLE("memory tagging not supported");
269}
270
271struct ScopedDisableMemoryTagChecks {
272 ScopedDisableMemoryTagChecks() {}
273};
274
275inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
276 (void)Ptr;
277 (void)ExcludeMask;
278 UNREACHABLE("memory tagging not supported");
279}
280
281inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) {
282 (void)Ptr;
283 (void)Tag;
284 UNREACHABLE("memory tagging not supported");
285}
286
287inline NORETURN uptr storeTags(uptr Begin, uptr End) {
288 (void)Begin;
289 (void)End;
290 UNREACHABLE("memory tagging not supported");
291}
292
293inline NORETURN void storeTag(uptr Ptr) {
294 (void)Ptr;
295 UNREACHABLE("memory tagging not supported");
296}
297
298inline NORETURN uptr loadTag(uptr Ptr) {
299 (void)Ptr;
300 UNREACHABLE("memory tagging not supported");
301}
302
303#endif
304
305#pragma GCC diagnostic push
306#pragma GCC diagnostic ignored "-Wmissing-noreturn"
307inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
308 uptr *TaggedBegin, uptr *TaggedEnd) {
309 *TaggedBegin = selectRandomTag(Ptr: reinterpret_cast<uptr>(Ptr), ExcludeMask);
310 *TaggedEnd = storeTags(Begin: *TaggedBegin, End: *TaggedBegin + Size);
311}
312#pragma GCC diagnostic pop
313
314inline void *untagPointer(void *Ptr) {
315 return reinterpret_cast<void *>(untagPointer(Ptr: reinterpret_cast<uptr>(Ptr)));
316}
317
318inline void *loadTag(void *Ptr) {
319 return reinterpret_cast<void *>(loadTag(Ptr: reinterpret_cast<uptr>(Ptr)));
320}
321
322inline void *addFixedTag(void *Ptr, uptr Tag) {
323 return reinterpret_cast<void *>(
324 addFixedTag(Ptr: reinterpret_cast<uptr>(Ptr), Tag));
325}
326
327template <typename Config>
328inline constexpr bool allocatorSupportsMemoryTagging() {
329 return archSupportsMemoryTagging() && Config::getMaySupportMemoryTagging() &&
330 (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
331}
332
333} // namespace scudo
334
335#endif
336

source code of compiler-rt/lib/scudo/standalone/memtag.h