1	//===-- combined.h ----------------------------------------------*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#ifndef SCUDO_COMBINED_H_
10	#define SCUDO_COMBINED_H_
11
12	#include "allocator_config_wrapper.h"
13	#include "atomic_helpers.h"
14	#include "chunk.h"
15	#include "common.h"
16	#include "flags.h"
17	#include "flags_parser.h"
18	#include "local_cache.h"
19	#include "mem_map.h"
20	#include "memtag.h"
21	#include "mutex.h"
22	#include "options.h"
23	#include "quarantine.h"
24	#include "report.h"
25	#include "secondary.h"
26	#include "stack_depot.h"
27	#include "string_utils.h"
28	#include "tsd.h"
29
30	#include "scudo/interface.h"
31
32	#ifdef GWP_ASAN_HOOKS
33	#include "gwp_asan/guarded_pool_allocator.h"
34	#include "gwp_asan/optional/backtrace.h"
35	#include "gwp_asan/optional/segv_handler.h"
36	#endif // GWP_ASAN_HOOKS
37
38	extern "C" inline void EmptyCallback() {}
39
40	#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
41	// This function is not part of the NDK so it does not appear in any public
42	// header files. We only declare/use it when targeting the platform.
43	extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
44	size_t num_entries);
45	#endif
46
47	namespace scudo {
48
49	template <class Config, void (*PostInitCallback)(void) = EmptyCallback>
50	class Allocator {
51	public:
52	using AllocatorConfig = BaseConfig<Config>;
53	using PrimaryT =
54	typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
55	using SecondaryT =
56	typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
57	using CacheT = typename PrimaryT::CacheT;
58	typedef Allocator<Config, PostInitCallback> ThisT;
59	typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
60
61	void callPostInitCallback() {
62	pthread_once(once_control: &PostInitNonce, init_routine: PostInitCallback);
63	}
64
65	struct QuarantineCallback {
66	explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache)
67	: Allocator(Instance), Cache(LocalCache) {}
68
69	// Chunk recycling function, returns a quarantined chunk to the backend,
70	// first making sure it hasn't been tampered with.
71	void recycle(void *Ptr) {
72	Chunk::UnpackedHeader Header;
73	Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
74	if (UNLIKELY(Header.State != Chunk::State::Quarantined))
75	reportInvalidChunkState(Action: AllocatorAction::Recycling, Ptr);
76
77	Header.State = Chunk::State::Available;
78	Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
79
80	if (allocatorSupportsMemoryTagging<AllocatorConfig>())
81	Ptr = untagPointer(Ptr);
82	void *BlockBegin = Allocator::getBlockBegin(Ptr, Header: &Header);
83	Cache.deallocate(Header.ClassId, BlockBegin);
84	}
85
86	// We take a shortcut when allocating a quarantine batch by working with the
87	// appropriate class ID instead of using Size. The compiler should optimize
88	// the class ID computation and work with the associated cache directly.
89	void *allocate(UNUSED uptr Size) {
90	const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
91	sizeof(QuarantineBatch) + Chunk::getHeaderSize());
92	void *Ptr = Cache.allocate(QuarantineClassId);
93	// Quarantine batch allocation failure is fatal.
94	if (UNLIKELY(!Ptr))
95	reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
96
97	Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) +
98	Chunk::getHeaderSize());
99	Chunk::UnpackedHeader Header = {};
100	Header.ClassId = QuarantineClassId & Chunk::ClassIdMask;
101	Header.SizeOrUnusedBytes = sizeof(QuarantineBatch);
102	Header.State = Chunk::State::Allocated;
103	Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
104
105	// Reset tag to 0 as this chunk may have been previously used for a tagged
106	// user allocation.
107	if (UNLIKELY(useMemoryTagging<AllocatorConfig>(
108	Allocator.Primary.Options.load())))
109	storeTags(Begin: reinterpret_cast<uptr>(Ptr),
110	End: reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
111
112	return Ptr;
113	}
114
115	void deallocate(void *Ptr) {
116	const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
117	sizeof(QuarantineBatch) + Chunk::getHeaderSize());
118	Chunk::UnpackedHeader Header;
119	Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
120
121	if (UNLIKELY(Header.State != Chunk::State::Allocated))
122	reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
123	DCHECK_EQ(Header.ClassId, QuarantineClassId);
124	DCHECK_EQ(Header.Offset, 0);
125	DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch));
126
127	Header.State = Chunk::State::Available;
128	Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
129	Cache.deallocate(QuarantineClassId,
130	reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
131	Chunk::getHeaderSize()));
132	}
133
134	private:
135	ThisT &Allocator;
136	CacheT &Cache;
137	};
138
139	typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
140	typedef typename QuarantineT::CacheT QuarantineCacheT;
141
142	void init() {
143	performSanityChecks();
144
145	// Check if hardware CRC32 is supported in the binary and by the platform,
146	// if so, opt for the CRC32 hardware version of the checksum.
147	if (&computeHardwareCRC32 && hasHardwareCRC32())
148	HashAlgorithm = Checksum::HardwareCRC32;
149
150	if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie))))
151	Cookie = static_cast<u32>(getMonotonicTime() ^
152	(reinterpret_cast<uptr>(this) >> 4));
153
154	initFlags();
155	reportUnrecognizedFlags();
156
157	// Store some flags locally.
158	if (getFlags()->may_return_null)
159	Primary.Options.set(OptionBit::MayReturnNull);
160	if (getFlags()->zero_contents)
161	Primary.Options.setFillContentsMode(ZeroFill);
162	else if (getFlags()->pattern_fill_contents)
163	Primary.Options.setFillContentsMode(PatternOrZeroFill);
164	if (getFlags()->dealloc_type_mismatch)
165	Primary.Options.set(OptionBit::DeallocTypeMismatch);
166	if (getFlags()->delete_size_mismatch)
167	Primary.Options.set(OptionBit::DeleteSizeMismatch);
168	if (allocatorSupportsMemoryTagging<AllocatorConfig>() &&
169	systemSupportsMemoryTagging())
170	Primary.Options.set(OptionBit::UseMemoryTagging);
171
172	QuarantineMaxChunkSize =
173	static_cast<u32>(getFlags()->quarantine_max_chunk_size);
174
175	Stats.init();
176	const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
177	Primary.init(ReleaseToOsIntervalMs);
178	Secondary.init(&Stats, ReleaseToOsIntervalMs);
179	Quarantine.init(
180	static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
181	static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
182	}
183
184	void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
185	AllocationRingBuffer *RB = getRingBuffer();
186	if (RB)
187	RB->Depot->enable();
188	RingBufferInitLock.unlock();
189	}
190
191	void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
192	RingBufferInitLock.lock();
193	AllocationRingBuffer *RB = getRingBuffer();
194	if (RB)
195	RB->Depot->disable();
196	}
197
198	// Initialize the embedded GWP-ASan instance. Requires the main allocator to
199	// be functional, best called from PostInitCallback.
200	void initGwpAsan() {
201	#ifdef GWP_ASAN_HOOKS
202	gwp_asan::options::Options Opt;
203	Opt.Enabled = getFlags()->GWP_ASAN_Enabled;
204	Opt.MaxSimultaneousAllocations =
205	getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
206	Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
207	Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;
208	Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;
209	// Embedded GWP-ASan is locked through the Scudo atfork handler (via
210	// Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
211	// handler.
212	Opt.InstallForkHandlers = false;
213	Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
214	GuardedAlloc.init(Opts: Opt);
215
216	if (Opt.InstallSignalHandlers)
217	gwp_asan::segv_handler::installSignalHandlers(
218	GPA: &GuardedAlloc, Printf,
219	PrintBacktrace: gwp_asan::backtrace::getPrintBacktraceFunction(),
220	SegvBacktrace: gwp_asan::backtrace::getSegvBacktraceFunction(),
221	Recoverable: Opt.Recoverable);
222
223	GuardedAllocSlotSize =
224	GuardedAlloc.getAllocatorState()->maximumAllocationSize();
225	Stats.add(I: StatFree, V: static_cast<uptr>(Opt.MaxSimultaneousAllocations) *
226	GuardedAllocSlotSize);
227	#endif // GWP_ASAN_HOOKS
228	}
229
230	#ifdef GWP_ASAN_HOOKS
231	const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {
232	return GuardedAlloc.getMetadataRegion();
233	}
234
235	const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {
236	return GuardedAlloc.getAllocatorState();
237	}
238	#endif // GWP_ASAN_HOOKS
239
240	ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
241	TSDRegistry.initThreadMaybe(this, MinimalInit);
242	}
243
244	void unmapTestOnly() {
245	unmapRingBuffer();
246	TSDRegistry.unmapTestOnly(this);
247	Primary.unmapTestOnly();
248	Secondary.unmapTestOnly();
249	#ifdef GWP_ASAN_HOOKS
250	if (getFlags()->GWP_ASAN_InstallSignalHandlers)
251	gwp_asan::segv_handler::uninstallSignalHandlers();
252	GuardedAlloc.uninitTestOnly();
253	#endif // GWP_ASAN_HOOKS
254	}
255
256	TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
257	QuarantineT *getQuarantine() { return &Quarantine; }
258
259	// The Cache must be provided zero-initialized.
260	void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); }
261
262	// Release the resources used by a TSD, which involves:
263	// - draining the local quarantine cache to the global quarantine;
264	// - releasing the cached pointers back to the Primary;
265	// - unlinking the local stats from the global ones (destroying the cache does
266	// the last two items).
267	void commitBack(TSD<ThisT> *TSD) {
268	TSD->assertLocked(/*BypassCheck=*/true);
269	Quarantine.drain(&TSD->getQuarantineCache(),
270	QuarantineCallback(*this, TSD->getCache()));
271	TSD->getCache().destroy(&Stats);
272	}
273
274	void drainCache(TSD<ThisT> *TSD) {
275	TSD->assertLocked(/*BypassCheck=*/true);
276	Quarantine.drainAndRecycle(&TSD->getQuarantineCache(),
277	QuarantineCallback(*this, TSD->getCache()));
278	TSD->getCache().drain();
279	}
280	void drainCaches() { TSDRegistry.drainCaches(this); }
281
282	ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
283	if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
284	return Ptr;
285	auto UntaggedPtr = untagPointer(Ptr);
286	if (UntaggedPtr != Ptr)
287	return UntaggedPtr;
288	// Secondary, or pointer allocated while memory tagging is unsupported or
289	// disabled. The tag mismatch is okay in the latter case because tags will
290	// not be checked.
291	return addHeaderTag(Ptr);
292	}
293
294	ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
295	if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
296	return Ptr;
297	return addFixedTag(Ptr, Tag: 2);
298	}
299
300	ALWAYS_INLINE void *addHeaderTag(void *Ptr) {
301	return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));
302	}
303
304	NOINLINE u32 collectStackTrace(UNUSED StackDepot *Depot) {
305	#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
306	// Discard collectStackTrace() frame and allocator function frame.
307	constexpr uptr DiscardFrames = 2;
308	uptr Stack[MaxTraceSize + DiscardFrames];
309	uptr Size =
310	android_unsafe_frame_pointer_chase(Stack, MaxTraceSize + DiscardFrames);
311	Size = Min<uptr>(Size, MaxTraceSize + DiscardFrames);
312	return Depot->insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
313	#else
314	return 0;
315	#endif
316	}
317
318	uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr,
319	uptr ClassId) {
320	if (!Options.get(Opt: OptionBit::UseOddEvenTags))
321	return 0;
322
323	// If a chunk's tag is odd, we want the tags of the surrounding blocks to be
324	// even, and vice versa. Blocks are laid out Size bytes apart, and adding
325	// Size to Ptr will flip the least significant set bit of Size in Ptr, so
326	// that bit will have the pattern 010101... for consecutive blocks, which we
327	// can use to determine which tag mask to use.
328	return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);
329	}
330
331	NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
332	uptr Alignment = MinAlignment,
333	bool ZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {
334	initThreadMaybe();
335
336	const Options Options = Primary.Options.load();
337	if (UNLIKELY(Alignment > MaxAlignment)) {
338	if (Options.get(Opt: OptionBit::MayReturnNull))
339	return nullptr;
340	reportAlignmentTooBig(Alignment, MaxAlignment);
341	}
342	if (Alignment < MinAlignment)
343	Alignment = MinAlignment;
344
345	#ifdef GWP_ASAN_HOOKS
346	if (UNLIKELY(GuardedAlloc.shouldSample())) {
347	if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
348	Stats.lock();
349	Stats.add(I: StatAllocated, V: GuardedAllocSlotSize);
350	Stats.sub(I: StatFree, V: GuardedAllocSlotSize);
351	Stats.unlock();
352	return Ptr;
353	}
354	}
355	#endif // GWP_ASAN_HOOKS
356
357	const FillContentsMode FillContents = ZeroContents ? ZeroFill
358	: TSDRegistry.getDisableMemInit()
359	? NoFill
360	: Options.getFillContentsMode();
361
362	// If the requested size happens to be 0 (more common than you might think),
363	// allocate MinAlignment bytes on top of the header. Then add the extra
364	// bytes required to fulfill the alignment requirements: we allocate enough
365	// to be sure that there will be an address in the block that will satisfy
366	// the alignment.
367	const uptr NeededSize =
368	roundUp(X: Size, Boundary: MinAlignment) +
369	((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
370
371	// Takes care of extravagantly large sizes as well as integer overflows.
372	static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
373	if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
374	if (Options.get(Opt: OptionBit::MayReturnNull))
375	return nullptr;
376	reportAllocationSizeTooBig(UserSize: Size, TotalSize: NeededSize, MaxSize: MaxAllowedMallocSize);
377	}
378	DCHECK_LE(Size, NeededSize);
379
380	void *Block = nullptr;
381	uptr ClassId = 0;
382	uptr SecondaryBlockEnd = 0;
383	if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
384	ClassId = SizeClassMap::getClassIdBySize(NeededSize);
385	DCHECK_NE(ClassId, 0U);
386	typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
387	Block = TSD->getCache().allocate(ClassId);
388	// If the allocation failed, retry in each successively larger class until
389	// it fits. If it fails to fit in the largest class, fallback to the
390	// Secondary.
391	if (UNLIKELY(!Block)) {
392	while (ClassId < SizeClassMap::LargestClassId && !Block)
393	Block = TSD->getCache().allocate(++ClassId);
394	if (!Block)
395	ClassId = 0;
396	}
397	}
398	if (UNLIKELY(ClassId == 0)) {
399	Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,
400	FillContents);
401	}
402
403	if (UNLIKELY(!Block)) {
404	if (Options.get(Opt: OptionBit::MayReturnNull))
405	return nullptr;
406	printStats();
407	reportOutOfMemory(RequestedSize: NeededSize);
408	}
409
410	const uptr BlockUptr = reinterpret_cast<uptr>(Block);
411	const uptr UnalignedUserPtr = BlockUptr + Chunk::getHeaderSize();
412	const uptr UserPtr = roundUp(X: UnalignedUserPtr, Boundary: Alignment);
413
414	void *Ptr = reinterpret_cast<void *>(UserPtr);
415	void *TaggedPtr = Ptr;
416	if (LIKELY(ClassId)) {
417	// We only need to zero or tag the contents for Primary backed
418	// allocations. We only set tags for primary allocations in order to avoid
419	// faulting potentially large numbers of pages for large secondary
420	// allocations. We assume that guard pages are enough to protect these
421	// allocations.
422	//
423	// FIXME: When the kernel provides a way to set the background tag of a
424	// mapping, we should be able to tag secondary allocations as well.
425	//
426	// When memory tagging is enabled, zeroing the contents is done as part of
427	// setting the tag.
428	if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
429	uptr PrevUserPtr;
430	Chunk::UnpackedHeader Header;
431	const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
432	const uptr BlockEnd = BlockUptr + BlockSize;
433	// If possible, try to reuse the UAF tag that was set by deallocate().
434	// For simplicity, only reuse tags if we have the same start address as
435	// the previous allocation. This handles the majority of cases since
436	// most allocations will not be more aligned than the minimum alignment.
437	//
438	// We need to handle situations involving reclaimed chunks, and retag
439	// the reclaimed portions if necessary. In the case where the chunk is
440	// fully reclaimed, the chunk's header will be zero, which will trigger
441	// the code path for new mappings and invalid chunks that prepares the
442	// chunk from scratch. There are three possibilities for partial
443	// reclaiming:
444	//
445	// (1) Header was reclaimed, data was partially reclaimed.
446	// (2) Header was not reclaimed, all data was reclaimed (e.g. because
447	// data started on a page boundary).
448	// (3) Header was not reclaimed, data was partially reclaimed.
449	//
450	// Case (1) will be handled in the same way as for full reclaiming,
451	// since the header will be zero.
452	//
453	// We can detect case (2) by loading the tag from the start
454	// of the chunk. If it is zero, it means that either all data was
455	// reclaimed (since we never use zero as the chunk tag), or that the
456	// previous allocation was of size zero. Either way, we need to prepare
457	// a new chunk from scratch.
458	//
459	// We can detect case (3) by moving to the next page (if covered by the
460	// chunk) and loading the tag of its first granule. If it is zero, it
461	// means that all following pages may need to be retagged. On the other
462	// hand, if it is nonzero, we can assume that all following pages are
463	// still tagged, according to the logic that if any of the pages
464	// following the next page were reclaimed, the next page would have been
465	// reclaimed as well.
466	uptr TaggedUserPtr;
467	if (getChunkFromBlock(Block: BlockUptr, Chunk: &PrevUserPtr, Header: &Header) &&
468	PrevUserPtr == UserPtr &&
469	(TaggedUserPtr = loadTag(Ptr: UserPtr)) != UserPtr) {
470	uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
471	const uptr NextPage = roundUp(X: TaggedUserPtr, Boundary: getPageSizeCached());
472	if (NextPage < PrevEnd && loadTag(Ptr: NextPage) != NextPage)
473	PrevEnd = NextPage;
474	TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
475	resizeTaggedChunk(OldPtr: PrevEnd, NewPtr: TaggedUserPtr + Size, NewSize: Size, BlockEnd);
476	if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
477	// If an allocation needs to be zeroed (i.e. calloc) we can normally
478	// avoid zeroing the memory now since we can rely on memory having
479	// been zeroed on free, as this is normally done while setting the
480	// UAF tag. But if tagging was disabled per-thread when the memory
481	// was freed, it would not have been retagged and thus zeroed, and
482	// therefore it needs to be zeroed now.
483	memset(s: TaggedPtr, c: 0,
484	n: Min(A: Size, B: roundUp(X: PrevEnd - TaggedUserPtr,
485	Boundary: archMemoryTagGranuleSize())));
486	} else if (Size) {
487	// Clear any stack metadata that may have previously been stored in
488	// the chunk data.
489	memset(s: TaggedPtr, c: 0, n: archMemoryTagGranuleSize());
490	}
491	} else {
492	const uptr OddEvenMask =
493	computeOddEvenMaskForPointerMaybe(Options, Ptr: BlockUptr, ClassId);
494	TaggedPtr = prepareTaggedChunk(Ptr, Size, ExcludeMask: OddEvenMask, BlockEnd);
495	}
496	storePrimaryAllocationStackMaybe(Options, Ptr);
497	} else {
498	Block = addHeaderTag(Block);
499	Ptr = addHeaderTag(Ptr);
500	if (UNLIKELY(FillContents != NoFill)) {
501	// This condition is not necessarily unlikely, but since memset is
502	// costly, we might as well mark it as such.
503	memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
504	PrimaryT::getSizeByClassId(ClassId));
505	}
506	}
507	} else {
508	Block = addHeaderTag(Block);
509	Ptr = addHeaderTag(Ptr);
510	if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
511	storeTags(Begin: reinterpret_cast<uptr>(Block), End: reinterpret_cast<uptr>(Ptr));
512	storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
513	}
514	}
515
516	Chunk::UnpackedHeader Header = {};
517	if (UNLIKELY(UnalignedUserPtr != UserPtr)) {
518	const uptr Offset = UserPtr - UnalignedUserPtr;
519	DCHECK_GE(Offset, 2 * sizeof(u32));
520	// The BlockMarker has no security purpose, but is specifically meant for
521	// the chunk iteration function that can be used in debugging situations.
522	// It is the only situation where we have to locate the start of a chunk
523	// based on its block address.
524	reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
525	reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
526	Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
527	}
528	Header.ClassId = ClassId & Chunk::ClassIdMask;
529	Header.State = Chunk::State::Allocated;
530	Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
531	Header.SizeOrUnusedBytes =
532	(ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size)) &
533	Chunk::SizeOrUnusedBytesMask;
534	Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
535
536	return TaggedPtr;
537	}
538
539	NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,
540	UNUSED uptr Alignment = MinAlignment) {
541	if (UNLIKELY(!Ptr))
542	return;
543
544	// For a deallocation, we only ensure minimal initialization, meaning thread
545	// local data will be left uninitialized for now (when using ELF TLS). The
546	// fallback cache will be used instead. This is a workaround for a situation
547	// where the only heap operation performed in a thread would be a free past
548	// the TLS destructors, ending up in initialized thread specific data never
549	// being destroyed properly. Any other heap operation will do a full init.
550	initThreadMaybe(/*MinimalInit=*/MinimalInit: true);
551
552	#ifdef GWP_ASAN_HOOKS
553	if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
554	GuardedAlloc.deallocate(Ptr);
555	Stats.lock();
556	Stats.add(I: StatFree, V: GuardedAllocSlotSize);
557	Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
558	Stats.unlock();
559	return;
560	}
561	#endif // GWP_ASAN_HOOKS
562
563	if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
564	reportMisalignedPointer(Action: AllocatorAction::Deallocating, Ptr);
565
566	void *TaggedPtr = Ptr;
567	Ptr = getHeaderTaggedPointer(Ptr);
568
569	Chunk::UnpackedHeader Header;
570	Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
571
572	if (UNLIKELY(Header.State != Chunk::State::Allocated))
573	reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
574
575	const Options Options = Primary.Options.load();
576	if (Options.get(Opt: OptionBit::DeallocTypeMismatch)) {
577	if (UNLIKELY(Header.OriginOrWasZeroed != Origin)) {
578	// With the exception of memalign'd chunks, that can be still be free'd.
579	if (Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
580	Origin != Chunk::Origin::Malloc)
581	reportDeallocTypeMismatch(Action: AllocatorAction::Deallocating, Ptr,
582	TypeA: Header.OriginOrWasZeroed, TypeB: Origin);
583	}
584	}
585
586	const uptr Size = getSize(Ptr, Header: &Header);
587	if (DeleteSize && Options.get(Opt: OptionBit::DeleteSizeMismatch)) {
588	if (UNLIKELY(DeleteSize != Size))
589	reportDeleteSizeMismatch(Ptr, Size: DeleteSize, ExpectedSize: Size);
590	}
591
592	quarantineOrDeallocateChunk(Options, TaggedPtr, Header: &Header, Size);
593	}
594
595	void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
596	initThreadMaybe();
597
598	const Options Options = Primary.Options.load();
599	if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
600	if (Options.get(Opt: OptionBit::MayReturnNull))
601	return nullptr;
602	reportAllocationSizeTooBig(UserSize: NewSize, TotalSize: 0, MaxSize: MaxAllowedMallocSize);
603	}
604
605	// The following cases are handled by the C wrappers.
606	DCHECK_NE(OldPtr, nullptr);
607	DCHECK_NE(NewSize, 0);
608
609	#ifdef GWP_ASAN_HOOKS
610	if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {
611	uptr OldSize = GuardedAlloc.getSize(Ptr: OldPtr);
612	void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
613	if (NewPtr)
614	memcpy(dest: NewPtr, src: OldPtr, n: (NewSize < OldSize) ? NewSize : OldSize);
615	GuardedAlloc.deallocate(Ptr: OldPtr);
616	Stats.lock();
617	Stats.add(I: StatFree, V: GuardedAllocSlotSize);
618	Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
619	Stats.unlock();
620	return NewPtr;
621	}
622	#endif // GWP_ASAN_HOOKS
623
624	void *OldTaggedPtr = OldPtr;
625	OldPtr = getHeaderTaggedPointer(Ptr: OldPtr);
626
627	if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))
628	reportMisalignedPointer(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
629
630	Chunk::UnpackedHeader Header;
631	Chunk::loadHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
632
633	if (UNLIKELY(Header.State != Chunk::State::Allocated))
634	reportInvalidChunkState(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
635
636	// Pointer has to be allocated with a malloc-type function. Some
637	// applications think that it is OK to realloc a memalign'ed pointer, which
638	// will trigger this check. It really isn't.
639	if (Options.get(Opt: OptionBit::DeallocTypeMismatch)) {
640	if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Malloc))
641	reportDeallocTypeMismatch(Action: AllocatorAction::Reallocating, Ptr: OldPtr,
642	TypeA: Header.OriginOrWasZeroed,
643	TypeB: Chunk::Origin::Malloc);
644	}
645
646	void *BlockBegin = getBlockBegin(Ptr: OldTaggedPtr, Header: &Header);
647	uptr BlockEnd;
648	uptr OldSize;
649	const uptr ClassId = Header.ClassId;
650	if (LIKELY(ClassId)) {
651	BlockEnd = reinterpret_cast<uptr>(BlockBegin) +
652	SizeClassMap::getSizeByClassId(ClassId);
653	OldSize = Header.SizeOrUnusedBytes;
654	} else {
655	BlockEnd = SecondaryT::getBlockEnd(BlockBegin);
656	OldSize = BlockEnd - (reinterpret_cast<uptr>(OldTaggedPtr) +
657	Header.SizeOrUnusedBytes);
658	}
659	// If the new chunk still fits in the previously allocated block (with a
660	// reasonable delta), we just keep the old block, and update the chunk
661	// header to reflect the size change.
662	if (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize <= BlockEnd) {
663	if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) {
664	Header.SizeOrUnusedBytes =
665	(ClassId ? NewSize
666	: BlockEnd -
667	(reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
668	Chunk::SizeOrUnusedBytesMask;
669	Chunk::storeHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
670	if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
671	if (ClassId) {
672	resizeTaggedChunk(OldPtr: reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
673	NewPtr: reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
674	NewSize, BlockEnd: untagPointer(Ptr: BlockEnd));
675	storePrimaryAllocationStackMaybe(Options, Ptr: OldPtr);
676	} else {
677	storeSecondaryAllocationStackMaybe(Options, Ptr: OldPtr, Size: NewSize);
678	}
679	}
680	return OldTaggedPtr;
681	}
682	}
683
684	// Otherwise we allocate a new one, and deallocate the old one. Some
685	// allocators will allocate an even larger chunk (by a fixed factor) to
686	// allow for potential further in-place realloc. The gains of such a trick
687	// are currently unclear.
688	void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
689	if (LIKELY(NewPtr)) {
690	memcpy(dest: NewPtr, src: OldTaggedPtr, n: Min(A: NewSize, B: OldSize));
691	quarantineOrDeallocateChunk(Options, TaggedPtr: OldTaggedPtr, Header: &Header, Size: OldSize);
692	}
693	return NewPtr;
694	}
695
696	// TODO(kostyak): disable() is currently best-effort. There are some small
697	// windows of time when an allocation could still succeed after
698	// this function finishes. We will revisit that later.
699	void disable() NO_THREAD_SAFETY_ANALYSIS {
700	initThreadMaybe();
701	#ifdef GWP_ASAN_HOOKS
702	GuardedAlloc.disable();
703	#endif
704	TSDRegistry.disable();
705	Stats.disable();
706	Quarantine.disable();
707	Primary.disable();
708	Secondary.disable();
709	disableRingBuffer();
710	}
711
712	void enable() NO_THREAD_SAFETY_ANALYSIS {
713	initThreadMaybe();
714	enableRingBuffer();
715	Secondary.enable();
716	Primary.enable();
717	Quarantine.enable();
718	Stats.enable();
719	TSDRegistry.enable();
720	#ifdef GWP_ASAN_HOOKS
721	GuardedAlloc.enable();
722	#endif
723	}
724
725	// The function returns the amount of bytes required to store the statistics,
726	// which might be larger than the amount of bytes provided. Note that the
727	// statistics buffer is not necessarily constant between calls to this
728	// function. This can be called with a null buffer or zero size for buffer
729	// sizing purposes.
730	uptr getStats(char *Buffer, uptr Size) {
731	ScopedString Str;
732	const uptr Length = getStats(&Str) + 1;
733	if (Length < Size)
734	Size = Length;
735	if (Buffer && Size) {
736	memcpy(dest: Buffer, src: Str.data(), n: Size);
737	Buffer[Size - 1] = '\0';
738	}
739	return Length;
740	}
741
742	void printStats() {
743	ScopedString Str;
744	getStats(&Str);
745	Str.output();
746	}
747
748	void printFragmentationInfo() {
749	ScopedString Str;
750	Primary.getFragmentationInfo(&Str);
751	// Secondary allocator dumps the fragmentation data in getStats().
752	Str.output();
753	}
754
755	void releaseToOS(ReleaseToOS ReleaseType) {
756	initThreadMaybe();
757	if (ReleaseType == ReleaseToOS::ForceAll)
758	drainCaches();
759	Primary.releaseToOS(ReleaseType);
760	Secondary.releaseToOS();
761	}
762
763	// Iterate over all chunks and call a callback for all busy chunks located
764	// within the provided memory range. Said callback must not use this allocator
765	// or a deadlock can ensue. This fits Android's malloc_iterate() needs.
766	void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
767	void *Arg) {
768	initThreadMaybe();
769	if (archSupportsMemoryTagging())
770	Base = untagPointer(Ptr: Base);
771	const uptr From = Base;
772	const uptr To = Base + Size;
773	bool MayHaveTaggedPrimary =
774	allocatorSupportsMemoryTagging<AllocatorConfig>() &&
775	systemSupportsMemoryTagging();
776	auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,
777	Arg](uptr Block) {
778	if (Block < From || Block >= To)
779	return;
780	uptr Chunk;
781	Chunk::UnpackedHeader Header;
782	if (MayHaveTaggedPrimary) {
783	// A chunk header can either have a zero tag (tagged primary) or the
784	// header tag (secondary, or untagged primary). We don't know which so
785	// try both.
786	ScopedDisableMemoryTagChecks x;
787	if (!getChunkFromBlock(Block, Chunk: &Chunk, Header: &Header) &&
788	!getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
789	return;
790	} else {
791	if (!getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
792	return;
793	}
794	if (Header.State == Chunk::State::Allocated) {
795	uptr TaggedChunk = Chunk;
796	if (allocatorSupportsMemoryTagging<AllocatorConfig>())
797	TaggedChunk = untagPointer(Ptr: TaggedChunk);
798	if (useMemoryTagging<AllocatorConfig>(Primary.Options.load()))
799	TaggedChunk = loadTag(Ptr: Chunk);
800	Callback(TaggedChunk, getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header),
801	Arg);
802	}
803	};
804	Primary.iterateOverBlocks(Lambda);
805	Secondary.iterateOverBlocks(Lambda);
806	#ifdef GWP_ASAN_HOOKS
807	GuardedAlloc.iterate(Base: reinterpret_cast<void *>(Base), Size, Cb: Callback, Arg);
808	#endif
809	}
810
811	bool canReturnNull() {
812	initThreadMaybe();
813	return Primary.Options.load().get(OptionBit::MayReturnNull);
814	}
815
816	bool setOption(Option O, sptr Value) {
817	initThreadMaybe();
818	if (O == Option::MemtagTuning) {
819	// Enabling odd/even tags involves a tradeoff between use-after-free
820	// detection and buffer overflow detection. Odd/even tags make it more
821	// likely for buffer overflows to be detected by increasing the size of
822	// the guaranteed "red zone" around the allocation, but on the other hand
823	// use-after-free is less likely to be detected because the tag space for
824	// any particular chunk is cut in half. Therefore we use this tuning
825	// setting to control whether odd/even tags are enabled.
826	if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
827	Primary.Options.set(OptionBit::UseOddEvenTags);
828	else if (Value == M_MEMTAG_TUNING_UAF)
829	Primary.Options.clear(OptionBit::UseOddEvenTags);
830	return true;
831	} else {
832	// We leave it to the various sub-components to decide whether or not they
833	// want to handle the option, but we do not want to short-circuit
834	// execution if one of the setOption was to return false.
835	const bool PrimaryResult = Primary.setOption(O, Value);
836	const bool SecondaryResult = Secondary.setOption(O, Value);
837	const bool RegistryResult = TSDRegistry.setOption(O, Value);
838	return PrimaryResult && SecondaryResult && RegistryResult;
839	}
840	return false;
841	}
842
843	// Return the usable size for a given chunk. Technically we lie, as we just
844	// report the actual size of a chunk. This is done to counteract code actively
845	// writing past the end of a chunk (like sqlite3) when the usable size allows
846	// for it, which then forces realloc to copy the usable size of a chunk as
847	// opposed to its actual size.
848	uptr getUsableSize(const void *Ptr) {
849	if (UNLIKELY(!Ptr))
850	return 0;
851
852	return getAllocSize(Ptr);
853	}
854
855	uptr getAllocSize(const void *Ptr) {
856	initThreadMaybe();
857
858	#ifdef GWP_ASAN_HOOKS
859	if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
860	return GuardedAlloc.getSize(Ptr);
861	#endif // GWP_ASAN_HOOKS
862
863	Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
864	Chunk::UnpackedHeader Header;
865	Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
866
867	// Getting the alloc size of a chunk only makes sense if it's allocated.
868	if (UNLIKELY(Header.State != Chunk::State::Allocated))
869	reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr: const_cast<void *>(Ptr));
870
871	return getSize(Ptr, Header: &Header);
872	}
873
874	void getStats(StatCounters S) {
875	initThreadMaybe();
876	Stats.get(S);
877	}
878
879	// Returns true if the pointer provided was allocated by the current
880	// allocator instance, which is compliant with tcmalloc's ownership concept.
881	// A corrupted chunk will not be reported as owned, which is WAI.
882	bool isOwned(const void *Ptr) {
883	initThreadMaybe();
884	#ifdef GWP_ASAN_HOOKS
885	if (GuardedAlloc.pointerIsMine(Ptr))
886	return true;
887	#endif // GWP_ASAN_HOOKS
888	if (!Ptr || !isAligned(X: reinterpret_cast<uptr>(Ptr), Alignment: MinAlignment))
889	return false;
890	Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
891	Chunk::UnpackedHeader Header;
892	return Chunk::isValid(Cookie, Ptr, NewUnpackedHeader: &Header) &&
893	Header.State == Chunk::State::Allocated;
894	}
895
896	bool useMemoryTaggingTestOnly() const {
897	return useMemoryTagging<AllocatorConfig>(Primary.Options.load());
898	}
899	void disableMemoryTagging() {
900	// If we haven't been initialized yet, we need to initialize now in order to
901	// prevent a future call to initThreadMaybe() from enabling memory tagging
902	// based on feature detection. But don't call initThreadMaybe() because it
903	// may end up calling the allocator (via pthread_atfork, via the post-init
904	// callback), which may cause mappings to be created with memory tagging
905	// enabled.
906	TSDRegistry.initOnceMaybe(this);
907	if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
908	Secondary.disableMemoryTagging();
909	Primary.Options.clear(OptionBit::UseMemoryTagging);
910	}
911	}
912
913	void setTrackAllocationStacks(bool Track) {
914	initThreadMaybe();
915	if (getFlags()->allocation_ring_buffer_size <= 0) {
916	DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));
917	return;
918	}
919
920	if (Track) {
921	initRingBufferMaybe();
922	Primary.Options.set(OptionBit::TrackAllocationStacks);
923	} else
924	Primary.Options.clear(OptionBit::TrackAllocationStacks);
925	}
926
927	void setFillContents(FillContentsMode FillContents) {
928	initThreadMaybe();
929	Primary.Options.setFillContentsMode(FillContents);
930	}
931
932	void setAddLargeAllocationSlack(bool AddSlack) {
933	initThreadMaybe();
934	if (AddSlack)
935	Primary.Options.set(OptionBit::AddLargeAllocationSlack);
936	else
937	Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
938	}
939
940	const char *getStackDepotAddress() {
941	initThreadMaybe();
942	AllocationRingBuffer *RB = getRingBuffer();
943	return RB ? reinterpret_cast<char *>(RB->Depot) : nullptr;
944	}
945
946	uptr getStackDepotSize() {
947	initThreadMaybe();
948	AllocationRingBuffer *RB = getRingBuffer();
949	return RB ? RB->StackDepotSize : 0;
950	}
951
952	const char *getRegionInfoArrayAddress() const {
953	return Primary.getRegionInfoArrayAddress();
954	}
955
956	static uptr getRegionInfoArraySize() {
957	return PrimaryT::getRegionInfoArraySize();
958	}
959
960	const char *getRingBufferAddress() {
961	initThreadMaybe();
962	return reinterpret_cast<char *>(getRingBuffer());
963	}
964
965	uptr getRingBufferSize() {
966	initThreadMaybe();
967	AllocationRingBuffer *RB = getRingBuffer();
968	return RB && RB->RingBufferElements
969	? ringBufferSizeInBytes(RingBufferElements: RB->RingBufferElements)
970	: 0;
971	}
972
973	static const uptr MaxTraceSize = 64;
974
975	static void collectTraceMaybe(const StackDepot *Depot,
976	uintptr_t (&Trace)[MaxTraceSize], u32 Hash) {
977	uptr RingPos, Size;
978	if (!Depot->find(Hash, RingPosPtr: &RingPos, SizePtr: &Size))
979	return;
980	for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
981	Trace[I] = static_cast<uintptr_t>(Depot->at(RingPos: RingPos + I));
982	}
983
984	static void getErrorInfo(struct scudo_error_info *ErrorInfo,
985	uintptr_t FaultAddr, const char *DepotPtr,
986	size_t DepotSize, const char *RegionInfoPtr,
987	const char *RingBufferPtr, size_t RingBufferSize,
988	const char *Memory, const char *MemoryTags,
989	uintptr_t MemoryAddr, size_t MemorySize) {
990	// N.B. we need to support corrupted data in any of the buffers here. We get
991	// this information from an external process (the crashing process) that
992	// should not be able to crash the crash dumper (crash_dump on Android).
993	// See also the get_error_info_fuzzer.
994	*ErrorInfo = {};
995	if (!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
996	MemoryAddr + MemorySize < MemoryAddr)
997	return;
998
999	const StackDepot *Depot = nullptr;
1000	if (DepotPtr) {
1001	// check for corrupted StackDepot. First we need to check whether we can
1002	// read the metadata, then whether the metadata matches the size.
1003	if (DepotSize < sizeof(*Depot))
1004	return;
1005	Depot = reinterpret_cast<const StackDepot *>(DepotPtr);
1006	if (!Depot->isValid(BufSize: DepotSize))
1007	return;
1008	}
1009
1010	size_t NextErrorReport = 0;
1011
1012	// Check for OOB in the current block and the two surrounding blocks. Beyond
1013	// that, UAF is more likely.
1014	if (extractTag(Ptr: FaultAddr) != 0)
1015	getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
1016	RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
1017	MemorySize, MinDistance: 0, MaxDistance: 2);
1018
1019	// Check the ring buffer. For primary allocations this will only find UAF;
1020	// for secondary allocations we can find either UAF or OOB.
1021	getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
1022	RingBufferPtr, RingBufferSize);
1023
1024	// Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
1025	// Beyond that we are likely to hit false positives.
1026	if (extractTag(Ptr: FaultAddr) != 0)
1027	getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
1028	RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
1029	MemorySize, MinDistance: 2, MaxDistance: 16);
1030	}
1031
1032	private:
1033	typedef typename PrimaryT::SizeClassMap SizeClassMap;
1034
1035	static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
1036	static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.
1037	static const uptr MinAlignment = 1UL << MinAlignmentLog;
1038	static const uptr MaxAlignment = 1UL << MaxAlignmentLog;
1039	static const uptr MaxAllowedMallocSize =
1040	FIRST_32_SECOND_64(1UL << 31, 1ULL << 40);
1041
1042	static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
1043	"Minimal alignment must at least cover a chunk header.");
1044	static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
1045	MinAlignment >= archMemoryTagGranuleSize(),
1046	"");
1047
1048	static const u32 BlockMarker = 0x44554353U;
1049
1050	// These are indexes into an "array" of 32-bit values that store information
1051	// inline with a chunk that is relevant to diagnosing memory tag faults, where
1052	// 0 corresponds to the address of the user memory. This means that only
1053	// negative indexes may be used. The smallest index that may be used is -2,
1054	// which corresponds to 8 bytes before the user memory, because the chunk
1055	// header size is 8 bytes and in allocators that support memory tagging the
1056	// minimum alignment is at least the tag granule size (16 on aarch64).
1057	static const sptr MemTagAllocationTraceIndex = -2;
1058	static const sptr MemTagAllocationTidIndex = -1;
1059
1060	u32 Cookie = 0;
1061	u32 QuarantineMaxChunkSize = 0;
1062
1063	GlobalStats Stats;
1064	PrimaryT Primary;
1065	SecondaryT Secondary;
1066	QuarantineT Quarantine;
1067	TSDRegistryT TSDRegistry;
1068	pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
1069
1070	#ifdef GWP_ASAN_HOOKS
1071	gwp_asan::GuardedPoolAllocator GuardedAlloc;
1072	uptr GuardedAllocSlotSize = 0;
1073	#endif // GWP_ASAN_HOOKS
1074
1075	struct AllocationRingBuffer {
1076	struct Entry {
1077	atomic_uptr Ptr;
1078	atomic_uptr AllocationSize;
1079	atomic_u32 AllocationTrace;
1080	atomic_u32 AllocationTid;
1081	atomic_u32 DeallocationTrace;
1082	atomic_u32 DeallocationTid;
1083	};
1084	StackDepot *Depot = nullptr;
1085	uptr StackDepotSize = 0;
1086	MemMapT RawRingBufferMap;
1087	MemMapT RawStackDepotMap;
1088	u32 RingBufferElements = 0;
1089	atomic_uptr Pos;
1090	// An array of Size (at least one) elements of type Entry is immediately
1091	// following to this struct.
1092	};
1093	static_assert(sizeof(AllocationRingBuffer) %
1094	alignof(typename AllocationRingBuffer::Entry) ==
1095	0,
1096	"invalid alignment");
1097
1098	// Lock to initialize the RingBuffer
1099	HybridMutex RingBufferInitLock;
1100
1101	// Pointer to memory mapped area starting with AllocationRingBuffer struct,
1102	// and immediately followed by Size elements of type Entry.
1103	atomic_uptr RingBufferAddress = {};
1104
1105	AllocationRingBuffer *getRingBuffer() {
1106	return reinterpret_cast<AllocationRingBuffer *>(
1107	atomic_load(A: &RingBufferAddress, MO: memory_order_acquire));
1108	}
1109
1110	// The following might get optimized out by the compiler.
1111	NOINLINE void performSanityChecks() {
1112	// Verify that the header offset field can hold the maximum offset. In the
1113	// case of the Secondary allocator, it takes care of alignment and the
1114	// offset will always be small. In the case of the Primary, the worst case
1115	// scenario happens in the last size class, when the backend allocation
1116	// would already be aligned on the requested alignment, which would happen
1117	// to be the maximum alignment that would fit in that size class. As a
1118	// result, the maximum offset will be at most the maximum alignment for the
1119	// last size class minus the header size, in multiples of MinAlignment.
1120	Chunk::UnpackedHeader Header = {};
1121	const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(
1122	SizeClassMap::MaxSize - MinAlignment);
1123	const uptr MaxOffset =
1124	(MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
1125	Header.Offset = MaxOffset & Chunk::OffsetMask;
1126	if (UNLIKELY(Header.Offset != MaxOffset))
1127	reportSanityCheckError(Field: "offset");
1128
1129	// Verify that we can fit the maximum size or amount of unused bytes in the
1130	// header. Given that the Secondary fits the allocation to a page, the worst
1131	// case scenario happens in the Primary. It will depend on the second to
1132	// last and last class sizes, as well as the dynamic base for the Primary.
1133	// The following is an over-approximation that works for our needs.
1134	const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
1135	Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
1136	if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))
1137	reportSanityCheckError(Field: "size (or unused bytes)");
1138
1139	const uptr LargestClassId = SizeClassMap::LargestClassId;
1140	Header.ClassId = LargestClassId;
1141	if (UNLIKELY(Header.ClassId != LargestClassId))
1142	reportSanityCheckError(Field: "class ID");
1143	}
1144
1145	static inline void *getBlockBegin(const void *Ptr,
1146	Chunk::UnpackedHeader *Header) {
1147	return reinterpret_cast<void *>(
1148	reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -
1149	(static_cast<uptr>(Header->Offset) << MinAlignmentLog));
1150	}
1151
1152	// Return the size of a chunk as requested during its allocation.
1153	inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
1154	const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
1155	if (LIKELY(Header->ClassId))
1156	return SizeOrUnusedBytes;
1157	if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1158	Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1159	return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
1160	reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
1161	}
1162
1163	void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,
1164	Chunk::UnpackedHeader *Header,
1165	uptr Size) NO_THREAD_SAFETY_ANALYSIS {
1166	void *Ptr = getHeaderTaggedPointer(Ptr: TaggedPtr);
1167	// If the quarantine is disabled, the actual size of a chunk is 0 or larger
1168	// than the maximum allowed, we return a chunk directly to the backend.
1169	// This purposefully underflows for Size == 0.
1170	const bool BypassQuarantine = !Quarantine.getCacheSize() ||
1171	((Size - 1) >= QuarantineMaxChunkSize) ||
1172	!Header->ClassId;
1173	if (BypassQuarantine)
1174	Header->State = Chunk::State::Available;
1175	else
1176	Header->State = Chunk::State::Quarantined;
1177	Header->OriginOrWasZeroed = useMemoryTagging<AllocatorConfig>(Options) &&
1178	Header->ClassId &&
1179	!TSDRegistry.getDisableMemInit();
1180	Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: Header);
1181
1182	if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
1183	u8 PrevTag = extractTag(Ptr: reinterpret_cast<uptr>(TaggedPtr));
1184	storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
1185	if (Header->ClassId) {
1186	if (!TSDRegistry.getDisableMemInit()) {
1187	uptr TaggedBegin, TaggedEnd;
1188	const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
1189	Options, Ptr: reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
1190	ClassId: Header->ClassId);
1191	// Exclude the previous tag so that immediate use after free is
1192	// detected 100% of the time.
1193	setRandomTag(Ptr, Size, ExcludeMask: OddEvenMask | (1UL << PrevTag), TaggedBegin: &TaggedBegin,
1194	TaggedEnd: &TaggedEnd);
1195	}
1196	}
1197	}
1198	if (BypassQuarantine) {
1199	if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1200	Ptr = untagPointer(Ptr);
1201	void *BlockBegin = getBlockBegin(Ptr, Header);
1202	const uptr ClassId = Header->ClassId;
1203	if (LIKELY(ClassId)) {
1204	bool CacheDrained;
1205	{
1206	typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1207	CacheDrained = TSD->getCache().deallocate(ClassId, BlockBegin);
1208	}
1209	// When we have drained some blocks back to the Primary from TSD, that
1210	// implies that we may have the chance to release some pages as well.
1211	// Note that in order not to block other thread's accessing the TSD,
1212	// release the TSD first then try the page release.
1213	if (CacheDrained)
1214	Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
1215	} else {
1216	if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options)))
1217	storeTags(Begin: reinterpret_cast<uptr>(BlockBegin),
1218	End: reinterpret_cast<uptr>(Ptr));
1219	Secondary.deallocate(Options, BlockBegin);
1220	}
1221	} else {
1222	typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1223	Quarantine.put(&TSD->getQuarantineCache(),
1224	QuarantineCallback(*this, TSD->getCache()), Ptr, Size);
1225	}
1226	}
1227
1228	bool getChunkFromBlock(uptr Block, uptr *Chunk,
1229	Chunk::UnpackedHeader *Header) {
1230	*Chunk =
1231	Block + getChunkOffsetFromBlock(Block: reinterpret_cast<const char *>(Block));
1232	return Chunk::isValid(Cookie, Ptr: reinterpret_cast<void *>(*Chunk), NewUnpackedHeader: Header);
1233	}
1234
1235	static uptr getChunkOffsetFromBlock(const char *Block) {
1236	u32 Offset = 0;
1237	if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)
1238	Offset = reinterpret_cast<const u32 *>(Block)[1];
1239	return Offset + Chunk::getHeaderSize();
1240	}
1241
1242	// Set the tag of the granule past the end of the allocation to 0, to catch
1243	// linear overflows even if a previous larger allocation used the same block
1244	// and tag. Only do this if the granule past the end is in our block, because
1245	// this would otherwise lead to a SEGV if the allocation covers the entire
1246	// block and our block is at the end of a mapping. The tag of the next block's
1247	// header granule will be set to 0, so it will serve the purpose of catching
1248	// linear overflows in this case.
1249	//
1250	// For allocations of size 0 we do not end up storing the address tag to the
1251	// memory tag space, which getInlineErrorInfo() normally relies on to match
1252	// address tags against chunks. To allow matching in this case we store the
1253	// address tag in the first byte of the chunk.
1254	void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {
1255	DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));
1256	uptr UntaggedEnd = untagPointer(Ptr: End);
1257	if (UntaggedEnd != BlockEnd) {
1258	storeTag(Ptr: UntaggedEnd);
1259	if (Size == 0)
1260	*reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(Ptr: End);
1261	}
1262	}
1263
1264	void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
1265	uptr BlockEnd) {
1266	// Prepare the granule before the chunk to store the chunk header by setting
1267	// its tag to 0. Normally its tag will already be 0, but in the case where a
1268	// chunk holding a low alignment allocation is reused for a higher alignment
1269	// allocation, the chunk may already have a non-zero tag from the previous
1270	// allocation.
1271	storeTag(Ptr: reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());
1272
1273	uptr TaggedBegin, TaggedEnd;
1274	setRandomTag(Ptr, Size, ExcludeMask, TaggedBegin: &TaggedBegin, TaggedEnd: &TaggedEnd);
1275
1276	storeEndMarker(End: TaggedEnd, Size, BlockEnd);
1277	return reinterpret_cast<void *>(TaggedBegin);
1278	}
1279
1280	void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,
1281	uptr BlockEnd) {
1282	uptr RoundOldPtr = roundUp(X: OldPtr, Boundary: archMemoryTagGranuleSize());
1283	uptr RoundNewPtr;
1284	if (RoundOldPtr >= NewPtr) {
1285	// If the allocation is shrinking we just need to set the tag past the end
1286	// of the allocation to 0. See explanation in storeEndMarker() above.
1287	RoundNewPtr = roundUp(X: NewPtr, Boundary: archMemoryTagGranuleSize());
1288	} else {
1289	// Set the memory tag of the region
1290	// [RoundOldPtr, roundUp(NewPtr, archMemoryTagGranuleSize()))
1291	// to the pointer tag stored in OldPtr.
1292	RoundNewPtr = storeTags(Begin: RoundOldPtr, End: NewPtr);
1293	}
1294	storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1295	}
1296
1297	void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {
1298	if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1299	return;
1300	AllocationRingBuffer *RB = getRingBuffer();
1301	if (!RB)
1302	return;
1303	auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1304	Ptr32[MemTagAllocationTraceIndex] = collectStackTrace(Depot: RB->Depot);
1305	Ptr32[MemTagAllocationTidIndex] = getThreadID();
1306	}
1307
1308	void storeRingBufferEntry(AllocationRingBuffer *RB, void *Ptr,
1309	u32 AllocationTrace, u32 AllocationTid,
1310	uptr AllocationSize, u32 DeallocationTrace,
1311	u32 DeallocationTid) {
1312	uptr Pos = atomic_fetch_add(&RB->Pos, 1, memory_order_relaxed);
1313	typename AllocationRingBuffer::Entry *Entry =
1314	getRingBufferEntry(RB, Pos % RB->RingBufferElements);
1315
1316	// First invalidate our entry so that we don't attempt to interpret a
1317	// partially written state in getSecondaryErrorInfo(). The fences below
1318	// ensure that the compiler does not move the stores to Ptr in between the
1319	// stores to the other fields.
1320	atomic_store_relaxed(&Entry->Ptr, 0);
1321
1322	__atomic_signal_fence(__ATOMIC_SEQ_CST);
1323	atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);
1324	atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);
1325	atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);
1326	atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);
1327	atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);
1328	__atomic_signal_fence(__ATOMIC_SEQ_CST);
1329
1330	atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));
1331	}
1332
1333	void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
1334	uptr Size) {
1335	if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1336	return;
1337	AllocationRingBuffer *RB = getRingBuffer();
1338	if (!RB)
1339	return;
1340	u32 Trace = collectStackTrace(Depot: RB->Depot);
1341	u32 Tid = getThreadID();
1342
1343	auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1344	Ptr32[MemTagAllocationTraceIndex] = Trace;
1345	Ptr32[MemTagAllocationTidIndex] = Tid;
1346
1347	storeRingBufferEntry(RB, Ptr: untagPointer(Ptr), AllocationTrace: Trace, AllocationTid: Tid, AllocationSize: Size, DeallocationTrace: 0, DeallocationTid: 0);
1348	}
1349
1350	void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
1351	u8 PrevTag, uptr Size) {
1352	if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1353	return;
1354	AllocationRingBuffer *RB = getRingBuffer();
1355	if (!RB)
1356	return;
1357	auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1358	u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];
1359	u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];
1360
1361	u32 DeallocationTrace = collectStackTrace(Depot: RB->Depot);
1362	u32 DeallocationTid = getThreadID();
1363
1364	storeRingBufferEntry(RB, Ptr: addFixedTag(Ptr: untagPointer(Ptr), Tag: PrevTag),
1365	AllocationTrace, AllocationTid, AllocationSize: Size,
1366	DeallocationTrace, DeallocationTid);
1367	}
1368
1369	static const size_t NumErrorReports =
1370	sizeof(((scudo_error_info *)nullptr)->reports) /
1371	sizeof(((scudo_error_info *)nullptr)->reports[0]);
1372
1373	static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,
1374	size_t &NextErrorReport, uintptr_t FaultAddr,
1375	const StackDepot *Depot,
1376	const char *RegionInfoPtr, const char *Memory,
1377	const char *MemoryTags, uintptr_t MemoryAddr,
1378	size_t MemorySize, size_t MinDistance,
1379	size_t MaxDistance) {
1380	uptr UntaggedFaultAddr = untagPointer(Ptr: FaultAddr);
1381	u8 FaultAddrTag = extractTag(Ptr: FaultAddr);
1382	BlockInfo Info =
1383	PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);
1384
1385	auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {
1386	if (Addr < MemoryAddr || Addr + archMemoryTagGranuleSize() < Addr ||
1387	Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)
1388	return false;
1389	*Data = &Memory[Addr - MemoryAddr];
1390	*Tag = static_cast<u8>(
1391	MemoryTags[(Addr - MemoryAddr) / archMemoryTagGranuleSize()]);
1392	return true;
1393	};
1394
1395	auto ReadBlock = [&](uptr Addr, uptr *ChunkAddr,
1396	Chunk::UnpackedHeader *Header, const u32 **Data,
1397	u8 *Tag) {
1398	const char *BlockBegin;
1399	u8 BlockBeginTag;
1400	if (!GetGranule(Addr, &BlockBegin, &BlockBeginTag))
1401	return false;
1402	uptr ChunkOffset = getChunkOffsetFromBlock(Block: BlockBegin);
1403	*ChunkAddr = Addr + ChunkOffset;
1404
1405	const char *ChunkBegin;
1406	if (!GetGranule(*ChunkAddr, &ChunkBegin, Tag))
1407	return false;
1408	*Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(
1409	ChunkBegin - Chunk::getHeaderSize());
1410	*Data = reinterpret_cast<const u32 *>(ChunkBegin);
1411
1412	// Allocations of size 0 will have stashed the tag in the first byte of
1413	// the chunk, see storeEndMarker().
1414	if (Header->SizeOrUnusedBytes == 0)
1415	*Tag = static_cast<u8>(*ChunkBegin);
1416
1417	return true;
1418	};
1419
1420	if (NextErrorReport == NumErrorReports)
1421	return;
1422
1423	auto CheckOOB = [&](uptr BlockAddr) {
1424	if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
1425	return false;
1426
1427	uptr ChunkAddr;
1428	Chunk::UnpackedHeader Header;
1429	const u32 *Data;
1430	uint8_t Tag;
1431	if (!ReadBlock(BlockAddr, &ChunkAddr, &Header, &Data, &Tag) ||
1432	Header.State != Chunk::State::Allocated || Tag != FaultAddrTag)
1433	return false;
1434
1435	auto *R = &ErrorInfo->reports[NextErrorReport++];
1436	R->error_type =
1437	UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;
1438	R->allocation_address = ChunkAddr;
1439	R->allocation_size = Header.SizeOrUnusedBytes;
1440	if (Depot) {
1441	collectTraceMaybe(Depot, Trace&: R->allocation_trace,
1442	Hash: Data[MemTagAllocationTraceIndex]);
1443	}
1444	R->allocation_tid = Data[MemTagAllocationTidIndex];
1445	return NextErrorReport == NumErrorReports;
1446	};
1447
1448	if (MinDistance == 0 && CheckOOB(Info.BlockBegin))
1449	return;
1450
1451	for (size_t I = Max<size_t>(A: MinDistance, B: 1); I != MaxDistance; ++I)
1452	if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
1453	CheckOOB(Info.BlockBegin - I * Info.BlockSize))
1454	return;
1455	}
1456
1457	static void getRingBufferErrorInfo(struct scudo_error_info *ErrorInfo,
1458	size_t &NextErrorReport,
1459	uintptr_t FaultAddr,
1460	const StackDepot *Depot,
1461	const char *RingBufferPtr,
1462	size_t RingBufferSize) {
1463	auto *RingBuffer =
1464	reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);
1465	size_t RingBufferElements = ringBufferElementsFromBytes(Bytes: RingBufferSize);
1466	if (!RingBuffer || RingBufferElements == 0 || !Depot)
1467	return;
1468	uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
1469
1470	for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements &&
1471	NextErrorReport != NumErrorReports;
1472	--I) {
1473	auto *Entry = getRingBufferEntry(RingBuffer, I % RingBufferElements);
1474	uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
1475	if (!EntryPtr)
1476	continue;
1477
1478	uptr UntaggedEntryPtr = untagPointer(Ptr: EntryPtr);
1479	uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);
1480	u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);
1481	u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);
1482	u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);
1483	u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);
1484
1485	if (DeallocationTid) {
1486	// For UAF we only consider in-bounds fault addresses because
1487	// out-of-bounds UAF is rare and attempting to detect it is very likely
1488	// to result in false positives.
1489	if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)
1490	continue;
1491	} else {
1492	// Ring buffer OOB is only possible with secondary allocations. In this
1493	// case we are guaranteed a guard region of at least a page on either
1494	// side of the allocation (guard page on the right, guard page + tagged
1495	// region on the left), so ignore any faults outside of that range.
1496	if (FaultAddr < EntryPtr - getPageSizeCached() ||
1497	FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())
1498	continue;
1499
1500	// For UAF the ring buffer will contain two entries, one for the
1501	// allocation and another for the deallocation. Don't report buffer
1502	// overflow/underflow using the allocation entry if we have already
1503	// collected a report from the deallocation entry.
1504	bool Found = false;
1505	for (uptr J = 0; J != NextErrorReport; ++J) {
1506	if (ErrorInfo->reports[J].allocation_address == UntaggedEntryPtr) {
1507	Found = true;
1508	break;
1509	}
1510	}
1511	if (Found)
1512	continue;
1513	}
1514
1515	auto *R = &ErrorInfo->reports[NextErrorReport++];
1516	if (DeallocationTid)
1517	R->error_type = USE_AFTER_FREE;
1518	else if (FaultAddr < EntryPtr)
1519	R->error_type = BUFFER_UNDERFLOW;
1520	else
1521	R->error_type = BUFFER_OVERFLOW;
1522
1523	R->allocation_address = UntaggedEntryPtr;
1524	R->allocation_size = EntrySize;
1525	collectTraceMaybe(Depot, Trace&: R->allocation_trace, Hash: AllocationTrace);
1526	R->allocation_tid = AllocationTid;
1527	collectTraceMaybe(Depot, Trace&: R->deallocation_trace, Hash: DeallocationTrace);
1528	R->deallocation_tid = DeallocationTid;
1529	}
1530	}
1531
1532	uptr getStats(ScopedString *Str) {
1533	Primary.getStats(Str);
1534	Secondary.getStats(Str);
1535	Quarantine.getStats(Str);
1536	TSDRegistry.getStats(Str);
1537	return Str->length();
1538	}
1539
1540	static typename AllocationRingBuffer::Entry *
1541	getRingBufferEntry(AllocationRingBuffer *RB, uptr N) {
1542	char *RBEntryStart =
1543	&reinterpret_cast<char *>(RB)[sizeof(AllocationRingBuffer)];
1544	return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(
1545	RBEntryStart)[N];
1546	}
1547	static const typename AllocationRingBuffer::Entry *
1548	getRingBufferEntry(const AllocationRingBuffer *RB, uptr N) {
1549	const char *RBEntryStart =
1550	&reinterpret_cast<const char *>(RB)[sizeof(AllocationRingBuffer)];
1551	return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(
1552	RBEntryStart)[N];
1553	}
1554
1555	void initRingBufferMaybe() {
1556	ScopedLock L(RingBufferInitLock);
1557	if (getRingBuffer() != nullptr)
1558	return;
1559
1560	int ring_buffer_size = getFlags()->allocation_ring_buffer_size;
1561	if (ring_buffer_size <= 0)
1562	return;
1563
1564	u32 AllocationRingBufferSize = static_cast<u32>(ring_buffer_size);
1565
1566	// We store alloc and free stacks for each entry.
1567	constexpr u32 kStacksPerRingBufferEntry = 2;
1568	constexpr u32 kMaxU32Pow2 = ~(UINT32_MAX >> 1);
1569	static_assert(isPowerOfTwo(X: kMaxU32Pow2));
1570	// On Android we always have 3 frames at the bottom: __start_main,
1571	// __libc_init, main, and 3 at the top: malloc, scudo_malloc and
1572	// Allocator::allocate. This leaves 10 frames for the user app. The next
1573	// smallest power of two (8) would only leave 2, which is clearly too
1574	// little.
1575	constexpr u32 kFramesPerStack = 16;
1576	static_assert(isPowerOfTwo(X: kFramesPerStack));
1577
1578	if (AllocationRingBufferSize > kMaxU32Pow2 / kStacksPerRingBufferEntry)
1579	return;
1580	u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(Size: kStacksPerRingBufferEntry *
1581	AllocationRingBufferSize));
1582	if (TabSize > UINT32_MAX / kFramesPerStack)
1583	return;
1584	u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);
1585
1586	uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +
1587	sizeof(atomic_u32) * TabSize;
1588	MemMapT DepotMap;
1589	DepotMap.map(
1590	/*Addr=*/Addr: 0U, Size: roundUp(X: StackDepotSize, Boundary: getPageSizeCached()),
1591	Name: "scudo:stack_depot");
1592	auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());
1593	Depot->init(RingSz: RingSize, TabSz: TabSize);
1594
1595	MemMapT MemMap;
1596	MemMap.map(
1597	/*Addr=*/Addr: 0U,
1598	Size: roundUp(X: ringBufferSizeInBytes(RingBufferElements: AllocationRingBufferSize),
1599	Boundary: getPageSizeCached()),
1600	Name: "scudo:ring_buffer");
1601	auto *RB = reinterpret_cast<AllocationRingBuffer *>(MemMap.getBase());
1602	RB->RawRingBufferMap = MemMap;
1603	RB->RingBufferElements = AllocationRingBufferSize;
1604	RB->Depot = Depot;
1605	RB->StackDepotSize = StackDepotSize;
1606	RB->RawStackDepotMap = DepotMap;
1607
1608	atomic_store(A: &RingBufferAddress, V: reinterpret_cast<uptr>(RB),
1609	MO: memory_order_release);
1610	}
1611
1612	void unmapRingBuffer() {
1613	AllocationRingBuffer *RB = getRingBuffer();
1614	if (RB == nullptr)
1615	return;
1616	// N.B. because RawStackDepotMap is part of RawRingBufferMap, the order
1617	// is very important.
1618	RB->RawStackDepotMap.unmap(RB->RawStackDepotMap.getBase(),
1619	RB->RawStackDepotMap.getCapacity());
1620	// Note that the `RB->RawRingBufferMap` is stored on the pages managed by
1621	// itself. Take over the ownership before calling unmap() so that any
1622	// operation along with unmap() won't touch inaccessible pages.
1623	MemMapT RawRingBufferMap = RB->RawRingBufferMap;
1624	RawRingBufferMap.unmap(Addr: RawRingBufferMap.getBase(),
1625	Size: RawRingBufferMap.getCapacity());
1626	atomic_store(A: &RingBufferAddress, V: 0, MO: memory_order_release);
1627	}
1628
1629	static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) {
1630	return sizeof(AllocationRingBuffer) +
1631	RingBufferElements * sizeof(typename AllocationRingBuffer::Entry);
1632	}
1633
1634	static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) {
1635	if (Bytes < sizeof(AllocationRingBuffer)) {
1636	return 0;
1637	}
1638	return (Bytes - sizeof(AllocationRingBuffer)) /
1639	sizeof(typename AllocationRingBuffer::Entry);
1640	}
1641	};
1642
1643	} // namespace scudo
1644
1645	#endif // SCUDO_COMBINED_H_
1646