1//===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Platform-specific code.
12//===----------------------------------------------------------------------===//
13
14#ifndef TSAN_PLATFORM_H
15#define TSAN_PLATFORM_H
16
17#if !defined(__LP64__) && !defined(_WIN64)
18# error "Only 64-bit is supported"
19#endif
20
21#include "sanitizer_common/sanitizer_common.h"
22#include "tsan_defs.h"
23
24namespace __tsan {
25
26enum {
27 // App memory is not mapped onto shadow memory range.
28 kBrokenMapping = 1 << 0,
29 // Mapping app memory and back does not produce the same address,
30 // this can lead to wrong addresses in reports and potentially
31 // other bad consequences.
32 kBrokenReverseMapping = 1 << 1,
33 // Mapping is non-linear for linear user range.
34 // This is bad and can lead to unpredictable memory corruptions, etc
35 // because range access functions assume linearity.
36 kBrokenLinearity = 1 << 2,
37 // Meta for an app region overlaps with the meta of another app region.
38 // This is determined by recomputing the individual meta regions for
39 // each app region.
40 //
41 // N.B. There is no "kBrokenReverseMetaMapping" constant because there
42 // is no MetaToMem function. However, note that (!kBrokenLinearity
43 // && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
44 kBrokenAliasedMetas = 1 << 3,
45};
46
47/*
48C/C++ on linux/x86_64 and freebsd/x86_64
490000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB)
500200 0000 0000 - 1000 0000 0000: -
511000 0000 0000 - 3000 0000 0000: shadow (32TB)
523000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB)
533800 0000 0000 - 5500 0000 0000: -
545500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels
555a00 0000 0000 - 7200 0000 0000: -
567200 0000 0000 - 7300 0000 0000: heap (1TB)
577300 0000 0000 - 7a00 0000 0000: -
587a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB)
59
60C/C++ on netbsd/amd64 can reuse the same mapping:
61 * The address space starts from 0x1000 (option with 0x0) and ends with
62 0x7f7ffffff000.
63 * LoAppMem-kHeapMemEnd can be reused as it is.
64 * No VDSO support.
65 * No MidAppMem region.
66 * No additional HeapMem region.
67 * HiAppMem contains the stack, loader, shared libraries and heap.
68 * Stack on NetBSD/amd64 has prereserved 128MB.
69 * Heap grows downwards (top-down).
70 * ASLR must be disabled per-process or globally.
71*/
72struct Mapping48AddressSpace {
73 static const uptr kMetaShadowBeg = 0x300000000000ull;
74 static const uptr kMetaShadowEnd = 0x380000000000ull;
75 static const uptr kShadowBeg = 0x100000000000ull;
76 static const uptr kShadowEnd = 0x300000000000ull;
77 static const uptr kHeapMemBeg = 0x720000000000ull;
78 static const uptr kHeapMemEnd = 0x730000000000ull;
79 static const uptr kLoAppMemBeg = 0x000000001000ull;
80 static const uptr kLoAppMemEnd = 0x020000000000ull;
81 static const uptr kMidAppMemBeg = 0x550000000000ull;
82 static const uptr kMidAppMemEnd = 0x5a0000000000ull;
83 static const uptr kHiAppMemBeg = 0x7a0000000000ull;
84 static const uptr kHiAppMemEnd = 0x800000000000ull;
85 static const uptr kShadowMsk = 0x700000000000ull;
86 static const uptr kShadowXor = 0x000000000000ull;
87 static const uptr kShadowAdd = 0x100000000000ull;
88 static const uptr kVdsoBeg = 0xf000000000000000ull;
89};
90
91/*
92C/C++ on linux/mips64 (40-bit VMA)
930000 0000 00 - 0100 0000 00: - (4 GB)
940100 0000 00 - 0200 0000 00: main binary (4 GB)
950200 0000 00 - 1200 0000 00: - (64 GB)
961200 0000 00 - 2200 0000 00: shadow (64 GB)
972200 0000 00 - 4000 0000 00: - (120 GB)
984000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
995000 0000 00 - aa00 0000 00: - (360 GB)
100aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
101ab00 0000 00 - fe00 0000 00: - (332 GB)
102fe00 0000 00 - ff00 0000 00: heap (4 GB)
103ff00 0000 00 - ff80 0000 00: - (2 GB)
104ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
105*/
106struct MappingMips64_40 {
107 static const uptr kMetaShadowBeg = 0x4000000000ull;
108 static const uptr kMetaShadowEnd = 0x5000000000ull;
109 static const uptr kShadowBeg = 0x1200000000ull;
110 static const uptr kShadowEnd = 0x2200000000ull;
111 static const uptr kHeapMemBeg = 0xfe00000000ull;
112 static const uptr kHeapMemEnd = 0xff00000000ull;
113 static const uptr kLoAppMemBeg = 0x0100000000ull;
114 static const uptr kLoAppMemEnd = 0x0200000000ull;
115 static const uptr kMidAppMemBeg = 0xaa00000000ull;
116 static const uptr kMidAppMemEnd = 0xab00000000ull;
117 static const uptr kHiAppMemBeg = 0xff80000000ull;
118 static const uptr kHiAppMemEnd = 0xffffffffffull;
119 static const uptr kShadowMsk = 0xf800000000ull;
120 static const uptr kShadowXor = 0x0800000000ull;
121 static const uptr kShadowAdd = 0x0000000000ull;
122 static const uptr kVdsoBeg = 0xfffff00000ull;
123};
124
125/*
126C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
1270000 0000 00 - 0100 0000 00: - (4 GB)
1280100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
1290200 0000 00 - 0300 0000 00: heap (4 GB)
1300300 0000 00 - 0400 0000 00: - (4 GB)
1310400 0000 00 - 0800 0000 00: shadow memory (16 GB)
1320800 0000 00 - 0d00 0000 00: - (20 GB)
1330d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
1340e00 0000 00 - 1000 0000 00: -
135*/
136struct MappingAppleAarch64 {
137 static const uptr kLoAppMemBeg = 0x0100000000ull;
138 static const uptr kLoAppMemEnd = 0x0200000000ull;
139 static const uptr kHeapMemBeg = 0x0200000000ull;
140 static const uptr kHeapMemEnd = 0x0300000000ull;
141 static const uptr kShadowBeg = 0x0400000000ull;
142 static const uptr kShadowEnd = 0x0800000000ull;
143 static const uptr kMetaShadowBeg = 0x0d00000000ull;
144 static const uptr kMetaShadowEnd = 0x0e00000000ull;
145 static const uptr kHiAppMemBeg = 0x0fc0000000ull;
146 static const uptr kHiAppMemEnd = 0x0fc0000000ull;
147 static const uptr kShadowMsk = 0x0ull;
148 static const uptr kShadowXor = 0x0ull;
149 static const uptr kShadowAdd = 0x0200000000ull;
150 static const uptr kVdsoBeg = 0x7000000000000000ull;
151 static const uptr kMidAppMemBeg = 0;
152 static const uptr kMidAppMemEnd = 0;
153};
154
155/*
156C/C++ on linux/aarch64 (39-bit VMA)
1570000 0010 00 - 0500 0000 00: main binary (20 GB)
1580100 0000 00 - 2000 0000 00: -
1592000 0000 00 - 4000 0000 00: shadow memory (128 GB)
1604000 0000 00 - 4800 0000 00: metainfo (32 GB)
1614800 0000 00 - 5500 0000 00: -
1625500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
1635600 0000 00 - 7c00 0000 00: -
1647a00 0000 00 - 7d00 0000 00: heap (12 GB)
1657d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
166*/
167struct MappingAarch64_39 {
168 static const uptr kLoAppMemBeg = 0x0000001000ull;
169 static const uptr kLoAppMemEnd = 0x0500000000ull;
170 static const uptr kShadowBeg = 0x2000000000ull;
171 static const uptr kShadowEnd = 0x4000000000ull;
172 static const uptr kMetaShadowBeg = 0x4000000000ull;
173 static const uptr kMetaShadowEnd = 0x4800000000ull;
174 static const uptr kMidAppMemBeg = 0x5500000000ull;
175 static const uptr kMidAppMemEnd = 0x5a00000000ull;
176 static const uptr kHeapMemBeg = 0x7a00000000ull;
177 static const uptr kHeapMemEnd = 0x7d00000000ull;
178 static const uptr kHiAppMemBeg = 0x7d00000000ull;
179 static const uptr kHiAppMemEnd = 0x7fffffffffull;
180 static const uptr kShadowMsk = 0x7000000000ull;
181 static const uptr kShadowXor = 0x1000000000ull;
182 static const uptr kShadowAdd = 0x0000000000ull;
183 static const uptr kVdsoBeg = 0x7f00000000ull;
184};
185
186/*
187C/C++ on linux/aarch64 (42-bit VMA)
18800000 0010 00 - 02000 0000 00: main binary (128 GB)
18902000 0000 00 - 08000 0000 00: -
19010000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
19120000 0000 00 - 24000 0000 00: metainfo (256 GB)
19224000 0000 00 - 2aa00 0000 00: -
1932aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
1942c000 0000 00 - 3c000 0000 00: -
1953c000 0000 00 - 3f000 0000 00: heap (192 GB)
1963f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
197*/
198struct MappingAarch64_42 {
199 static const uptr kLoAppMemBeg = 0x00000001000ull;
200 static const uptr kLoAppMemEnd = 0x02000000000ull;
201 static const uptr kShadowBeg = 0x10000000000ull;
202 static const uptr kShadowEnd = 0x20000000000ull;
203 static const uptr kMetaShadowBeg = 0x20000000000ull;
204 static const uptr kMetaShadowEnd = 0x24000000000ull;
205 static const uptr kMidAppMemBeg = 0x2aa00000000ull;
206 static const uptr kMidAppMemEnd = 0x2c000000000ull;
207 static const uptr kHeapMemBeg = 0x3c000000000ull;
208 static const uptr kHeapMemEnd = 0x3f000000000ull;
209 static const uptr kHiAppMemBeg = 0x3f000000000ull;
210 static const uptr kHiAppMemEnd = 0x3ffffffffffull;
211 static const uptr kShadowMsk = 0x38000000000ull;
212 static const uptr kShadowXor = 0x08000000000ull;
213 static const uptr kShadowAdd = 0x00000000000ull;
214 static const uptr kVdsoBeg = 0x37f00000000ull;
215};
216
217/*
218C/C++ on linux/aarch64 (48-bit VMA)
2190000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
2200a00 0000 1000 - 1554 0000 0000: -
2211554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
2225400 0000 1000 - 8000 0000 0000: -
2238000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
224a000 0000 1000 - aaaa 0000 0000: -
225aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
226ac00 0000 1000 - fc00 0000 0000: -
227fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
228
229N.B. the shadow memory region has a strange start address, because it
230contains the shadows for the mid, high and low app regions (in this
231unusual order).
232*/
233struct MappingAarch64_48 {
234 static const uptr kLoAppMemBeg = 0x0000000001000ull;
235 static const uptr kLoAppMemEnd = 0x00a0000000000ull;
236 static const uptr kShadowBeg = 0x0155400000000ull;
237 static const uptr kShadowEnd = 0x0540000000000ull;
238 static const uptr kMetaShadowBeg = 0x0800000000000ull;
239 static const uptr kMetaShadowEnd = 0x0a00000000000ull;
240 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
241 static const uptr kMidAppMemEnd = 0x0ac0000000000ull;
242 static const uptr kHiAppMemBeg = 0x0fc0000000000ull;
243 static const uptr kHiAppMemEnd = 0x1000000000000ull;
244 static const uptr kHeapMemBeg = 0x0fc0000000000ull;
245 static const uptr kHeapMemEnd = 0x0fc0000000000ull;
246 static const uptr kShadowMsk = 0x0c00000000000ull;
247 static const uptr kShadowXor = 0x0200000000000ull;
248 static const uptr kShadowAdd = 0x0000000000000ull;
249 static const uptr kVdsoBeg = 0xffff000000000ull;
250};
251
252/* C/C++ on linux/loongarch64 (47-bit VMA)
2530000 0000 4000 - 0080 0000 0000: main binary
2540080 0000 0000 - 0100 0000 0000: -
2550100 0000 0000 - 1000 0000 0000: shadow memory
2561000 0000 0000 - 3000 0000 0000: -
2573000 0000 0000 - 3400 0000 0000: metainfo
2583400 0000 0000 - 5555 0000 0000: -
2595555 0000 0000 - 5556 0000 0000: main binary (PIE)
2605556 0000 0000 - 7ffe 0000 0000: -
2617ffe 0000 0000 - 7fff 0000 0000: heap
2627fff 0000 0000 - 7fff 8000 0000: -
2637fff 8000 0000 - 8000 0000 0000: modules and main thread stack
264*/
265struct MappingLoongArch64_47 {
266 static const uptr kMetaShadowBeg = 0x300000000000ull;
267 static const uptr kMetaShadowEnd = 0x340000000000ull;
268 static const uptr kShadowBeg = 0x010000000000ull;
269 static const uptr kShadowEnd = 0x100000000000ull;
270 static const uptr kHeapMemBeg = 0x7ffe00000000ull;
271 static const uptr kHeapMemEnd = 0x7fff00000000ull;
272 static const uptr kLoAppMemBeg = 0x000000004000ull;
273 static const uptr kLoAppMemEnd = 0x008000000000ull;
274 static const uptr kMidAppMemBeg = 0x555500000000ull;
275 static const uptr kMidAppMemEnd = 0x555600000000ull;
276 static const uptr kHiAppMemBeg = 0x7fff80000000ull;
277 static const uptr kHiAppMemEnd = 0x800000000000ull;
278 static const uptr kShadowMsk = 0x780000000000ull;
279 static const uptr kShadowXor = 0x040000000000ull;
280 static const uptr kShadowAdd = 0x000000000000ull;
281 static const uptr kVdsoBeg = 0x7fffffffc000ull;
282};
283
284/*
285C/C++ on linux/powerpc64 (44-bit VMA)
2860000 0000 0100 - 0001 0000 0000: main binary
2870001 0000 0000 - 0001 0000 0000: -
2880001 0000 0000 - 0b00 0000 0000: shadow
2890b00 0000 0000 - 0b00 0000 0000: -
2900b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
2910d00 0000 0000 - 0f00 0000 0000: -
2920f00 0000 0000 - 0f50 0000 0000: heap
2930f50 0000 0000 - 0f60 0000 0000: -
2940f60 0000 0000 - 1000 0000 0000: modules and main thread stack
295*/
296struct MappingPPC64_44 {
297 static const uptr kBroken = kBrokenMapping | kBrokenReverseMapping |
298 kBrokenLinearity | kBrokenAliasedMetas;
299 static const uptr kMetaShadowBeg = 0x0b0000000000ull;
300 static const uptr kMetaShadowEnd = 0x0d0000000000ull;
301 static const uptr kShadowBeg = 0x000100000000ull;
302 static const uptr kShadowEnd = 0x0b0000000000ull;
303 static const uptr kLoAppMemBeg = 0x000000000100ull;
304 static const uptr kLoAppMemEnd = 0x000100000000ull;
305 static const uptr kHeapMemBeg = 0x0f0000000000ull;
306 static const uptr kHeapMemEnd = 0x0f5000000000ull;
307 static const uptr kHiAppMemBeg = 0x0f6000000000ull;
308 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
309 static const uptr kShadowMsk = 0x0f0000000000ull;
310 static const uptr kShadowXor = 0x002100000000ull;
311 static const uptr kShadowAdd = 0x000000000000ull;
312 static const uptr kVdsoBeg = 0x3c0000000000000ull;
313 static const uptr kMidAppMemBeg = 0;
314 static const uptr kMidAppMemEnd = 0;
315};
316
317/*
318C/C++ on linux/powerpc64 (46-bit VMA)
3190000 0000 1000 - 0100 0000 0000: main binary
3200100 0000 0000 - 0200 0000 0000: -
3210100 0000 0000 - 0800 0000 0000: shadow
3220800 0000 0000 - 1000 0000 0000: -
3231000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
3241200 0000 0000 - 3d00 0000 0000: -
3253d00 0000 0000 - 3e00 0000 0000: heap
3263e00 0000 0000 - 3e80 0000 0000: -
3273e80 0000 0000 - 4000 0000 0000: modules and main thread stack
328*/
329struct MappingPPC64_46 {
330 static const uptr kMetaShadowBeg = 0x100000000000ull;
331 static const uptr kMetaShadowEnd = 0x120000000000ull;
332 static const uptr kShadowBeg = 0x010000000000ull;
333 static const uptr kShadowEnd = 0x080000000000ull;
334 static const uptr kHeapMemBeg = 0x3d0000000000ull;
335 static const uptr kHeapMemEnd = 0x3e0000000000ull;
336 static const uptr kLoAppMemBeg = 0x000000001000ull;
337 static const uptr kLoAppMemEnd = 0x010000000000ull;
338 static const uptr kHiAppMemBeg = 0x3e8000000000ull;
339 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
340 static const uptr kShadowMsk = 0x3c0000000000ull;
341 static const uptr kShadowXor = 0x020000000000ull;
342 static const uptr kShadowAdd = 0x000000000000ull;
343 static const uptr kVdsoBeg = 0x7800000000000000ull;
344 static const uptr kMidAppMemBeg = 0;
345 static const uptr kMidAppMemEnd = 0;
346};
347
348/*
349C/C++ on linux/powerpc64 (47-bit VMA)
3500000 0000 1000 - 0100 0000 0000: main binary
3510100 0000 0000 - 0200 0000 0000: -
3520100 0000 0000 - 0800 0000 0000: shadow
3530800 0000 0000 - 1000 0000 0000: -
3541000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
3551200 0000 0000 - 7d00 0000 0000: -
3567d00 0000 0000 - 7e00 0000 0000: heap
3577e00 0000 0000 - 7e80 0000 0000: -
3587e80 0000 0000 - 8000 0000 0000: modules and main thread stack
359*/
360struct MappingPPC64_47 {
361 static const uptr kMetaShadowBeg = 0x100000000000ull;
362 static const uptr kMetaShadowEnd = 0x120000000000ull;
363 static const uptr kShadowBeg = 0x010000000000ull;
364 static const uptr kShadowEnd = 0x080000000000ull;
365 static const uptr kHeapMemBeg = 0x7d0000000000ull;
366 static const uptr kHeapMemEnd = 0x7e0000000000ull;
367 static const uptr kLoAppMemBeg = 0x000000001000ull;
368 static const uptr kLoAppMemEnd = 0x010000000000ull;
369 static const uptr kHiAppMemBeg = 0x7e8000000000ull;
370 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
371 static const uptr kShadowMsk = 0x7c0000000000ull;
372 static const uptr kShadowXor = 0x020000000000ull;
373 static const uptr kShadowAdd = 0x000000000000ull;
374 static const uptr kVdsoBeg = 0x7800000000000000ull;
375 static const uptr kMidAppMemBeg = 0;
376 static const uptr kMidAppMemEnd = 0;
377};
378
379/*
380C/C++ on linux/riscv64 (39-bit VMA)
3810000 0010 00 - 0200 0000 00: main binary ( 8 GB)
3820200 0000 00 - 1000 0000 00: -
3831000 0000 00 - 4000 0000 00: shadow memory (64 GB)
3844000 0000 00 - 4800 0000 00: metainfo (16 GB)
3854800 0000 00 - 5500 0000 00: -
3865500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB)
3875600 0000 00 - 7c00 0000 00: -
3887d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB)
389
390mmap by default allocates from top downwards
391VDSO sits below loader and above dynamic libraries, within HiApp region.
392Heap starts after program region whose position depends on pie or non-pie.
393Disable tracking them since their locations are not fixed.
394*/
395struct MappingRiscv64_39 {
396 static const uptr kLoAppMemBeg = 0x0000001000ull;
397 static const uptr kLoAppMemEnd = 0x0200000000ull;
398 static const uptr kShadowBeg = 0x1000000000ull;
399 static const uptr kShadowEnd = 0x2000000000ull;
400 static const uptr kMetaShadowBeg = 0x2000000000ull;
401 static const uptr kMetaShadowEnd = 0x2400000000ull;
402 static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
403 static const uptr kMidAppMemEnd = 0x2c00000000ull;
404 static const uptr kHeapMemBeg = 0x2c00000000ull;
405 static const uptr kHeapMemEnd = 0x2c00000000ull;
406 static const uptr kHiAppMemBeg = 0x3c00000000ull;
407 static const uptr kHiAppMemEnd = 0x3fffffffffull;
408 static const uptr kShadowMsk = 0x3800000000ull;
409 static const uptr kShadowXor = 0x0800000000ull;
410 static const uptr kShadowAdd = 0x0000000000ull;
411 static const uptr kVdsoBeg = 0x4000000000ull;
412};
413
414/*
415C/C++ on linux/riscv64 (48-bit VMA)
4160000 0000 1000 - 0400 0000 0000: main binary ( 4 TB)
4170500 0000 0000 - 2000 0000 0000: -
4182000 0000 0000 - 4000 0000 0000: shadow memory (32 TB)
4194000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB)
4204800 0000 0000 - 5555 5555 5000: -
4215555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB)
4225a00 0000 0000 - 7a00 0000 0000: -
4237a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 6 TB)
424*/
425struct MappingRiscv64_48 {
426 static const uptr kLoAppMemBeg = 0x000000001000ull;
427 static const uptr kLoAppMemEnd = 0x040000000000ull;
428 static const uptr kShadowBeg = 0x200000000000ull;
429 static const uptr kShadowEnd = 0x400000000000ull;
430 static const uptr kMetaShadowBeg = 0x400000000000ull;
431 static const uptr kMetaShadowEnd = 0x480000000000ull;
432 static const uptr kMidAppMemBeg = 0x555555555000ull;
433 static const uptr kMidAppMemEnd = 0x5a0000000000ull;
434 static const uptr kHeapMemBeg = 0x5a0000000000ull;
435 static const uptr kHeapMemEnd = 0x5a0000000000ull;
436 static const uptr kHiAppMemBeg = 0x7a0000000000ull;
437 static const uptr kHiAppMemEnd = 0x7fffffffffffull;
438 static const uptr kShadowMsk = 0x700000000000ull;
439 static const uptr kShadowXor = 0x100000000000ull;
440 static const uptr kShadowAdd = 0x000000000000ull;
441 static const uptr kVdsoBeg = 0x800000000000ull;
442};
443
444/*
445C/C++ on linux/s390x
446While the kernel provides a 64-bit address space, we have to restrict ourselves
447to 48 bits due to how e.g. SyncVar::GetId() works.
4480000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
4490e00 0000 0000 - 2000 0000 0000: -
4502000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
4514000 0000 0000 - 9000 0000 0000: -
4529000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
4539800 0000 0000 - be00 0000 0000: -
454be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
455*/
456struct MappingS390x {
457 static const uptr kMetaShadowBeg = 0x900000000000ull;
458 static const uptr kMetaShadowEnd = 0x980000000000ull;
459 static const uptr kShadowBeg = 0x200000000000ull;
460 static const uptr kShadowEnd = 0x400000000000ull;
461 static const uptr kHeapMemBeg = 0xbe0000000000ull;
462 static const uptr kHeapMemEnd = 0xc00000000000ull;
463 static const uptr kLoAppMemBeg = 0x000000001000ull;
464 static const uptr kLoAppMemEnd = 0x0e0000000000ull;
465 static const uptr kHiAppMemBeg = 0xc00000004000ull;
466 static const uptr kHiAppMemEnd = 0xc00000004000ull;
467 static const uptr kShadowMsk = 0xb00000000000ull;
468 static const uptr kShadowXor = 0x100000000000ull;
469 static const uptr kShadowAdd = 0x000000000000ull;
470 static const uptr kVdsoBeg = 0xfffffffff000ull;
471 static const uptr kMidAppMemBeg = 0;
472 static const uptr kMidAppMemEnd = 0;
473};
474
475/* Go on linux, darwin and freebsd on x86_64
4760000 0000 1000 - 0000 1000 0000: executable
4770000 1000 0000 - 00c0 0000 0000: -
47800c0 0000 0000 - 00e0 0000 0000: heap
47900e0 0000 0000 - 2000 0000 0000: -
4802000 0000 0000 - 21c0 0000 0000: shadow
48121c0 0000 0000 - 3000 0000 0000: -
4823000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
4834000 0000 0000 - 8000 0000 0000: -
484*/
485
486struct MappingGo48 {
487 static const uptr kMetaShadowBeg = 0x300000000000ull;
488 static const uptr kMetaShadowEnd = 0x400000000000ull;
489 static const uptr kShadowBeg = 0x200000000000ull;
490 static const uptr kShadowEnd = 0x21c000000000ull;
491 static const uptr kLoAppMemBeg = 0x000000001000ull;
492 static const uptr kLoAppMemEnd = 0x00e000000000ull;
493 static const uptr kMidAppMemBeg = 0;
494 static const uptr kMidAppMemEnd = 0;
495 static const uptr kHiAppMemBeg = 0;
496 static const uptr kHiAppMemEnd = 0;
497 static const uptr kHeapMemBeg = 0;
498 static const uptr kHeapMemEnd = 0;
499 static const uptr kVdsoBeg = 0;
500 static const uptr kShadowMsk = 0;
501 static const uptr kShadowXor = 0;
502 static const uptr kShadowAdd = 0x200000000000ull;
503};
504
505/* Go on windows
5060000 0000 1000 - 0000 1000 0000: executable
5070000 1000 0000 - 00f8 0000 0000: -
50800c0 0000 0000 - 00e0 0000 0000: heap
50900e0 0000 0000 - 0100 0000 0000: -
5100100 0000 0000 - 0300 0000 0000: shadow
5110300 0000 0000 - 0700 0000 0000: -
5120700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
51307d0 0000 0000 - 8000 0000 0000: -
514PIE binaries currently not supported, but it should be theoretically possible.
515*/
516
517struct MappingGoWindows {
518 static const uptr kMetaShadowBeg = 0x070000000000ull;
519 static const uptr kMetaShadowEnd = 0x077000000000ull;
520 static const uptr kShadowBeg = 0x010000000000ull;
521 static const uptr kShadowEnd = 0x030000000000ull;
522 static const uptr kLoAppMemBeg = 0x000000001000ull;
523 static const uptr kLoAppMemEnd = 0x00e000000000ull;
524 static const uptr kMidAppMemBeg = 0;
525 static const uptr kMidAppMemEnd = 0;
526 static const uptr kHiAppMemBeg = 0;
527 static const uptr kHiAppMemEnd = 0;
528 static const uptr kHeapMemBeg = 0;
529 static const uptr kHeapMemEnd = 0;
530 static const uptr kVdsoBeg = 0;
531 static const uptr kShadowMsk = 0;
532 static const uptr kShadowXor = 0;
533 static const uptr kShadowAdd = 0x010000000000ull;
534};
535
536/* Go on linux/powerpc64 (46-bit VMA)
5370000 0000 1000 - 0000 1000 0000: executable
5380000 1000 0000 - 00c0 0000 0000: -
53900c0 0000 0000 - 00e0 0000 0000: heap
54000e0 0000 0000 - 2000 0000 0000: -
5412000 0000 0000 - 21c0 0000 0000: shadow
54221c0 0000 0000 - 2400 0000 0000: -
5432400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
5442470 0000 0000 - 4000 0000 0000: -
545*/
546
547struct MappingGoPPC64_46 {
548 static const uptr kMetaShadowBeg = 0x240000000000ull;
549 static const uptr kMetaShadowEnd = 0x247000000000ull;
550 static const uptr kShadowBeg = 0x200000000000ull;
551 static const uptr kShadowEnd = 0x21c000000000ull;
552 static const uptr kLoAppMemBeg = 0x000000001000ull;
553 static const uptr kLoAppMemEnd = 0x00e000000000ull;
554 static const uptr kMidAppMemBeg = 0;
555 static const uptr kMidAppMemEnd = 0;
556 static const uptr kHiAppMemBeg = 0;
557 static const uptr kHiAppMemEnd = 0;
558 static const uptr kHeapMemBeg = 0;
559 static const uptr kHeapMemEnd = 0;
560 static const uptr kVdsoBeg = 0;
561 static const uptr kShadowMsk = 0;
562 static const uptr kShadowXor = 0;
563 static const uptr kShadowAdd = 0x200000000000ull;
564};
565
566/* Go on linux/powerpc64 (47-bit VMA)
5670000 0000 1000 - 0000 1000 0000: executable
5680000 1000 0000 - 00c0 0000 0000: -
56900c0 0000 0000 - 00e0 0000 0000: heap
57000e0 0000 0000 - 2000 0000 0000: -
5712000 0000 0000 - 2800 0000 0000: shadow
5722800 0000 0000 - 3000 0000 0000: -
5733000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
5743200 0000 0000 - 8000 0000 0000: -
575*/
576
577struct MappingGoPPC64_47 {
578 static const uptr kMetaShadowBeg = 0x300000000000ull;
579 static const uptr kMetaShadowEnd = 0x320000000000ull;
580 static const uptr kShadowBeg = 0x200000000000ull;
581 static const uptr kShadowEnd = 0x280000000000ull;
582 static const uptr kLoAppMemBeg = 0x000000001000ull;
583 static const uptr kLoAppMemEnd = 0x00e000000000ull;
584 static const uptr kMidAppMemBeg = 0;
585 static const uptr kMidAppMemEnd = 0;
586 static const uptr kHiAppMemBeg = 0;
587 static const uptr kHiAppMemEnd = 0;
588 static const uptr kHeapMemBeg = 0;
589 static const uptr kHeapMemEnd = 0;
590 static const uptr kVdsoBeg = 0;
591 static const uptr kShadowMsk = 0;
592 static const uptr kShadowXor = 0;
593 static const uptr kShadowAdd = 0x200000000000ull;
594};
595
596/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
5970000 0000 1000 - 0000 1000 0000: executable
5980000 1000 0000 - 00c0 0000 0000: -
59900c0 0000 0000 - 00e0 0000 0000: heap
60000e0 0000 0000 - 2000 0000 0000: -
6012000 0000 0000 - 2800 0000 0000: shadow
6022800 0000 0000 - 3000 0000 0000: -
6033000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6043200 0000 0000 - 8000 0000 0000: -
605*/
606struct MappingGoAarch64 {
607 static const uptr kMetaShadowBeg = 0x300000000000ull;
608 static const uptr kMetaShadowEnd = 0x320000000000ull;
609 static const uptr kShadowBeg = 0x200000000000ull;
610 static const uptr kShadowEnd = 0x280000000000ull;
611 static const uptr kLoAppMemBeg = 0x000000001000ull;
612 static const uptr kLoAppMemEnd = 0x00e000000000ull;
613 static const uptr kMidAppMemBeg = 0;
614 static const uptr kMidAppMemEnd = 0;
615 static const uptr kHiAppMemBeg = 0;
616 static const uptr kHiAppMemEnd = 0;
617 static const uptr kHeapMemBeg = 0;
618 static const uptr kHeapMemEnd = 0;
619 static const uptr kVdsoBeg = 0;
620 static const uptr kShadowMsk = 0;
621 static const uptr kShadowXor = 0;
622 static const uptr kShadowAdd = 0x200000000000ull;
623};
624
625/* Go on linux/loongarch64 (47-bit VMA)
6260000 0000 1000 - 0000 1000 0000: executable
6270000 1000 0000 - 00c0 0000 0000: -
62800c0 0000 0000 - 00e0 0000 0000: heap
62900e0 0000 0000 - 2000 0000 0000: -
6302000 0000 0000 - 2800 0000 0000: shadow
6312800 0000 0000 - 3000 0000 0000: -
6323000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6333200 0000 0000 - 8000 0000 0000: -
634*/
635struct MappingGoLoongArch64_47 {
636 static const uptr kMetaShadowBeg = 0x300000000000ull;
637 static const uptr kMetaShadowEnd = 0x320000000000ull;
638 static const uptr kShadowBeg = 0x200000000000ull;
639 static const uptr kShadowEnd = 0x280000000000ull;
640 static const uptr kLoAppMemBeg = 0x000000001000ull;
641 static const uptr kLoAppMemEnd = 0x00e000000000ull;
642 static const uptr kMidAppMemBeg = 0;
643 static const uptr kMidAppMemEnd = 0;
644 static const uptr kHiAppMemBeg = 0;
645 static const uptr kHiAppMemEnd = 0;
646 static const uptr kHeapMemBeg = 0;
647 static const uptr kHeapMemEnd = 0;
648 static const uptr kVdsoBeg = 0;
649 static const uptr kShadowMsk = 0;
650 static const uptr kShadowXor = 0;
651 static const uptr kShadowAdd = 0x200000000000ull;
652};
653
654/*
655Go on linux/mips64 (47-bit VMA)
6560000 0000 1000 - 0000 1000 0000: executable
6570000 1000 0000 - 00c0 0000 0000: -
65800c0 0000 0000 - 00e0 0000 0000: heap
65900e0 0000 0000 - 2000 0000 0000: -
6602000 0000 0000 - 2800 0000 0000: shadow
6612800 0000 0000 - 3000 0000 0000: -
6623000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6633200 0000 0000 - 8000 0000 0000: -
664*/
665struct MappingGoMips64_47 {
666 static const uptr kMetaShadowBeg = 0x300000000000ull;
667 static const uptr kMetaShadowEnd = 0x320000000000ull;
668 static const uptr kShadowBeg = 0x200000000000ull;
669 static const uptr kShadowEnd = 0x280000000000ull;
670 static const uptr kLoAppMemBeg = 0x000000001000ull;
671 static const uptr kLoAppMemEnd = 0x00e000000000ull;
672 static const uptr kMidAppMemBeg = 0;
673 static const uptr kMidAppMemEnd = 0;
674 static const uptr kHiAppMemBeg = 0;
675 static const uptr kHiAppMemEnd = 0;
676 static const uptr kHeapMemBeg = 0;
677 static const uptr kHeapMemEnd = 0;
678 static const uptr kVdsoBeg = 0;
679 static const uptr kShadowMsk = 0;
680 static const uptr kShadowXor = 0;
681 static const uptr kShadowAdd = 0x200000000000ull;
682};
683
684/*
685Go on linux/s390x
6860000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
6871000 0000 0000 - 4000 0000 0000: -
6884000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
6896000 0000 0000 - 9000 0000 0000: -
6909000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
691*/
692struct MappingGoS390x {
693 static const uptr kMetaShadowBeg = 0x900000000000ull;
694 static const uptr kMetaShadowEnd = 0x980000000000ull;
695 static const uptr kShadowBeg = 0x400000000000ull;
696 static const uptr kShadowEnd = 0x600000000000ull;
697 static const uptr kLoAppMemBeg = 0x000000001000ull;
698 static const uptr kLoAppMemEnd = 0x100000000000ull;
699 static const uptr kMidAppMemBeg = 0;
700 static const uptr kMidAppMemEnd = 0;
701 static const uptr kHiAppMemBeg = 0;
702 static const uptr kHiAppMemEnd = 0;
703 static const uptr kHeapMemBeg = 0;
704 static const uptr kHeapMemEnd = 0;
705 static const uptr kVdsoBeg = 0;
706 static const uptr kShadowMsk = 0;
707 static const uptr kShadowXor = 0;
708 static const uptr kShadowAdd = 0x400000000000ull;
709};
710
711extern uptr vmaSize;
712
713template <typename Func, typename Arg>
714ALWAYS_INLINE auto SelectMapping(Arg arg) {
715#if SANITIZER_GO
716# if defined(__powerpc64__)
717 switch (vmaSize) {
718 case 46:
719 return Func::template Apply<MappingGoPPC64_46>(arg);
720 case 47:
721 return Func::template Apply<MappingGoPPC64_47>(arg);
722 }
723# elif defined(__mips64)
724 return Func::template Apply<MappingGoMips64_47>(arg);
725# elif defined(__s390x__)
726 return Func::template Apply<MappingGoS390x>(arg);
727# elif defined(__aarch64__)
728 return Func::template Apply<MappingGoAarch64>(arg);
729# elif defined(__loongarch_lp64)
730 return Func::template Apply<MappingGoLoongArch64_47>(arg);
731# elif SANITIZER_WINDOWS
732 return Func::template Apply<MappingGoWindows>(arg);
733# else
734 return Func::template Apply<MappingGo48>(arg);
735# endif
736#else // SANITIZER_GO
737# if SANITIZER_IOS && !SANITIZER_IOSSIM
738 return Func::template Apply<MappingAppleAarch64>(arg);
739# elif defined(__x86_64__) || SANITIZER_APPLE
740 return Func::template Apply<Mapping48AddressSpace>(arg);
741# elif defined(__aarch64__)
742 switch (vmaSize) {
743 case 39:
744 return Func::template Apply<MappingAarch64_39>(arg);
745 case 42:
746 return Func::template Apply<MappingAarch64_42>(arg);
747 case 48:
748 return Func::template Apply<MappingAarch64_48>(arg);
749 }
750# elif SANITIZER_LOONGARCH64
751 return Func::template Apply<MappingLoongArch64_47>(arg);
752# elif defined(__powerpc64__)
753 switch (vmaSize) {
754 case 44:
755 return Func::template Apply<MappingPPC64_44>(arg);
756 case 46:
757 return Func::template Apply<MappingPPC64_46>(arg);
758 case 47:
759 return Func::template Apply<MappingPPC64_47>(arg);
760 }
761# elif defined(__mips64)
762 return Func::template Apply<MappingMips64_40>(arg);
763# elif SANITIZER_RISCV64
764 switch (vmaSize) {
765 case 39:
766 return Func::template Apply<MappingRiscv64_39>(arg);
767 case 48:
768 return Func::template Apply<MappingRiscv64_48>(arg);
769 }
770# elif defined(__s390x__)
771 return Func::template Apply<MappingS390x>(arg);
772# else
773# error "unsupported platform"
774# endif
775#endif
776 Die();
777}
778
779template <typename Func>
780void ForEachMapping() {
781 Func::template Apply<Mapping48AddressSpace>();
782 Func::template Apply<MappingMips64_40>();
783 Func::template Apply<MappingAppleAarch64>();
784 Func::template Apply<MappingAarch64_39>();
785 Func::template Apply<MappingAarch64_42>();
786 Func::template Apply<MappingAarch64_48>();
787 Func::template Apply<MappingLoongArch64_47>();
788 Func::template Apply<MappingPPC64_44>();
789 Func::template Apply<MappingPPC64_46>();
790 Func::template Apply<MappingPPC64_47>();
791 Func::template Apply<MappingRiscv64_39>();
792 Func::template Apply<MappingRiscv64_48>();
793 Func::template Apply<MappingS390x>();
794 Func::template Apply<MappingGo48>();
795 Func::template Apply<MappingGoWindows>();
796 Func::template Apply<MappingGoPPC64_46>();
797 Func::template Apply<MappingGoPPC64_47>();
798 Func::template Apply<MappingGoAarch64>();
799 Func::template Apply<MappingGoLoongArch64_47>();
800 Func::template Apply<MappingGoMips64_47>();
801 Func::template Apply<MappingGoS390x>();
802}
803
804enum MappingType {
805 kLoAppMemBeg,
806 kLoAppMemEnd,
807 kHiAppMemBeg,
808 kHiAppMemEnd,
809 kMidAppMemBeg,
810 kMidAppMemEnd,
811 kHeapMemBeg,
812 kHeapMemEnd,
813 kShadowBeg,
814 kShadowEnd,
815 kMetaShadowBeg,
816 kMetaShadowEnd,
817 kVdsoBeg,
818};
819
820struct MappingField {
821 template <typename Mapping>
822 static uptr Apply(MappingType type) {
823 switch (type) {
824 case kLoAppMemBeg:
825 return Mapping::kLoAppMemBeg;
826 case kLoAppMemEnd:
827 return Mapping::kLoAppMemEnd;
828 case kMidAppMemBeg:
829 return Mapping::kMidAppMemBeg;
830 case kMidAppMemEnd:
831 return Mapping::kMidAppMemEnd;
832 case kHiAppMemBeg:
833 return Mapping::kHiAppMemBeg;
834 case kHiAppMemEnd:
835 return Mapping::kHiAppMemEnd;
836 case kHeapMemBeg:
837 return Mapping::kHeapMemBeg;
838 case kHeapMemEnd:
839 return Mapping::kHeapMemEnd;
840 case kVdsoBeg:
841 return Mapping::kVdsoBeg;
842 case kShadowBeg:
843 return Mapping::kShadowBeg;
844 case kShadowEnd:
845 return Mapping::kShadowEnd;
846 case kMetaShadowBeg:
847 return Mapping::kMetaShadowBeg;
848 case kMetaShadowEnd:
849 return Mapping::kMetaShadowEnd;
850 }
851 Die();
852 }
853};
854
855ALWAYS_INLINE
856uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(arg: kLoAppMemBeg); }
857ALWAYS_INLINE
858uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(arg: kLoAppMemEnd); }
859
860ALWAYS_INLINE
861uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(arg: kMidAppMemBeg); }
862ALWAYS_INLINE
863uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(arg: kMidAppMemEnd); }
864
865ALWAYS_INLINE
866uptr HeapMemBeg(void) { return SelectMapping<MappingField>(arg: kHeapMemBeg); }
867ALWAYS_INLINE
868uptr HeapMemEnd(void) { return SelectMapping<MappingField>(arg: kHeapMemEnd); }
869
870ALWAYS_INLINE
871uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(arg: kHiAppMemBeg); }
872ALWAYS_INLINE
873uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(arg: kHiAppMemEnd); }
874
875ALWAYS_INLINE
876uptr VdsoBeg(void) { return SelectMapping<MappingField>(arg: kVdsoBeg); }
877
878ALWAYS_INLINE
879uptr ShadowBeg(void) { return SelectMapping<MappingField>(arg: kShadowBeg); }
880ALWAYS_INLINE
881uptr ShadowEnd(void) { return SelectMapping<MappingField>(arg: kShadowEnd); }
882
883ALWAYS_INLINE
884uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(arg: kMetaShadowBeg); }
885ALWAYS_INLINE
886uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(arg: kMetaShadowEnd); }
887
888struct IsAppMemImpl {
889 template <typename Mapping>
890 static bool Apply(uptr mem) {
891 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
892 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
893 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
894 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
895 }
896};
897
898ALWAYS_INLINE
899bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(arg: mem); }
900
901struct IsShadowMemImpl {
902 template <typename Mapping>
903 static bool Apply(uptr mem) {
904 return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
905 }
906};
907
908ALWAYS_INLINE
909bool IsShadowMem(RawShadow *p) {
910 return SelectMapping<IsShadowMemImpl>(arg: reinterpret_cast<uptr>(p));
911}
912
913struct IsMetaMemImpl {
914 template <typename Mapping>
915 static bool Apply(uptr mem) {
916 return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
917 }
918};
919
920ALWAYS_INLINE
921bool IsMetaMem(const u32 *p) {
922 return SelectMapping<IsMetaMemImpl>(arg: reinterpret_cast<uptr>(p));
923}
924
925struct MemToShadowImpl {
926 template <typename Mapping>
927 static uptr Apply(uptr x) {
928 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
929 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
930 Mapping::kShadowXor) *
931 kShadowMultiplier +
932 Mapping::kShadowAdd;
933 }
934};
935
936ALWAYS_INLINE
937RawShadow *MemToShadow(uptr x) {
938 return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(arg: x));
939}
940
941struct MemToMetaImpl {
942 template <typename Mapping>
943 static u32 *Apply(uptr x) {
944 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
945 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
946 kMetaShadowCell * kMetaShadowSize) |
947 Mapping::kMetaShadowBeg);
948 }
949};
950
951ALWAYS_INLINE
952u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(arg: x); }
953
954struct ShadowToMemImpl {
955 template <typename Mapping>
956 static uptr Apply(uptr sp) {
957 if (!IsShadowMemImpl::Apply<Mapping>(sp))
958 return 0;
959 // The shadow mapping is non-linear and we've lost some bits, so we don't
960 // have an easy way to restore the original app address. But the mapping is
961 // a bijection, so we try to restore the address as belonging to
962 // low/mid/high range consecutively and see if shadow->app->shadow mapping
963 // gives us the same address.
964 uptr p =
965 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
966 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
967 MemToShadowImpl::Apply<Mapping>(p) == sp)
968 return p;
969 if (Mapping::kMidAppMemBeg) {
970 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
971 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
972 MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
973 return p_mid;
974 }
975 return p | Mapping::kShadowMsk;
976 }
977};
978
979ALWAYS_INLINE
980uptr ShadowToMem(RawShadow *s) {
981 return SelectMapping<ShadowToMemImpl>(arg: reinterpret_cast<uptr>(s));
982}
983
984// Compresses addr to kCompressedAddrBits stored in least significant bits.
985ALWAYS_INLINE uptr CompressAddr(uptr addr) {
986 return addr & ((1ull << kCompressedAddrBits) - 1);
987}
988
989struct RestoreAddrImpl {
990 typedef uptr Result;
991 template <typename Mapping>
992 static Result Apply(uptr addr) {
993 // To restore the address we go over all app memory ranges and check if top
994 // 3 bits of the compressed addr match that of the app range. If yes, we
995 // assume that the compressed address come from that range and restore the
996 // missing top bits to match the app range address.
997 const uptr ranges[] = {
998 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
999 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
1000 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
1001 };
1002 const uptr indicator = 0x0e0000000000ull;
1003 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(x: indicator);
1004 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
1005 uptr beg = ranges[i];
1006 uptr end = ranges[i + 1];
1007 if (beg == end)
1008 continue;
1009 for (uptr p = beg; p < end; p = RoundDown(p: p + ind_lsb, align: ind_lsb)) {
1010 if ((addr & indicator) == (p & indicator))
1011 return addr | (p & ~(ind_lsb - 1));
1012 }
1013 }
1014 Printf(format: "ThreadSanitizer: failed to restore address 0x%zx\n", addr);
1015 Die();
1016 }
1017};
1018
1019// Restores compressed addr from kCompressedAddrBits to full representation.
1020// This is called only during reporting and is not performance-critical.
1021inline uptr RestoreAddr(uptr addr) {
1022 return SelectMapping<RestoreAddrImpl>(arg: addr);
1023}
1024
1025void InitializePlatform();
1026void InitializePlatformEarly();
1027bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings);
1028void InitializeShadowMemoryPlatform();
1029void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
1030int ExtractResolvFDs(void *state, int *fds, int nfd);
1031int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
1032uptr ExtractLongJmpSp(uptr *env);
1033void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
1034
1035int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
1036 void (*cleanup)(void *arg), void *arg);
1037
1038void DestroyThreadState();
1039void PlatformCleanUpThreadState(ThreadState *thr);
1040
1041} // namespace __tsan
1042
1043#endif // TSAN_PLATFORM_H
1044

source code of compiler-rt/lib/tsan/rtl/tsan_platform.h