1//===-- sanitizer_linux_libcdep.cpp ---------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is shared between AddressSanitizer and ThreadSanitizer
10// run-time libraries and implements linux-specific functions from
11// sanitizer_libc.h.
12//===----------------------------------------------------------------------===//
13
14#include "sanitizer_platform.h"
15
16#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
17 SANITIZER_SOLARIS
18
19# include "sanitizer_allocator_internal.h"
20# include "sanitizer_atomic.h"
21# include "sanitizer_common.h"
22# include "sanitizer_file.h"
23# include "sanitizer_flags.h"
24# include "sanitizer_getauxval.h"
25# include "sanitizer_glibc_version.h"
26# include "sanitizer_linux.h"
27# include "sanitizer_placement_new.h"
28# include "sanitizer_procmaps.h"
29# include "sanitizer_solaris.h"
30
31# if SANITIZER_NETBSD
32# define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
33# endif
34
35# include <dlfcn.h> // for dlsym()
36# include <link.h>
37# include <pthread.h>
38# include <signal.h>
39# include <sys/mman.h>
40# include <sys/resource.h>
41# include <syslog.h>
42
43# if !defined(ElfW)
44# define ElfW(type) Elf_##type
45# endif
46
47# if SANITIZER_FREEBSD
48# include <pthread_np.h>
49# include <sys/auxv.h>
50# include <sys/sysctl.h>
51# define pthread_getattr_np pthread_attr_get_np
52// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
53// that, it was never implemented. So just define it to zero.
54# undef MAP_NORESERVE
55# define MAP_NORESERVE 0
56extern const Elf_Auxinfo *__elf_aux_vector;
57# endif
58
59# if SANITIZER_NETBSD
60# include <lwp.h>
61# include <sys/sysctl.h>
62# include <sys/tls.h>
63# endif
64
65# if SANITIZER_SOLARIS
66# include <stddef.h>
67# include <stdlib.h>
68# include <thread.h>
69# endif
70
71# if SANITIZER_ANDROID
72# include <android/api-level.h>
73# if !defined(CPU_COUNT) && !defined(__aarch64__)
74# include <dirent.h>
75# include <fcntl.h>
76struct __sanitizer::linux_dirent {
77 long d_ino;
78 off_t d_off;
79 unsigned short d_reclen;
80 char d_name[];
81};
82# endif
83# endif
84
85# if !SANITIZER_ANDROID
86# include <elf.h>
87# include <unistd.h>
88# endif
89
90namespace __sanitizer {
91
92SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act,
93 void *oldact);
94
95int internal_sigaction(int signum, const void *act, void *oldact) {
96# if !SANITIZER_GO
97 if (&real_sigaction)
98 return real_sigaction(signum, act, oldact);
99# endif
100 return sigaction(sig: signum, act: (const struct sigaction *)act,
101 oact: (struct sigaction *)oldact);
102}
103
104void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
105 uptr *stack_bottom) {
106 CHECK(stack_top);
107 CHECK(stack_bottom);
108 if (at_initialization) {
109 // This is the main thread. Libpthread may not be initialized yet.
110 struct rlimit rl;
111 CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
112
113 // Find the mapping that contains a stack variable.
114 MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
115 if (proc_maps.Error()) {
116 *stack_top = *stack_bottom = 0;
117 return;
118 }
119 MemoryMappedSegment segment;
120 uptr prev_end = 0;
121 while (proc_maps.Next(segment: &segment)) {
122 if ((uptr)&rl < segment.end)
123 break;
124 prev_end = segment.end;
125 }
126 CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
127
128 // Get stacksize from rlimit, but clip it so that it does not overlap
129 // with other mappings.
130 uptr stacksize = rl.rlim_cur;
131 if (stacksize > segment.end - prev_end)
132 stacksize = segment.end - prev_end;
133 // When running with unlimited stack size, we still want to set some limit.
134 // The unlimited stack size is caused by 'ulimit -s unlimited'.
135 // Also, for some reason, GNU make spawns subprocesses with unlimited stack.
136 if (stacksize > kMaxThreadStackSize)
137 stacksize = kMaxThreadStackSize;
138 *stack_top = segment.end;
139 *stack_bottom = segment.end - stacksize;
140 return;
141 }
142 uptr stacksize = 0;
143 void *stackaddr = nullptr;
144# if SANITIZER_SOLARIS
145 stack_t ss;
146 CHECK_EQ(thr_stksegment(&ss), 0);
147 stacksize = ss.ss_size;
148 stackaddr = (char *)ss.ss_sp - stacksize;
149# else // !SANITIZER_SOLARIS
150 pthread_attr_t attr;
151 pthread_attr_init(attr: &attr);
152 CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
153 internal_pthread_attr_getstack(attr: &attr, addr: &stackaddr, size: &stacksize);
154 pthread_attr_destroy(attr: &attr);
155# endif // SANITIZER_SOLARIS
156
157 *stack_top = (uptr)stackaddr + stacksize;
158 *stack_bottom = (uptr)stackaddr;
159}
160
161# if !SANITIZER_GO
162bool SetEnv(const char *name, const char *value) {
163 void *f = dlsym(RTLD_NEXT, name: "setenv");
164 if (!f)
165 return false;
166 typedef int (*setenv_ft)(const char *name, const char *value, int overwrite);
167 setenv_ft setenv_f;
168 CHECK_EQ(sizeof(setenv_f), sizeof(f));
169 internal_memcpy(dest: &setenv_f, src: &f, n: sizeof(f));
170 return setenv_f(name, value, 1) == 0;
171}
172# endif
173
174__attribute__((unused)) static bool GetLibcVersion(int *major, int *minor,
175 int *patch) {
176# ifdef _CS_GNU_LIBC_VERSION
177 char buf[64];
178 uptr len = confstr(_CS_GNU_LIBC_VERSION, buf: buf, len: sizeof(buf));
179 if (len >= sizeof(buf))
180 return false;
181 buf[len] = 0;
182 static const char kGLibC[] = "glibc ";
183 if (internal_strncmp(s1: buf, s2: kGLibC, n: sizeof(kGLibC) - 1) != 0)
184 return false;
185 const char *p = buf + sizeof(kGLibC) - 1;
186 *major = internal_simple_strtoll(nptr: p, endptr: &p, base: 10);
187 *minor = (*p == '.') ? internal_simple_strtoll(nptr: p + 1, endptr: &p, base: 10) : 0;
188 *patch = (*p == '.') ? internal_simple_strtoll(nptr: p + 1, endptr: &p, base: 10) : 0;
189 return true;
190# else
191 return false;
192# endif
193}
194
195// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
196// #19826) so dlpi_tls_data cannot be used.
197//
198// musl before 1.2.3 and FreeBSD as of 12.2 incorrectly set dlpi_tls_data to
199// the TLS initialization image
200// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774
201__attribute__((unused)) static int g_use_dlpi_tls_data;
202
203# if SANITIZER_GLIBC && !SANITIZER_GO
204__attribute__((unused)) static size_t g_tls_size;
205void InitTlsSize() {
206 int major, minor, patch;
207 g_use_dlpi_tls_data =
208 GetLibcVersion(major: &major, minor: &minor, patch: &patch) && major == 2 && minor >= 25;
209
210# if defined(__aarch64__) || defined(__x86_64__) || \
211 defined(__powerpc64__) || defined(__loongarch__)
212 void *get_tls_static_info = dlsym(RTLD_NEXT, name: "_dl_get_tls_static_info");
213 size_t tls_align;
214 ((void (*)(size_t *, size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
215# endif
216}
217# else
218void InitTlsSize() {}
219# endif // SANITIZER_GLIBC && !SANITIZER_GO
220
221// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
222// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
223// to get the pointer to thread-specific data keys in the thread control block.
224# if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \
225 !SANITIZER_ANDROID && !SANITIZER_GO
226// sizeof(struct pthread) from glibc.
227static atomic_uintptr_t thread_descriptor_size;
228
229static uptr ThreadDescriptorSizeFallback() {
230 uptr val = 0;
231# if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
232 int major;
233 int minor;
234 int patch;
235 if (GetLibcVersion(major: &major, minor: &minor, patch: &patch) && major == 2) {
236 /* sizeof(struct pthread) values from various glibc versions. */
237 if (SANITIZER_X32)
238 val = 1728; // Assume only one particular version for x32.
239 // For ARM sizeof(struct pthread) changed in Glibc 2.23.
240 else if (SANITIZER_ARM)
241 val = minor <= 22 ? 1120 : 1216;
242 else if (minor <= 3)
243 val = FIRST_32_SECOND_64(1104, 1696);
244 else if (minor == 4)
245 val = FIRST_32_SECOND_64(1120, 1728);
246 else if (minor == 5)
247 val = FIRST_32_SECOND_64(1136, 1728);
248 else if (minor <= 9)
249 val = FIRST_32_SECOND_64(1136, 1712);
250 else if (minor == 10)
251 val = FIRST_32_SECOND_64(1168, 1776);
252 else if (minor == 11 || (minor == 12 && patch == 1))
253 val = FIRST_32_SECOND_64(1168, 2288);
254 else if (minor <= 14)
255 val = FIRST_32_SECOND_64(1168, 2304);
256 else if (minor < 32) // Unknown version
257 val = FIRST_32_SECOND_64(1216, 2304);
258 else // minor == 32
259 val = FIRST_32_SECOND_64(1344, 2496);
260 }
261# elif defined(__s390__) || defined(__sparc__)
262 // The size of a prefix of TCB including pthread::{specific_1stblock,specific}
263 // suffices. Just return offsetof(struct pthread, specific_used), which hasn't
264 // changed since 2007-05. Technically this applies to i386/x86_64 as well but
265 // we call _dl_get_tls_static_info and need the precise size of struct
266 // pthread.
267 return FIRST_32_SECOND_64(524, 1552);
268# elif defined(__mips__)
269 // TODO(sagarthakur): add more values as per different glibc versions.
270 val = FIRST_32_SECOND_64(1152, 1776);
271# elif SANITIZER_LOONGARCH64
272 val = 1856; // from glibc 2.36
273# elif SANITIZER_RISCV64
274 int major;
275 int minor;
276 int patch;
277 if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
278 // TODO: consider adding an optional runtime check for an unknown (untested)
279 // glibc version
280 if (minor <= 28) // WARNING: the highest tested version is 2.29
281 val = 1772; // no guarantees for this one
282 else if (minor <= 31)
283 val = 1772; // tested against glibc 2.29, 2.31
284 else
285 val = 1936; // tested against glibc 2.32
286 }
287
288# elif defined(__aarch64__)
289 // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
290 val = 1776;
291# elif defined(__powerpc64__)
292 val = 1776; // from glibc.ppc64le 2.20-8.fc21
293# endif
294 return val;
295}
296
297uptr ThreadDescriptorSize() {
298 uptr val = atomic_load_relaxed(a: &thread_descriptor_size);
299 if (val)
300 return val;
301 // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
302 // glibc 2.34 and later.
303 if (unsigned *psizeof = static_cast<unsigned *>(
304 dlsym(RTLD_DEFAULT, name: "_thread_db_sizeof_pthread")))
305 val = *psizeof;
306 if (!val)
307 val = ThreadDescriptorSizeFallback();
308 atomic_store_relaxed(a: &thread_descriptor_size, v: val);
309 return val;
310}
311
312# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 || \
313 SANITIZER_LOONGARCH64
314// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
315// head structure. It lies before the static tls blocks.
316static uptr TlsPreTcbSize() {
317# if defined(__mips__)
318 const uptr kTcbHead = 16; // sizeof (tcbhead_t)
319# elif defined(__powerpc64__)
320 const uptr kTcbHead = 88; // sizeof (tcbhead_t)
321# elif SANITIZER_RISCV64
322 const uptr kTcbHead = 16; // sizeof (tcbhead_t)
323# elif SANITIZER_LOONGARCH64
324 const uptr kTcbHead = 16; // sizeof (tcbhead_t)
325# endif
326 const uptr kTlsAlign = 16;
327 const uptr kTlsPreTcbSize =
328 RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
329 return kTlsPreTcbSize;
330}
331# endif
332
333namespace {
334struct TlsBlock {
335 uptr begin, end, align;
336 size_t tls_modid;
337 bool operator<(const TlsBlock &rhs) const { return begin < rhs.begin; }
338};
339} // namespace
340
341# ifdef __s390__
342extern "C" uptr __tls_get_offset(void *arg);
343
344static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
345 // The __tls_get_offset ABI requires %r12 to point to GOT and %r2 to be an
346 // offset of a struct tls_index inside GOT. We don't possess either of the
347 // two, so violate the letter of the "ELF Handling For Thread-Local
348 // Storage" document and assume that the implementation just dereferences
349 // %r2 + %r12.
350 uptr tls_index[2] = {ti_module, ti_offset};
351 register uptr r2 asm("2") = 0;
352 register void *r12 asm("12") = tls_index;
353 asm("basr %%r14, %[__tls_get_offset]"
354 : "+r"(r2)
355 : [__tls_get_offset] "r"(__tls_get_offset), "r"(r12)
356 : "memory", "cc", "0", "1", "3", "4", "5", "14");
357 return r2;
358}
359# else
360extern "C" void *__tls_get_addr(size_t *);
361# endif
362
363static size_t main_tls_modid;
364
365static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
366 void *data) {
367 size_t tls_modid;
368# if SANITIZER_SOLARIS
369 // dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use
370 // dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,
371 // 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in
372 // 11.4 to match other implementations.
373 if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid))
374 main_tls_modid = 1;
375 else
376 main_tls_modid = 0;
377 g_use_dlpi_tls_data = 0;
378 Rt_map *map;
379 dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);
380 tls_modid = map->rt_tlsmodid;
381# else
382 main_tls_modid = 1;
383 tls_modid = info->dlpi_tls_modid;
384# endif
385
386 if (tls_modid < main_tls_modid)
387 return 0;
388 uptr begin;
389# if !SANITIZER_SOLARIS
390 begin = (uptr)info->dlpi_tls_data;
391# endif
392 if (!g_use_dlpi_tls_data) {
393 // Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
394 // and FreeBSD.
395# ifdef __s390__
396 begin = (uptr)__builtin_thread_pointer() + TlsGetOffset(tls_modid, 0);
397# else
398 size_t mod_and_off[2] = {tls_modid, 0};
399 begin = (uptr)__tls_get_addr(mod_and_off);
400# endif
401 }
402 for (unsigned i = 0; i != info->dlpi_phnum; ++i)
403 if (info->dlpi_phdr[i].p_type == PT_TLS) {
404 static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(
405 element: TlsBlock{.begin: begin, .end: begin + info->dlpi_phdr[i].p_memsz,
406 .align: info->dlpi_phdr[i].p_align, .tls_modid: tls_modid});
407 break;
408 }
409 return 0;
410}
411
412__attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size,
413 uptr *align) {
414 InternalMmapVector<TlsBlock> ranges;
415 dl_iterate_phdr(callback: CollectStaticTlsBlocks, data: &ranges);
416 uptr len = ranges.size();
417 Sort(v: ranges.begin(), size: len);
418 // Find the range with tls_modid == main_tls_modid. For glibc, because
419 // libc.so uses PT_TLS, this module is guaranteed to exist and is one of
420 // the initially loaded modules.
421 uptr one = 0;
422 while (one != len && ranges[one].tls_modid != main_tls_modid) ++one;
423 if (one == len) {
424 // This may happen with musl if no module uses PT_TLS.
425 *addr = 0;
426 *size = 0;
427 *align = 1;
428 return;
429 }
430 // Find the maximum consecutive ranges. We consider two modules consecutive if
431 // the gap is smaller than the alignment of the latter range. The dynamic
432 // loader places static TLS blocks this way not to waste space.
433 uptr l = one;
434 *align = ranges[l].align;
435 while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align)
436 *align = Max(a: *align, b: ranges[--l].align);
437 uptr r = one + 1;
438 while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align)
439 *align = Max(a: *align, b: ranges[r++].align);
440 *addr = ranges[l].begin;
441 *size = ranges[r - 1].end - ranges[l].begin;
442}
443# endif // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD ||
444 // SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
445
446# if SANITIZER_NETBSD
447static struct tls_tcb *ThreadSelfTlsTcb() {
448 struct tls_tcb *tcb = nullptr;
449# ifdef __HAVE___LWP_GETTCB_FAST
450 tcb = (struct tls_tcb *)__lwp_gettcb_fast();
451# elif defined(__HAVE___LWP_GETPRIVATE_FAST)
452 tcb = (struct tls_tcb *)__lwp_getprivate_fast();
453# endif
454 return tcb;
455}
456
457uptr ThreadSelf() { return (uptr)ThreadSelfTlsTcb()->tcb_pthread; }
458
459int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) {
460 const Elf_Phdr *hdr = info->dlpi_phdr;
461 const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum;
462
463 for (; hdr != last_hdr; ++hdr) {
464 if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
465 *(uptr *)data = hdr->p_memsz;
466 break;
467 }
468 }
469 return 0;
470}
471# endif // SANITIZER_NETBSD
472
473# if SANITIZER_ANDROID
474// Bionic provides this API since S.
475extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,
476 void **);
477# endif
478
479# if !SANITIZER_GO
480static void GetTls(uptr *addr, uptr *size) {
481# if SANITIZER_ANDROID
482 if (&__libc_get_static_tls_bounds) {
483 void *start_addr;
484 void *end_addr;
485 __libc_get_static_tls_bounds(&start_addr, &end_addr);
486 *addr = reinterpret_cast<uptr>(start_addr);
487 *size =
488 reinterpret_cast<uptr>(end_addr) - reinterpret_cast<uptr>(start_addr);
489 } else {
490 *addr = 0;
491 *size = 0;
492 }
493# elif SANITIZER_GLIBC && defined(__x86_64__)
494 // For aarch64 and x86-64, use an O(1) approach which requires relatively
495 // precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
496# if SANITIZER_X32
497 asm("mov %%fs:8,%0" : "=r"(*addr));
498# else
499 asm("mov %%fs:16,%0" : "=r"(*addr));
500# endif
501 *size = g_tls_size;
502 *addr -= *size;
503 *addr += ThreadDescriptorSize();
504# elif SANITIZER_GLIBC && defined(__aarch64__)
505 *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
506 ThreadDescriptorSize();
507 *size = g_tls_size + ThreadDescriptorSize();
508# elif SANITIZER_GLIBC && defined(__loongarch__)
509# ifdef __clang__
510 *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
511 ThreadDescriptorSize();
512# else
513 asm("or %0,$tp,$zero" : "=r"(*addr));
514 *addr -= ThreadDescriptorSize();
515# endif
516 *size = g_tls_size + ThreadDescriptorSize();
517# elif SANITIZER_GLIBC && defined(__powerpc64__)
518 // Workaround for glibc<2.25(?). 2.27 is known to not need this.
519 uptr tp;
520 asm("addi %0,13,-0x7000" : "=r"(tp));
521 const uptr pre_tcb_size = TlsPreTcbSize();
522 *addr = tp - pre_tcb_size;
523 *size = g_tls_size + pre_tcb_size;
524# elif SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS
525 uptr align;
526 GetStaticTlsBoundary(addr, size, &align);
527# if defined(__x86_64__) || defined(__i386__) || defined(__s390__) || \
528 defined(__sparc__)
529 if (SANITIZER_GLIBC) {
530# if defined(__x86_64__) || defined(__i386__)
531 align = Max<uptr>(align, 64);
532# else
533 align = Max<uptr>(align, 16);
534# endif
535 }
536 const uptr tp = RoundUpTo(*addr + *size, align);
537
538 // lsan requires the range to additionally cover the static TLS surplus
539 // (elf/dl-tls.c defines 1664). Otherwise there may be false positives for
540 // allocations only referenced by tls in dynamically loaded modules.
541 if (SANITIZER_GLIBC)
542 *size += 1644;
543 else if (SANITIZER_FREEBSD)
544 *size += 128; // RTLD_STATIC_TLS_EXTRA
545
546 // Extend the range to include the thread control block. On glibc, lsan needs
547 // the range to include pthread::{specific_1stblock,specific} so that
548 // allocations only referenced by pthread_setspecific can be scanned. This may
549 // underestimate by at most TLS_TCB_ALIGN-1 bytes but it should be fine
550 // because the number of bytes after pthread::specific is larger.
551 *addr = tp - RoundUpTo(*size, align);
552 *size = tp - *addr + ThreadDescriptorSize();
553# else
554 if (SANITIZER_GLIBC)
555 *size += 1664;
556 else if (SANITIZER_FREEBSD)
557 *size += 128; // RTLD_STATIC_TLS_EXTRA
558# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
559 const uptr pre_tcb_size = TlsPreTcbSize();
560 *addr -= pre_tcb_size;
561 *size += pre_tcb_size;
562# else
563 // arm and aarch64 reserve two words at TP, so this underestimates the range.
564 // However, this is sufficient for the purpose of finding the pointers to
565 // thread-specific data keys.
566 const uptr tcb_size = ThreadDescriptorSize();
567 *addr -= tcb_size;
568 *size += tcb_size;
569# endif
570# endif
571# elif SANITIZER_NETBSD
572 struct tls_tcb *const tcb = ThreadSelfTlsTcb();
573 *addr = 0;
574 *size = 0;
575 if (tcb != 0) {
576 // Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program).
577 // ld.elf_so hardcodes the index 1.
578 dl_iterate_phdr(GetSizeFromHdr, size);
579
580 if (*size != 0) {
581 // The block has been found and tcb_dtv[1] contains the base address
582 *addr = (uptr)tcb->tcb_dtv[1];
583 }
584 }
585# else
586# error "Unknown OS"
587# endif
588}
589# endif
590
591# if !SANITIZER_GO
592uptr GetTlsSize() {
593# if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
594 SANITIZER_SOLARIS
595 uptr addr, size;
596 GetTls(addr: &addr, size: &size);
597 return size;
598# else
599 return 0;
600# endif
601}
602# endif
603
604void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
605 uptr *tls_addr, uptr *tls_size) {
606# if SANITIZER_GO
607 // Stub implementation for Go.
608 *stk_addr = *stk_size = *tls_addr = *tls_size = 0;
609# else
610 GetTls(addr: tls_addr, size: tls_size);
611
612 uptr stack_top, stack_bottom;
613 GetThreadStackTopAndBottom(at_initialization: main, stack_top: &stack_top, stack_bottom: &stack_bottom);
614 *stk_addr = stack_bottom;
615 *stk_size = stack_top - stack_bottom;
616
617 if (!main) {
618 // If stack and tls intersect, make them non-intersecting.
619 if (*tls_addr > *stk_addr && *tls_addr < *stk_addr + *stk_size) {
620 if (*stk_addr + *stk_size < *tls_addr + *tls_size)
621 *tls_size = *stk_addr + *stk_size - *tls_addr;
622 *stk_size = *tls_addr - *stk_addr;
623 }
624 }
625# endif
626}
627
628# if !SANITIZER_FREEBSD
629typedef ElfW(Phdr) Elf_Phdr;
630# endif
631
632struct DlIteratePhdrData {
633 InternalMmapVectorNoCtor<LoadedModule> *modules;
634 bool first;
635};
636
637static int AddModuleSegments(const char *module_name, dl_phdr_info *info,
638 InternalMmapVectorNoCtor<LoadedModule> *modules) {
639 if (module_name[0] == '\0')
640 return 0;
641 LoadedModule cur_module;
642 cur_module.set(module_name, base_address: info->dlpi_addr);
643 for (int i = 0; i < (int)info->dlpi_phnum; i++) {
644 const Elf_Phdr *phdr = &info->dlpi_phdr[i];
645 if (phdr->p_type == PT_LOAD) {
646 uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
647 uptr cur_end = cur_beg + phdr->p_memsz;
648 bool executable = phdr->p_flags & PF_X;
649 bool writable = phdr->p_flags & PF_W;
650 cur_module.addAddressRange(beg: cur_beg, end: cur_end, executable, writable);
651 } else if (phdr->p_type == PT_NOTE) {
652# ifdef NT_GNU_BUILD_ID
653 uptr off = 0;
654 while (off + sizeof(ElfW(Nhdr)) < phdr->p_memsz) {
655 auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(info->dlpi_addr +
656 phdr->p_vaddr + off);
657 constexpr auto kGnuNamesz = 4; // "GNU" with NUL-byte.
658 static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4.");
659 if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) {
660 if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz >
661 phdr->p_memsz) {
662 // Something is very wrong, bail out instead of reading potentially
663 // arbitrary memory.
664 break;
665 }
666 const char *name =
667 reinterpret_cast<const char *>(nhdr) + sizeof(*nhdr);
668 if (internal_memcmp(s1: name, s2: "GNU", n: 3) == 0) {
669 const char *value = reinterpret_cast<const char *>(nhdr) +
670 sizeof(*nhdr) + kGnuNamesz;
671 cur_module.setUuid(uuid: value, size: nhdr->n_descsz);
672 break;
673 }
674 }
675 off += sizeof(*nhdr) + RoundUpTo(size: nhdr->n_namesz, boundary: 4) +
676 RoundUpTo(size: nhdr->n_descsz, boundary: 4);
677 }
678# endif
679 }
680 }
681 modules->push_back(element: cur_module);
682 return 0;
683}
684
685static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
686 DlIteratePhdrData *data = (DlIteratePhdrData *)arg;
687 if (data->first) {
688 InternalMmapVector<char> module_name(kMaxPathLength);
689 data->first = false;
690 // First module is the binary itself.
691 ReadBinaryNameCached(buf: module_name.data(), buf_len: module_name.size());
692 return AddModuleSegments(module_name: module_name.data(), info, modules: data->modules);
693 }
694
695 if (info->dlpi_name)
696 return AddModuleSegments(module_name: info->dlpi_name, info, modules: data->modules);
697
698 return 0;
699}
700
701# if SANITIZER_ANDROID && __ANDROID_API__ < 21
702extern "C" __attribute__((weak)) int dl_iterate_phdr(
703 int (*)(struct dl_phdr_info *, size_t, void *), void *);
704# endif
705
706static bool requiresProcmaps() {
707# if SANITIZER_ANDROID && __ANDROID_API__ <= 22
708 // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
709 // The runtime check allows the same library to work with
710 // both K and L (and future) Android releases.
711 return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
712# else
713 return false;
714# endif
715}
716
717static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
718 MemoryMappingLayout memory_mapping(/*cache_enabled*/ true);
719 memory_mapping.DumpListOfModules(modules);
720}
721
722void ListOfModules::init() {
723 clearOrInit();
724 if (requiresProcmaps()) {
725 procmapsInit(modules: &modules_);
726 } else {
727 DlIteratePhdrData data = {.modules: &modules_, .first: true};
728 dl_iterate_phdr(callback: dl_iterate_phdr_cb, data: &data);
729 }
730}
731
732// When a custom loader is used, dl_iterate_phdr may not contain the full
733// list of modules. Allow callers to fall back to using procmaps.
734void ListOfModules::fallbackInit() {
735 if (!requiresProcmaps()) {
736 clearOrInit();
737 procmapsInit(modules: &modules_);
738 } else {
739 clear();
740 }
741}
742
743// getrusage does not give us the current RSS, only the max RSS.
744// Still, this is better than nothing if /proc/self/statm is not available
745// for some reason, e.g. due to a sandbox.
746static uptr GetRSSFromGetrusage() {
747 struct rusage usage;
748 if (getrusage(RUSAGE_SELF, usage: &usage)) // Failed, probably due to a sandbox.
749 return 0;
750 return usage.ru_maxrss << 10; // ru_maxrss is in Kb.
751}
752
753uptr GetRSS() {
754 if (!common_flags()->can_use_proc_maps_statm)
755 return GetRSSFromGetrusage();
756 fd_t fd = OpenFile(filename: "/proc/self/statm", mode: RdOnly);
757 if (fd == kInvalidFd)
758 return GetRSSFromGetrusage();
759 char buf[64];
760 uptr len = internal_read(fd, buf, count: sizeof(buf) - 1);
761 internal_close(fd);
762 if ((sptr)len <= 0)
763 return 0;
764 buf[len] = 0;
765 // The format of the file is:
766 // 1084 89 69 11 0 79 0
767 // We need the second number which is RSS in pages.
768 char *pos = buf;
769 // Skip the first number.
770 while (*pos >= '0' && *pos <= '9') pos++;
771 // Skip whitespaces.
772 while (!(*pos >= '0' && *pos <= '9') && *pos != 0) pos++;
773 // Read the number.
774 uptr rss = 0;
775 while (*pos >= '0' && *pos <= '9') rss = rss * 10 + *pos++ - '0';
776 return rss * GetPageSizeCached();
777}
778
779// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
780// they allocate memory.
781u32 GetNumberOfCPUs() {
782# if SANITIZER_FREEBSD || SANITIZER_NETBSD
783 u32 ncpu;
784 int req[2];
785 uptr len = sizeof(ncpu);
786 req[0] = CTL_HW;
787 req[1] = HW_NCPU;
788 CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
789 return ncpu;
790# elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
791 // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
792 // exist in sched.h. That is the case for toolchains generated with older
793 // NDKs.
794 // This code doesn't work on AArch64 because internal_getdents makes use of
795 // the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64.
796 uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY | O_DIRECTORY);
797 if (internal_iserror(fd))
798 return 0;
799 InternalMmapVector<u8> buffer(4096);
800 uptr bytes_read = buffer.size();
801 uptr n_cpus = 0;
802 u8 *d_type;
803 struct linux_dirent *entry = (struct linux_dirent *)&buffer[bytes_read];
804 while (true) {
805 if ((u8 *)entry >= &buffer[bytes_read]) {
806 bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(),
807 buffer.size());
808 if (internal_iserror(bytes_read) || !bytes_read)
809 break;
810 entry = (struct linux_dirent *)buffer.data();
811 }
812 d_type = (u8 *)entry + entry->d_reclen - 1;
813 if (d_type >= &buffer[bytes_read] ||
814 (u8 *)&entry->d_name[3] >= &buffer[bytes_read])
815 break;
816 if (entry->d_ino != 0 && *d_type == DT_DIR) {
817 if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
818 entry->d_name[2] == 'u' && entry->d_name[3] >= '0' &&
819 entry->d_name[3] <= '9')
820 n_cpus++;
821 }
822 entry = (struct linux_dirent *)(((u8 *)entry) + entry->d_reclen);
823 }
824 internal_close(fd);
825 return n_cpus;
826# elif SANITIZER_SOLARIS
827 return sysconf(_SC_NPROCESSORS_ONLN);
828# else
829 cpu_set_t CPUs;
830 CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
831 return CPU_COUNT(&CPUs);
832# endif
833}
834
835# if SANITIZER_LINUX
836
837# if SANITIZER_ANDROID
838static atomic_uint8_t android_log_initialized;
839
840void AndroidLogInit() {
841 openlog(GetProcessName(), 0, LOG_USER);
842 atomic_store(&android_log_initialized, 1, memory_order_release);
843}
844
845static bool ShouldLogAfterPrintf() {
846 return atomic_load(&android_log_initialized, memory_order_acquire);
847}
848
849extern "C" SANITIZER_WEAK_ATTRIBUTE int async_safe_write_log(int pri,
850 const char *tag,
851 const char *msg);
852extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,
853 const char *tag,
854 const char *msg);
855
856// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
857# define SANITIZER_ANDROID_LOG_INFO 4
858
859// async_safe_write_log is a new public version of __libc_write_log that is
860// used behind syslog. It is preferable to syslog as it will not do any dynamic
861// memory allocation or formatting.
862// If the function is not available, syslog is preferred for L+ (it was broken
863// pre-L) as __android_log_write triggers a racey behavior with the strncpy
864// interceptor. Fallback to __android_log_write pre-L.
865void WriteOneLineToSyslog(const char *s) {
866 if (&async_safe_write_log) {
867 async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
868 } else if (AndroidGetApiLevel() > ANDROID_KITKAT) {
869 syslog(LOG_INFO, "%s", s);
870 } else {
871 CHECK(&__android_log_write);
872 __android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s);
873 }
874}
875
876extern "C" SANITIZER_WEAK_ATTRIBUTE void android_set_abort_message(
877 const char *);
878
879void SetAbortMessage(const char *str) {
880 if (&android_set_abort_message)
881 android_set_abort_message(str);
882}
883# else
884void AndroidLogInit() {}
885
886static bool ShouldLogAfterPrintf() { return true; }
887
888void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, fmt: "%s", s); }
889
890void SetAbortMessage(const char *str) {}
891# endif // SANITIZER_ANDROID
892
893void LogMessageOnPrintf(const char *str) {
894 if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
895 WriteToSyslog(buffer: str);
896}
897
898# endif // SANITIZER_LINUX
899
900# if SANITIZER_GLIBC && !SANITIZER_GO
901// glibc crashes when using clock_gettime from a preinit_array function as the
902// vDSO function pointers haven't been initialized yet. __progname is
903// initialized after the vDSO function pointers, so if it exists, is not null
904// and is not empty, we can use clock_gettime.
905extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
906inline bool CanUseVDSO() { return &__progname && __progname && *__progname; }
907
908// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
909// clock_gettime. real_clock_gettime only exists if clock_gettime is
910// intercepted, so define it weakly and use it if available.
911extern "C" SANITIZER_WEAK_ATTRIBUTE int real_clock_gettime(u32 clk_id,
912 void *tp);
913u64 MonotonicNanoTime() {
914 timespec ts;
915 if (CanUseVDSO()) {
916 if (&real_clock_gettime)
917 real_clock_gettime(CLOCK_MONOTONIC, tp: &ts);
918 else
919 clock_gettime(CLOCK_MONOTONIC, tp: &ts);
920 } else {
921 internal_clock_gettime(CLOCK_MONOTONIC, tp: &ts);
922 }
923 return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
924}
925# else
926// Non-glibc & Go always use the regular function.
927u64 MonotonicNanoTime() {
928 timespec ts;
929 clock_gettime(CLOCK_MONOTONIC, &ts);
930 return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
931}
932# endif // SANITIZER_GLIBC && !SANITIZER_GO
933
934void ReExec() {
935 const char *pathname = "/proc/self/exe";
936
937# if SANITIZER_FREEBSD
938 for (const auto *aux = __elf_aux_vector; aux->a_type != AT_NULL; aux++) {
939 if (aux->a_type == AT_EXECPATH) {
940 pathname = static_cast<const char *>(aux->a_un.a_ptr);
941 break;
942 }
943 }
944# elif SANITIZER_NETBSD
945 static const int name[] = {
946 CTL_KERN,
947 KERN_PROC_ARGS,
948 -1,
949 KERN_PROC_PATHNAME,
950 };
951 char path[400];
952 uptr len;
953
954 len = sizeof(path);
955 if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
956 pathname = path;
957# elif SANITIZER_SOLARIS
958 pathname = getexecname();
959 CHECK_NE(pathname, NULL);
960# elif SANITIZER_USE_GETAUXVAL
961 // Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that
962 // rely on that will fail to load shared libraries. Query AT_EXECFN instead.
963 pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));
964# endif
965
966 uptr rv = internal_execve(filename: pathname, argv: GetArgv(), envp: GetEnviron());
967 int rverrno;
968 CHECK_EQ(internal_iserror(rv, &rverrno), true);
969 Printf(format: "execve failed, errno %d\n", rverrno);
970 Die();
971}
972
973void UnmapFromTo(uptr from, uptr to) {
974 if (to == from)
975 return;
976 CHECK(to >= from);
977 uptr res = internal_munmap(addr: reinterpret_cast<void *>(from), length: to - from);
978 if (UNLIKELY(internal_iserror(res))) {
979 Report(format: "ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
980 SanitizerToolName, to - from, to - from, (void *)from);
981 CHECK("unable to unmap" && 0);
982 }
983}
984
985uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
986 uptr min_shadow_base_alignment,
987 UNUSED uptr &high_mem_end) {
988 const uptr granularity = GetMmapGranularity();
989 const uptr alignment =
990 Max<uptr>(a: granularity << shadow_scale, b: 1ULL << min_shadow_base_alignment);
991 const uptr left_padding =
992 Max<uptr>(a: granularity, b: 1ULL << min_shadow_base_alignment);
993
994 const uptr shadow_size = RoundUpTo(size: shadow_size_bytes, boundary: granularity);
995 const uptr map_size = shadow_size + left_padding + alignment;
996
997 const uptr map_start = (uptr)MmapNoAccess(size: map_size);
998 CHECK_NE(map_start, ~(uptr)0);
999
1000 const uptr shadow_start = RoundUpTo(size: map_start + left_padding, boundary: alignment);
1001
1002 UnmapFromTo(from: map_start, to: shadow_start - left_padding);
1003 UnmapFromTo(from: shadow_start + shadow_size, to: map_start + map_size);
1004
1005 return shadow_start;
1006}
1007
1008static uptr MmapSharedNoReserve(uptr addr, uptr size) {
1009 return internal_mmap(
1010 addr: reinterpret_cast<void *>(addr), length: size, PROT_READ | PROT_WRITE,
1011 MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, fd: -1, offset: 0);
1012}
1013
1014static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,
1015 uptr alias_size) {
1016# if SANITIZER_LINUX
1017 return internal_mremap(old_address: reinterpret_cast<void *>(base_addr), old_size: 0, new_size: alias_size,
1018 MREMAP_MAYMOVE | MREMAP_FIXED,
1019 new_address: reinterpret_cast<void *>(alias_addr));
1020# else
1021 CHECK(false && "mremap is not supported outside of Linux");
1022 return 0;
1023# endif
1024}
1025
1026static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {
1027 uptr total_size = alias_size * num_aliases;
1028 uptr mapped = MmapSharedNoReserve(addr: start_addr, size: total_size);
1029 CHECK_EQ(mapped, start_addr);
1030
1031 for (uptr i = 1; i < num_aliases; ++i) {
1032 uptr alias_addr = start_addr + i * alias_size;
1033 CHECK_EQ(MremapCreateAlias(start_addr, alias_addr, alias_size), alias_addr);
1034 }
1035}
1036
1037uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
1038 uptr num_aliases, uptr ring_buffer_size) {
1039 CHECK_EQ(alias_size & (alias_size - 1), 0);
1040 CHECK_EQ(num_aliases & (num_aliases - 1), 0);
1041 CHECK_EQ(ring_buffer_size & (ring_buffer_size - 1), 0);
1042
1043 const uptr granularity = GetMmapGranularity();
1044 shadow_size = RoundUpTo(size: shadow_size, boundary: granularity);
1045 CHECK_EQ(shadow_size & (shadow_size - 1), 0);
1046
1047 const uptr alias_region_size = alias_size * num_aliases;
1048 const uptr alignment =
1049 2 * Max(a: Max(a: shadow_size, b: alias_region_size), b: ring_buffer_size);
1050 const uptr left_padding = ring_buffer_size;
1051
1052 const uptr right_size = alignment;
1053 const uptr map_size = left_padding + 2 * alignment;
1054
1055 const uptr map_start = reinterpret_cast<uptr>(MmapNoAccess(size: map_size));
1056 CHECK_NE(map_start, static_cast<uptr>(-1));
1057 const uptr right_start = RoundUpTo(size: map_start + left_padding, boundary: alignment);
1058
1059 UnmapFromTo(from: map_start, to: right_start - left_padding);
1060 UnmapFromTo(from: right_start + right_size, to: map_start + map_size);
1061
1062 CreateAliases(start_addr: right_start + right_size / 2, alias_size, num_aliases);
1063
1064 return right_start;
1065}
1066
1067void InitializePlatformCommonFlags(CommonFlags *cf) {
1068# if SANITIZER_ANDROID
1069 if (&__libc_get_static_tls_bounds == nullptr)
1070 cf->detect_leaks = false;
1071# endif
1072}
1073
1074} // namespace __sanitizer
1075
1076#endif
1077

source code of compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp