1//===-- Implementation of tls for x86_64 ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "src/__support/OSUtil/syscall.h"
10#include "src/string/memory_utils/inline_memcpy.h"
11#include "startup/linux/do_start.h"
12
13#include <asm/prctl.h>
14#include <sys/mman.h>
15#include <sys/syscall.h>
16
17namespace LIBC_NAMESPACE {
18
19#ifdef SYS_mmap2
20static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
21#elif SYS_mmap
22static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
23#else
24#error "mmap and mmap2 syscalls not available."
25#endif
26
27// TODO: Also generalize this routine and handle dynamic loading properly.
28void init_tls(TLSDescriptor &tls_descriptor) {
29 if (app.tls.size == 0) {
30 tls_descriptor.size = 0;
31 tls_descriptor.tp = 0;
32 return;
33 }
34
35 // We will assume the alignment is always a power of two.
36 uintptr_t tls_size = app.tls.size & -app.tls.align;
37 if (tls_size != app.tls.size)
38 tls_size += app.tls.align;
39
40 // Per the x86_64 TLS ABI, the entry pointed to by the thread pointer is the
41 // address of the TLS block. So, we add more size to accomodate this address
42 // entry.
43 // We also need to include space for the stack canary. The canary is at
44 // offset 0x28 (40) and is of size uintptr_t.
45 uintptr_t tls_size_with_addr = tls_size + sizeof(uintptr_t) + 40;
46
47 // We cannot call the mmap function here as the functions set errno on
48 // failure. Since errno is implemented via a thread local variable, we cannot
49 // use errno before TLS is setup.
50 long mmap_retval = syscall_impl<long>(
51 number: MMAP_SYSCALL_NUMBER, ts: nullptr, ts: tls_size_with_addr, PROT_READ | PROT_WRITE,
52 MAP_ANONYMOUS | MAP_PRIVATE, ts: -1, ts: 0);
53 // We cannot check the return value with MAP_FAILED as that is the return
54 // of the mmap function and not the mmap syscall.
55 if (mmap_retval < 0 && static_cast<uintptr_t>(mmap_retval) > -app.page_size)
56 syscall_impl<long>(SYS_exit, ts: 1);
57 uintptr_t *tls_addr = reinterpret_cast<uintptr_t *>(mmap_retval);
58
59 // x86_64 TLS faces down from the thread pointer with the first entry
60 // pointing to the address of the first real TLS byte.
61 uintptr_t end_ptr = reinterpret_cast<uintptr_t>(tls_addr) + tls_size;
62 *reinterpret_cast<uintptr_t *>(end_ptr) = end_ptr;
63
64 inline_memcpy(dst: reinterpret_cast<char *>(tls_addr),
65 src: reinterpret_cast<const char *>(app.tls.address),
66 count: app.tls.init_size);
67 uintptr_t *stack_guard_addr = reinterpret_cast<uintptr_t *>(end_ptr + 40);
68 // Setting the stack guard to a random value.
69 // We cannot call the get_random function here as the function sets errno on
70 // failure. Since errno is implemented via a thread local variable, we cannot
71 // use errno before TLS is setup.
72 long stack_guard_retval =
73 syscall_impl(SYS_getrandom, arg1: reinterpret_cast<long>(stack_guard_addr),
74 arg2: sizeof(uint64_t), arg3: 0);
75 if (stack_guard_retval < 0)
76 syscall_impl(SYS_exit, arg1: 1);
77
78 tls_descriptor = {.size: tls_size_with_addr, .addr: reinterpret_cast<uintptr_t>(tls_addr),
79 .tp: end_ptr};
80 return;
81}
82
83void cleanup_tls(uintptr_t addr, uintptr_t size) {
84 if (size == 0)
85 return;
86 syscall_impl<long>(SYS_munmap, ts: addr, ts: size);
87}
88
89// Sets the thread pointer to |val|. Returns true on success, false on failure.
90bool set_thread_ptr(uintptr_t val) {
91 return syscall_impl(SYS_arch_prctl, ARCH_SET_FS, arg2: val) != -1;
92}
93} // namespace LIBC_NAMESPACE
94

source code of libc/startup/linux/x86_64/tls.cpp