1//===-- memprof_rtl.cpp --------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemProfiler, a memory profiler.
10//
11// Main file of the MemProf run-time library.
12//===----------------------------------------------------------------------===//
13
14#include "memprof_allocator.h"
15#include "memprof_interceptors.h"
16#include "memprof_interface_internal.h"
17#include "memprof_internal.h"
18#include "memprof_mapping.h"
19#include "memprof_stack.h"
20#include "memprof_stats.h"
21#include "memprof_thread.h"
22#include "sanitizer_common/sanitizer_atomic.h"
23#include "sanitizer_common/sanitizer_flags.h"
24#include "sanitizer_common/sanitizer_interface_internal.h"
25#include "sanitizer_common/sanitizer_libc.h"
26#include "sanitizer_common/sanitizer_symbolizer.h"
27
28#include <time.h>
29
30uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol.
31
32// Allow the user to specify a profile output file via the binary.
33SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1];
34
35namespace __memprof {
36
37static void MemprofDie() {
38 static atomic_uint32_t num_calls;
39 if (atomic_fetch_add(a: &num_calls, v: 1, mo: memory_order_relaxed) != 0) {
40 // Don't die twice - run a busy loop.
41 while (1) {
42 internal_sched_yield();
43 }
44 }
45 if (common_flags()->print_module_map >= 1)
46 DumpProcessMap();
47 if (flags()->unmap_shadow_on_exit) {
48 if (kHighShadowEnd)
49 UnmapOrDie(addr: (void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
50 }
51}
52
53static void MemprofOnDeadlySignal(int signo, void *siginfo, void *context) {
54 // We call StartReportDeadlySignal not HandleDeadlySignal so we get the
55 // deadly signal message to stderr but no writing to the profile output file
56 StartReportDeadlySignal();
57 __memprof_profile_dump();
58 Die();
59}
60
61static void CheckUnwind() {
62 GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check);
63 stack.Print();
64}
65
66// -------------------------- Globals --------------------- {{{1
67int memprof_inited;
68bool memprof_init_is_running;
69int memprof_timestamp_inited;
70long memprof_init_timestamp_s;
71
72uptr kHighMemEnd;
73
74// -------------------------- Run-time entry ------------------- {{{1
75// exported functions
76
77#define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr);
78
79#define MEMPROF_MEMORY_ACCESS_CALLBACK(type) \
80 extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) { \
81 MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() \
82 }
83
84MEMPROF_MEMORY_ACCESS_CALLBACK(load)
85MEMPROF_MEMORY_ACCESS_CALLBACK(store)
86
87// Force the linker to keep the symbols for various MemProf interface
88// functions. We want to keep those in the executable in order to let the
89// instrumented dynamic libraries access the symbol even if it is not used by
90// the executable itself. This should help if the build system is removing dead
91// code at link time.
92static NOINLINE void force_interface_symbols() {
93 volatile int fake_condition = 0; // prevent dead condition elimination.
94 // clang-format off
95 switch (fake_condition) {
96 case 1: __memprof_record_access(addr: nullptr); break;
97 case 2: __memprof_record_access_range(addr: nullptr, size: 0); break;
98 }
99 // clang-format on
100}
101
102static void memprof_atexit() {
103 Printf(format: "MemProfiler exit stats:\n");
104 __memprof_print_accumulated_stats();
105}
106
107static void InitializeHighMemEnd() {
108 kHighMemEnd = GetMaxUserVirtualAddress();
109 // Increase kHighMemEnd to make sure it's properly
110 // aligned together with kHighMemBeg:
111 kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
112}
113
114void PrintAddressSpaceLayout() {
115 if (kHighMemBeg) {
116 Printf(format: "|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemBeg,
117 (void *)kHighMemEnd);
118 Printf(format: "|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg,
119 (void *)kHighShadowEnd);
120 }
121 Printf(format: "|| `[%p, %p]` || ShadowGap ||\n", (void *)kShadowGapBeg,
122 (void *)kShadowGapEnd);
123 if (kLowShadowBeg) {
124 Printf(format: "|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowBeg,
125 (void *)kLowShadowEnd);
126 Printf(format: "|| `[%p, %p]` || LowMem ||\n", (void *)kLowMemBeg,
127 (void *)kLowMemEnd);
128 }
129 Printf(format: "MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg),
130 (void *)MEM_TO_SHADOW(kLowShadowEnd));
131 if (kHighMemBeg) {
132 Printf(format: " %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg),
133 (void *)MEM_TO_SHADOW(kHighShadowEnd));
134 }
135 Printf(format: "\n");
136 Printf(format: "malloc_context_size=%zu\n",
137 (uptr)common_flags()->malloc_context_size);
138
139 Printf(format: "SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
140 Printf(format: "SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
141 Printf(format: "SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
142 CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
143}
144
145static void MemprofInitInternal() {
146 if (LIKELY(memprof_inited))
147 return;
148 SanitizerToolName = "MemProfiler";
149 CHECK(!memprof_init_is_running && "MemProf init calls itself!");
150 memprof_init_is_running = true;
151
152 CacheBinaryName();
153
154 // Initialize flags. This must be done early, because most of the
155 // initialization steps look at flags().
156 InitializeFlags();
157
158 AvoidCVE_2016_2143();
159
160 SetMallocContextSize(common_flags()->malloc_context_size);
161
162 InitializeHighMemEnd();
163
164 // Make sure we are not statically linked.
165 __interception::DoesNotSupportStaticLinking();
166
167 // Install tool-specific callbacks in sanitizer_common.
168 AddDieCallback(callback: MemprofDie);
169 SetCheckUnwindCallback(CheckUnwind);
170
171 // Use profile name specified via the binary itself if it exists, and hasn't
172 // been overrriden by a flag at runtime.
173 if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path)
174 __sanitizer_set_report_path(path: __memprof_profile_filename);
175 else
176 __sanitizer_set_report_path(path: common_flags()->log_path);
177
178 __sanitizer::InitializePlatformEarly();
179
180 // Setup internal allocator callback.
181 SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
182
183 InitializeMemprofInterceptors();
184 CheckASLR();
185
186 ReplaceSystemMalloc();
187
188 DisableCoreDumperIfNecessary();
189
190 InitializeShadowMemory();
191
192 TSDInit(destructor: PlatformTSDDtor);
193 InstallDeadlySignalHandlers(handler: MemprofOnDeadlySignal);
194
195 InitializeAllocator();
196
197 if (flags()->atexit)
198 Atexit(function: memprof_atexit);
199
200 InitializeCoverage(enabled: common_flags()->coverage, coverage_dir: common_flags()->coverage_dir);
201
202 // interceptors
203 InitTlsSize();
204
205 // Create main thread.
206 MemprofThread *main_thread = CreateMainThread();
207 CHECK_EQ(0, main_thread->tid());
208 force_interface_symbols(); // no-op.
209 SanitizerInitializeUnwinder();
210
211 Symbolizer::LateInitialize();
212
213 VReport(1, "MemProfiler Init done\n");
214
215 memprof_init_is_running = false;
216 memprof_inited = 1;
217}
218
219void MemprofInitTime() {
220 if (LIKELY(memprof_timestamp_inited))
221 return;
222 timespec ts;
223 clock_gettime(CLOCK_REALTIME, tp: &ts);
224 memprof_init_timestamp_s = ts.tv_sec;
225 memprof_timestamp_inited = 1;
226}
227
228// Initialize as requested from some part of MemProf runtime library
229// (interceptors, allocator, etc).
230void MemprofInitFromRtl() { MemprofInitInternal(); }
231
232#if MEMPROF_DYNAMIC
233// Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable
234// (and thus normal initializers from .preinit_array or modules haven't run).
235
236class MemprofInitializer {
237public:
238 MemprofInitializer() { MemprofInitFromRtl(); }
239};
240
241static MemprofInitializer memprof_initializer;
242#endif // MEMPROF_DYNAMIC
243
244} // namespace __memprof
245
246// ---------------------- Interface ---------------- {{{1
247using namespace __memprof;
248
249// Initialize as requested from instrumented application code.
250void __memprof_init() {
251 MemprofInitTime();
252 MemprofInitInternal();
253}
254
255void __memprof_preinit() { MemprofInitInternal(); }
256
257void __memprof_version_mismatch_check_v1() {}
258
259void __memprof_record_access(void const volatile *addr) {
260 __memprof::RecordAccess(a: (uptr)addr);
261}
262
263void __memprof_record_access_range(void const volatile *addr, uptr size) {
264 for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize)
265 __memprof::RecordAccess(a);
266}
267
268extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16
269__sanitizer_unaligned_load16(const uu16 *p) {
270 __memprof_record_access(addr: p);
271 return *p;
272}
273
274extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32
275__sanitizer_unaligned_load32(const uu32 *p) {
276 __memprof_record_access(addr: p);
277 return *p;
278}
279
280extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
281__sanitizer_unaligned_load64(const uu64 *p) {
282 __memprof_record_access(addr: p);
283 return *p;
284}
285
286extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
287__sanitizer_unaligned_store16(uu16 *p, u16 x) {
288 __memprof_record_access(addr: p);
289 *p = x;
290}
291
292extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
293__sanitizer_unaligned_store32(uu32 *p, u32 x) {
294 __memprof_record_access(addr: p);
295 *p = x;
296}
297
298extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
299__sanitizer_unaligned_store64(uu64 *p, u64 x) {
300 __memprof_record_access(addr: p);
301 *p = x;
302}
303

source code of compiler-rt/lib/memprof/memprof_rtl.cpp