1//===-- tsan_rtl.cpp ------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Main file (entry points) for the TSan run-time.
12//===----------------------------------------------------------------------===//
13
14#include "tsan_rtl.h"
15
16#include "sanitizer_common/sanitizer_atomic.h"
17#include "sanitizer_common/sanitizer_common.h"
18#include "sanitizer_common/sanitizer_file.h"
19#include "sanitizer_common/sanitizer_interface_internal.h"
20#include "sanitizer_common/sanitizer_libc.h"
21#include "sanitizer_common/sanitizer_placement_new.h"
22#include "sanitizer_common/sanitizer_stackdepot.h"
23#include "sanitizer_common/sanitizer_symbolizer.h"
24#include "tsan_defs.h"
25#include "tsan_interface.h"
26#include "tsan_mman.h"
27#include "tsan_platform.h"
28#include "tsan_suppressions.h"
29#include "tsan_symbolize.h"
30#include "ubsan/ubsan_init.h"
31
32volatile int __tsan_resumed = 0;
33
34extern "C" void __tsan_resume() {
35 __tsan_resumed = 1;
36}
37
38#if SANITIZER_APPLE
39SANITIZER_WEAK_DEFAULT_IMPL
40void __tsan_test_only_on_fork() {}
41#endif
42
43namespace __tsan {
44
45#if !SANITIZER_GO
46void (*on_initialize)(void);
47int (*on_finalize)(int);
48#endif
49
50#if !SANITIZER_GO && !SANITIZER_APPLE
51alignas(SANITIZER_CACHE_LINE_SIZE) THREADLOCAL __attribute__((tls_model(
52 "initial-exec"))) char cur_thread_placeholder[sizeof(ThreadState)];
53#endif
54alignas(SANITIZER_CACHE_LINE_SIZE) static char ctx_placeholder[sizeof(Context)];
55Context *ctx;
56
57// Can be overriden by a front-end.
58#ifdef TSAN_EXTERNAL_HOOKS
59bool OnFinalize(bool failed);
60void OnInitialize();
61#else
62SANITIZER_WEAK_CXX_DEFAULT_IMPL
63bool OnFinalize(bool failed) {
64# if !SANITIZER_GO
65 if (on_finalize)
66 return on_finalize(failed);
67# endif
68 return failed;
69}
70
71SANITIZER_WEAK_CXX_DEFAULT_IMPL
72void OnInitialize() {
73# if !SANITIZER_GO
74 if (on_initialize)
75 on_initialize();
76# endif
77}
78#endif
79
80static TracePart* TracePartAlloc(ThreadState* thr) {
81 TracePart* part = nullptr;
82 {
83 Lock lock(&ctx->slot_mtx);
84 uptr max_parts = Trace::kMinParts + flags()->history_size;
85 Trace* trace = &thr->tctx->trace;
86 if (trace->parts_allocated == max_parts ||
87 ctx->trace_part_finished_excess) {
88 part = ctx->trace_part_recycle.PopFront();
89 DPrintf("#%d: TracePartAlloc: part=%p\n", thr->tid, part);
90 if (part && part->trace) {
91 Trace* trace1 = part->trace;
92 Lock trace_lock(&trace1->mtx);
93 part->trace = nullptr;
94 TracePart* part1 = trace1->parts.PopFront();
95 CHECK_EQ(part, part1);
96 if (trace1->parts_allocated > trace1->parts.Size()) {
97 ctx->trace_part_finished_excess +=
98 trace1->parts_allocated - trace1->parts.Size();
99 trace1->parts_allocated = trace1->parts.Size();
100 }
101 }
102 }
103 if (trace->parts_allocated < max_parts) {
104 trace->parts_allocated++;
105 if (ctx->trace_part_finished_excess)
106 ctx->trace_part_finished_excess--;
107 }
108 if (!part)
109 ctx->trace_part_total_allocated++;
110 else if (ctx->trace_part_recycle_finished)
111 ctx->trace_part_recycle_finished--;
112 }
113 if (!part)
114 part = new (MmapOrDie(size: sizeof(*part), mem_type: "TracePart")) TracePart();
115 return part;
116}
117
118static void TracePartFree(TracePart* part) SANITIZER_REQUIRES(ctx->slot_mtx) {
119 DCHECK(part->trace);
120 part->trace = nullptr;
121 ctx->trace_part_recycle.PushFront(e: part);
122}
123
124void TraceResetForTesting() {
125 Lock lock(&ctx->slot_mtx);
126 while (auto* part = ctx->trace_part_recycle.PopFront()) {
127 if (auto trace = part->trace)
128 CHECK_EQ(trace->parts.PopFront(), part);
129 UnmapOrDie(addr: part, size: sizeof(*part));
130 }
131 ctx->trace_part_total_allocated = 0;
132 ctx->trace_part_recycle_finished = 0;
133 ctx->trace_part_finished_excess = 0;
134}
135
136static void DoResetImpl(uptr epoch) {
137 ThreadRegistryLock lock0(&ctx->thread_registry);
138 Lock lock1(&ctx->slot_mtx);
139 CHECK_EQ(ctx->global_epoch, epoch);
140 ctx->global_epoch++;
141 CHECK(!ctx->resetting);
142 ctx->resetting = true;
143 for (u32 i = ctx->thread_registry.NumThreadsLocked(); i--;) {
144 ThreadContext* tctx = (ThreadContext*)ctx->thread_registry.GetThreadLocked(
145 tid: static_cast<Tid>(i));
146 // Potentially we could purge all ThreadStatusDead threads from the
147 // registry. Since we reset all shadow, they can't race with anything
148 // anymore. However, their tid's can still be stored in some aux places
149 // (e.g. tid of thread that created something).
150 auto trace = &tctx->trace;
151 Lock lock(&trace->mtx);
152 bool attached = tctx->thr && tctx->thr->slot;
153 auto parts = &trace->parts;
154 bool local = false;
155 while (!parts->Empty()) {
156 auto part = parts->Front();
157 local = local || part == trace->local_head;
158 if (local)
159 CHECK(!ctx->trace_part_recycle.Queued(part));
160 else
161 ctx->trace_part_recycle.Remove(e: part);
162 if (attached && parts->Size() == 1) {
163 // The thread is running and this is the last/current part.
164 // Set the trace position to the end of the current part
165 // to force the thread to call SwitchTracePart and re-attach
166 // to a new slot and allocate a new trace part.
167 // Note: the thread is concurrently modifying the position as well,
168 // so this is only best-effort. The thread can only modify position
169 // within this part, because switching parts is protected by
170 // slot/trace mutexes that we hold here.
171 atomic_store_relaxed(
172 a: &tctx->thr->trace_pos,
173 v: reinterpret_cast<uptr>(&part->events[TracePart::kSize]));
174 break;
175 }
176 parts->Remove(e: part);
177 TracePartFree(part);
178 }
179 CHECK_LE(parts->Size(), 1);
180 trace->local_head = parts->Front();
181 if (tctx->thr && !tctx->thr->slot) {
182 atomic_store_relaxed(a: &tctx->thr->trace_pos, v: 0);
183 tctx->thr->trace_prev_pc = 0;
184 }
185 if (trace->parts_allocated > trace->parts.Size()) {
186 ctx->trace_part_finished_excess +=
187 trace->parts_allocated - trace->parts.Size();
188 trace->parts_allocated = trace->parts.Size();
189 }
190 }
191 while (ctx->slot_queue.PopFront()) {
192 }
193 for (auto& slot : ctx->slots) {
194 slot.SetEpoch(kEpochZero);
195 slot.journal.Reset();
196 slot.thr = nullptr;
197 ctx->slot_queue.PushBack(e: &slot);
198 }
199
200 DPrintf("Resetting shadow...\n");
201 auto shadow_begin = ShadowBeg();
202 auto shadow_end = ShadowEnd();
203#if SANITIZER_GO
204 CHECK_NE(0, ctx->mapped_shadow_begin);
205 shadow_begin = ctx->mapped_shadow_begin;
206 shadow_end = ctx->mapped_shadow_end;
207 VPrintf(2, "shadow_begin-shadow_end: (0x%zx-0x%zx)\n",
208 shadow_begin, shadow_end);
209#endif
210
211#if SANITIZER_WINDOWS
212 auto resetFailed =
213 !ZeroMmapFixedRegion(shadow_begin, shadow_end - shadow_begin);
214#else
215 auto resetFailed =
216 !MmapFixedSuperNoReserve(fixed_addr: shadow_begin, size: shadow_end-shadow_begin, name: "shadow");
217# if !SANITIZER_GO
218 DontDumpShadow(addr: shadow_begin, size: shadow_end - shadow_begin);
219# endif
220#endif
221 if (resetFailed) {
222 Printf(format: "failed to reset shadow memory\n");
223 Die();
224 }
225 DPrintf("Resetting meta shadow...\n");
226 ctx->metamap.ResetClocks();
227 StoreShadow(sp: &ctx->last_spurious_race, s: Shadow::kEmpty);
228 ctx->resetting = false;
229}
230
231// Clang does not understand locking all slots in the loop:
232// error: expecting mutex 'slot.mtx' to be held at start of each loop
233void DoReset(ThreadState* thr, uptr epoch) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
234 for (auto& slot : ctx->slots) {
235 slot.mtx.Lock();
236 if (UNLIKELY(epoch == 0))
237 epoch = ctx->global_epoch;
238 if (UNLIKELY(epoch != ctx->global_epoch)) {
239 // Epoch can't change once we've locked the first slot.
240 CHECK_EQ(slot.sid, 0);
241 slot.mtx.Unlock();
242 return;
243 }
244 }
245 DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch);
246 DoResetImpl(epoch);
247 for (auto& slot : ctx->slots) slot.mtx.Unlock();
248}
249
250void FlushShadowMemory() { DoReset(thr: nullptr, epoch: 0); }
251
252static TidSlot* FindSlotAndLock(ThreadState* thr)
253 SANITIZER_ACQUIRE(thr->slot->mtx) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
254 CHECK(!thr->slot);
255 TidSlot* slot = nullptr;
256 for (;;) {
257 uptr epoch;
258 {
259 Lock lock(&ctx->slot_mtx);
260 epoch = ctx->global_epoch;
261 if (slot) {
262 // This is an exhausted slot from the previous iteration.
263 if (ctx->slot_queue.Queued(e: slot))
264 ctx->slot_queue.Remove(e: slot);
265 thr->slot_locked = false;
266 slot->mtx.Unlock();
267 }
268 for (;;) {
269 slot = ctx->slot_queue.PopFront();
270 if (!slot)
271 break;
272 if (slot->epoch() != kEpochLast) {
273 ctx->slot_queue.PushBack(e: slot);
274 break;
275 }
276 }
277 }
278 if (!slot) {
279 DoReset(thr, epoch);
280 continue;
281 }
282 slot->mtx.Lock();
283 CHECK(!thr->slot_locked);
284 thr->slot_locked = true;
285 if (slot->thr) {
286 DPrintf("#%d: preempting sid=%d tid=%d\n", thr->tid, (u32)slot->sid,
287 slot->thr->tid);
288 slot->SetEpoch(slot->thr->fast_state.epoch());
289 slot->thr = nullptr;
290 }
291 if (slot->epoch() != kEpochLast)
292 return slot;
293 }
294}
295
296void SlotAttachAndLock(ThreadState* thr) {
297 TidSlot* slot = FindSlotAndLock(thr);
298 DPrintf("#%d: SlotAttach: slot=%u\n", thr->tid, static_cast<int>(slot->sid));
299 CHECK(!slot->thr);
300 CHECK(!thr->slot);
301 slot->thr = thr;
302 thr->slot = slot;
303 Epoch epoch = EpochInc(epoch: slot->epoch());
304 CHECK(!EpochOverflow(epoch));
305 slot->SetEpoch(epoch);
306 thr->fast_state.SetSid(slot->sid);
307 thr->fast_state.SetEpoch(epoch);
308 if (thr->slot_epoch != ctx->global_epoch) {
309 thr->slot_epoch = ctx->global_epoch;
310 thr->clock.Reset();
311#if !SANITIZER_GO
312 thr->last_sleep_stack_id = kInvalidStackID;
313 thr->last_sleep_clock.Reset();
314#endif
315 }
316 thr->clock.Set(sid: slot->sid, v: epoch);
317 slot->journal.PushBack(v: {.tid: thr->tid, .epoch: epoch});
318}
319
320static void SlotDetachImpl(ThreadState* thr, bool exiting) {
321 TidSlot* slot = thr->slot;
322 thr->slot = nullptr;
323 if (thr != slot->thr) {
324 slot = nullptr; // we don't own the slot anymore
325 if (thr->slot_epoch != ctx->global_epoch) {
326 TracePart* part = nullptr;
327 auto* trace = &thr->tctx->trace;
328 {
329 Lock l(&trace->mtx);
330 auto* parts = &trace->parts;
331 // The trace can be completely empty in an unlikely event
332 // the thread is preempted right after it acquired the slot
333 // in ThreadStart and did not trace any events yet.
334 CHECK_LE(parts->Size(), 1);
335 part = parts->PopFront();
336 thr->tctx->trace.local_head = nullptr;
337 atomic_store_relaxed(a: &thr->trace_pos, v: 0);
338 thr->trace_prev_pc = 0;
339 }
340 if (part) {
341 Lock l(&ctx->slot_mtx);
342 TracePartFree(part);
343 }
344 }
345 return;
346 }
347 CHECK(exiting || thr->fast_state.epoch() == kEpochLast);
348 slot->SetEpoch(thr->fast_state.epoch());
349 slot->thr = nullptr;
350}
351
352void SlotDetach(ThreadState* thr) {
353 Lock lock(&thr->slot->mtx);
354 SlotDetachImpl(thr, exiting: true);
355}
356
357void SlotLock(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
358 DCHECK(!thr->slot_locked);
359#if SANITIZER_DEBUG
360 // Check these mutexes are not locked.
361 // We can call DoReset from SlotAttachAndLock, which will lock
362 // these mutexes, but it happens only every once in a while.
363 { ThreadRegistryLock lock(&ctx->thread_registry); }
364 { Lock lock(&ctx->slot_mtx); }
365#endif
366 TidSlot* slot = thr->slot;
367 slot->mtx.Lock();
368 thr->slot_locked = true;
369 if (LIKELY(thr == slot->thr && thr->fast_state.epoch() != kEpochLast))
370 return;
371 SlotDetachImpl(thr, exiting: false);
372 thr->slot_locked = false;
373 slot->mtx.Unlock();
374 SlotAttachAndLock(thr);
375}
376
377void SlotUnlock(ThreadState* thr) {
378 DCHECK(thr->slot_locked);
379 thr->slot_locked = false;
380 thr->slot->mtx.Unlock();
381}
382
383Context::Context()
384 : initialized(),
385 report_mtx(MutexTypeReport),
386 nreported(),
387 thread_registry([](Tid tid) -> ThreadContextBase* {
388 return new (Alloc(sz: sizeof(ThreadContext))) ThreadContext(tid);
389 }),
390 racy_mtx(MutexTypeRacy),
391 racy_stacks(),
392 fired_suppressions_mtx(MutexTypeFired),
393 slot_mtx(MutexTypeSlots),
394 resetting() {
395 fired_suppressions.reserve(new_size: 8);
396 for (uptr i = 0; i < ARRAY_SIZE(slots); i++) {
397 TidSlot* slot = &slots[i];
398 slot->sid = static_cast<Sid>(i);
399 slot_queue.PushBack(e: slot);
400 }
401 global_epoch = 1;
402}
403
404TidSlot::TidSlot() : mtx(MutexTypeSlot) {}
405
406// The objects are allocated in TLS, so one may rely on zero-initialization.
407ThreadState::ThreadState(Tid tid)
408 // Do not touch these, rely on zero initialization,
409 // they may be accessed before the ctor.
410 // ignore_reads_and_writes()
411 // ignore_interceptors()
412 : tid(tid) {
413 CHECK_EQ(reinterpret_cast<uptr>(this) % SANITIZER_CACHE_LINE_SIZE, 0);
414#if !SANITIZER_GO
415 // C/C++ uses fixed size shadow stack.
416 const int kInitStackSize = kShadowStackSize;
417 shadow_stack = static_cast<uptr*>(
418 MmapNoReserveOrDie(size: kInitStackSize * sizeof(uptr), mem_type: "shadow stack"));
419 SetShadowRegionHugePageMode(addr: reinterpret_cast<uptr>(shadow_stack),
420 length: kInitStackSize * sizeof(uptr));
421#else
422 // Go uses malloc-allocated shadow stack with dynamic size.
423 const int kInitStackSize = 8;
424 shadow_stack = static_cast<uptr*>(Alloc(kInitStackSize * sizeof(uptr)));
425#endif
426 shadow_stack_pos = shadow_stack;
427 shadow_stack_end = shadow_stack + kInitStackSize;
428}
429
430#if !SANITIZER_GO
431void MemoryProfiler(u64 uptime) {
432 if (ctx->memprof_fd == kInvalidFd)
433 return;
434 InternalMmapVector<char> buf(4096);
435 WriteMemoryProfile(buf: buf.data(), buf_size: buf.size(), uptime_ns: uptime);
436 WriteToFile(fd: ctx->memprof_fd, buff: buf.data(), buff_size: internal_strlen(s: buf.data()));
437}
438
439static bool InitializeMemoryProfiler() {
440 ctx->memprof_fd = kInvalidFd;
441 const char *fname = flags()->profile_memory;
442 if (!fname || !fname[0])
443 return false;
444 if (internal_strcmp(s1: fname, s2: "stdout") == 0) {
445 ctx->memprof_fd = 1;
446 } else if (internal_strcmp(s1: fname, s2: "stderr") == 0) {
447 ctx->memprof_fd = 2;
448 } else {
449 InternalScopedString filename;
450 filename.AppendF(format: "%s.%d", fname, (int)internal_getpid());
451 ctx->memprof_fd = OpenFile(filename: filename.data(), mode: WrOnly);
452 if (ctx->memprof_fd == kInvalidFd) {
453 Printf(format: "ThreadSanitizer: failed to open memory profile file '%s'\n",
454 filename.data());
455 return false;
456 }
457 }
458 MemoryProfiler(uptime: 0);
459 return true;
460}
461
462static void *BackgroundThread(void *arg) {
463 // This is a non-initialized non-user thread, nothing to see here.
464 // We don't use ScopedIgnoreInterceptors, because we want ignores to be
465 // enabled even when the thread function exits (e.g. during pthread thread
466 // shutdown code).
467 cur_thread_init()->ignore_interceptors++;
468 const u64 kMs2Ns = 1000 * 1000;
469 const u64 start = NanoTime();
470
471 u64 last_flush = start;
472 uptr last_rss = 0;
473 while (!atomic_load_relaxed(a: &ctx->stop_background_thread)) {
474 SleepForMillis(millis: 100);
475 u64 now = NanoTime();
476
477 // Flush memory if requested.
478 if (flags()->flush_memory_ms > 0) {
479 if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
480 VReport(1, "ThreadSanitizer: periodic memory flush\n");
481 FlushShadowMemory();
482 now = last_flush = NanoTime();
483 }
484 }
485 if (flags()->memory_limit_mb > 0) {
486 uptr rss = GetRSS();
487 uptr limit = uptr(flags()->memory_limit_mb) << 20;
488 VReport(1,
489 "ThreadSanitizer: memory flush check"
490 " RSS=%llu LAST=%llu LIMIT=%llu\n",
491 (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
492 if (2 * rss > limit + last_rss) {
493 VReport(1, "ThreadSanitizer: flushing memory due to RSS\n");
494 FlushShadowMemory();
495 rss = GetRSS();
496 now = NanoTime();
497 VReport(1, "ThreadSanitizer: memory flushed RSS=%llu\n",
498 (u64)rss >> 20);
499 }
500 last_rss = rss;
501 }
502
503 MemoryProfiler(uptime: now - start);
504
505 // Flush symbolizer cache if requested.
506 if (flags()->flush_symbolizer_ms > 0) {
507 u64 last = atomic_load(a: &ctx->last_symbolize_time_ns,
508 mo: memory_order_relaxed);
509 if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) {
510 Lock l(&ctx->report_mtx);
511 ScopedErrorReportLock l2;
512 SymbolizeFlush();
513 atomic_store(a: &ctx->last_symbolize_time_ns, v: 0, mo: memory_order_relaxed);
514 }
515 }
516 }
517 return nullptr;
518}
519
520static void StartBackgroundThread() {
521 ctx->background_thread = internal_start_thread(func: &BackgroundThread, arg: 0);
522}
523
524#ifndef __mips__
525static void StopBackgroundThread() {
526 atomic_store(a: &ctx->stop_background_thread, v: 1, mo: memory_order_relaxed);
527 internal_join_thread(th: ctx->background_thread);
528 ctx->background_thread = 0;
529}
530#endif
531#endif
532
533void DontNeedShadowFor(uptr addr, uptr size) {
534 ReleaseMemoryPagesToOS(beg: reinterpret_cast<uptr>(MemToShadow(x: addr)),
535 end: reinterpret_cast<uptr>(MemToShadow(x: addr + size)));
536}
537
538#if !SANITIZER_GO
539// We call UnmapShadow before the actual munmap, at that point we don't yet
540// know if the provided address/size are sane. We can't call UnmapShadow
541// after the actual munmap becuase at that point the memory range can
542// already be reused for something else, so we can't rely on the munmap
543// return value to understand is the values are sane.
544// While calling munmap with insane values (non-canonical address, negative
545// size, etc) is an error, the kernel won't crash. We must also try to not
546// crash as the failure mode is very confusing (paging fault inside of the
547// runtime on some derived shadow address).
548static bool IsValidMmapRange(uptr addr, uptr size) {
549 if (size == 0)
550 return true;
551 if (static_cast<sptr>(size) < 0)
552 return false;
553 if (!IsAppMem(mem: addr) || !IsAppMem(mem: addr + size - 1))
554 return false;
555 // Check that if the start of the region belongs to one of app ranges,
556 // end of the region belongs to the same region.
557 const uptr ranges[][2] = {
558 {LoAppMemBeg(), LoAppMemEnd()},
559 {MidAppMemBeg(), MidAppMemEnd()},
560 {HiAppMemBeg(), HiAppMemEnd()},
561 };
562 for (auto range : ranges) {
563 if (addr >= range[0] && addr < range[1])
564 return addr + size <= range[1];
565 }
566 return false;
567}
568
569void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
570 if (size == 0 || !IsValidMmapRange(addr, size))
571 return;
572 DontNeedShadowFor(addr, size);
573 ScopedGlobalProcessor sgp;
574 SlotLocker locker(thr, true);
575 ctx->metamap.ResetRange(proc: thr->proc(), p: addr, sz: size, reset: true);
576}
577#endif
578
579void MapShadow(uptr addr, uptr size) {
580 // Ensure thead registry lock held, so as to synchronize
581 // with DoReset, which also access the mapped_shadow_* ctxt fields.
582 ThreadRegistryLock lock0(&ctx->thread_registry);
583 static bool data_mapped = false;
584
585#if !SANITIZER_GO
586 // Global data is not 64K aligned, but there are no adjacent mappings,
587 // so we can get away with unaligned mapping.
588 // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
589 const uptr kPageSize = GetPageSizeCached();
590 uptr shadow_begin = RoundDownTo(x: (uptr)MemToShadow(x: addr), boundary: kPageSize);
591 uptr shadow_end = RoundUpTo(size: (uptr)MemToShadow(x: addr + size), boundary: kPageSize);
592 if (!MmapFixedNoReserve(fixed_addr: shadow_begin, size: shadow_end - shadow_begin, name: "shadow"))
593 Die();
594#else
595 uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), (64 << 10));
596 uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), (64 << 10));
597 VPrintf(2, "MapShadow for (0x%zx-0x%zx), begin/end: (0x%zx-0x%zx)\n",
598 addr, addr + size, shadow_begin, shadow_end);
599
600 if (!data_mapped) {
601 // First call maps data+bss.
602 if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
603 Die();
604 } else {
605 VPrintf(2, "ctx->mapped_shadow_{begin,end} = (0x%zx-0x%zx)\n",
606 ctx->mapped_shadow_begin, ctx->mapped_shadow_end);
607 // Second and subsequent calls map heap.
608 if (shadow_end <= ctx->mapped_shadow_end)
609 return;
610 if (!ctx->mapped_shadow_begin || ctx->mapped_shadow_begin > shadow_begin)
611 ctx->mapped_shadow_begin = shadow_begin;
612 if (shadow_begin < ctx->mapped_shadow_end)
613 shadow_begin = ctx->mapped_shadow_end;
614 VPrintf(2, "MapShadow begin/end = (0x%zx-0x%zx)\n",
615 shadow_begin, shadow_end);
616 if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
617 "shadow"))
618 Die();
619 ctx->mapped_shadow_end = shadow_end;
620 }
621#endif
622
623 // Meta shadow is 2:1, so tread carefully.
624 static uptr mapped_meta_end = 0;
625 uptr meta_begin = (uptr)MemToMeta(x: addr);
626 uptr meta_end = (uptr)MemToMeta(x: addr + size);
627 meta_begin = RoundDownTo(x: meta_begin, boundary: 64 << 10);
628 meta_end = RoundUpTo(size: meta_end, boundary: 64 << 10);
629 if (!data_mapped) {
630 // First call maps data+bss.
631 data_mapped = true;
632 if (!MmapFixedSuperNoReserve(fixed_addr: meta_begin, size: meta_end - meta_begin,
633 name: "meta shadow"))
634 Die();
635 } else {
636 // Mapping continuous heap.
637 // Windows wants 64K alignment.
638 meta_begin = RoundDownTo(x: meta_begin, boundary: 64 << 10);
639 meta_end = RoundUpTo(size: meta_end, boundary: 64 << 10);
640 CHECK_GT(meta_end, mapped_meta_end);
641 if (meta_begin < mapped_meta_end)
642 meta_begin = mapped_meta_end;
643 if (!MmapFixedSuperNoReserve(fixed_addr: meta_begin, size: meta_end - meta_begin,
644 name: "meta shadow"))
645 Die();
646 mapped_meta_end = meta_end;
647 }
648 VPrintf(2, "mapped meta shadow for (0x%zx-0x%zx) at (0x%zx-0x%zx)\n", addr,
649 addr + size, meta_begin, meta_end);
650}
651
652#if !SANITIZER_GO
653static void OnStackUnwind(const SignalContext &sig, const void *,
654 BufferedStackTrace *stack) {
655 stack->Unwind(pc: StackTrace::GetNextInstructionPc(pc: sig.pc), bp: sig.bp, context: sig.context,
656 request_fast: common_flags()->fast_unwind_on_fatal);
657}
658
659static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) {
660 HandleDeadlySignal(siginfo, context, tid: GetTid(), unwind: &OnStackUnwind, unwind_context: nullptr);
661}
662#endif
663
664void CheckUnwind() {
665 // There is high probability that interceptors will check-fail as well,
666 // on the other hand there is no sense in processing interceptors
667 // since we are going to die soon.
668 ScopedIgnoreInterceptors ignore;
669#if !SANITIZER_GO
670 ThreadState* thr = cur_thread();
671 thr->nomalloc = false;
672 thr->ignore_sync++;
673 thr->ignore_reads_and_writes++;
674 atomic_store_relaxed(a: &thr->in_signal_handler, v: 0);
675#endif
676 PrintCurrentStack(pc: StackTrace::GetCurrentPc(),
677 fast: common_flags()->fast_unwind_on_fatal);
678}
679
680bool is_initialized;
681
682// Symbolization indirectly calls dl_iterate_phdr. If a CHECK() fails early on
683// (prior to the dl_iterate_phdr interceptor setup), resulting in an attempted
684// symbolization, it will segfault.
685// dl_iterate_phdr is not intercepted for Android.
686bool ready_to_symbolize = SANITIZER_ANDROID;
687
688void Initialize(ThreadState *thr) {
689 // Thread safe because done before all threads exist.
690 if (is_initialized)
691 return;
692 is_initialized = true;
693 // We are not ready to handle interceptors yet.
694 ScopedIgnoreInterceptors ignore;
695 SanitizerToolName = "ThreadSanitizer";
696 // Install tool-specific callbacks in sanitizer_common.
697 SetCheckUnwindCallback(CheckUnwind);
698
699 ctx = new(ctx_placeholder) Context;
700 const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS";
701 const char *options = GetEnv(name: env_name);
702 CacheBinaryName();
703 CheckASLR();
704 InitializeFlags(flags: &ctx->flags, env: options, env_option_name: env_name);
705 AvoidCVE_2016_2143();
706 __sanitizer::InitializePlatformEarly();
707 __tsan::InitializePlatformEarly();
708
709#if !SANITIZER_GO
710 InitializeAllocator();
711 ReplaceSystemMalloc();
712#endif
713 if (common_flags()->detect_deadlocks)
714 ctx->dd = DDetector::Create(flags: flags());
715 Processor *proc = ProcCreate();
716 ProcWire(proc, thr);
717 InitializeInterceptors();
718 InitializePlatform();
719 InitializeDynamicAnnotations();
720#if !SANITIZER_GO
721 InitializeShadowMemory();
722 InitializeAllocatorLate();
723 InstallDeadlySignalHandlers(handler: TsanOnDeadlySignal);
724#endif
725 // Setup correct file descriptor for error reports.
726 __sanitizer_set_report_path(path: common_flags()->log_path);
727 InitializeSuppressions();
728#if !SANITIZER_GO
729 InitializeLibIgnore();
730 Symbolizer::GetOrInit()->AddHooks(start_hook: EnterSymbolizer, end_hook: ExitSymbolizer);
731#endif
732
733 VPrintf(1, "***** Running under ThreadSanitizer v3 (pid %d) *****\n",
734 (int)internal_getpid());
735
736 // Initialize thread 0.
737 Tid tid = ThreadCreate(thr: nullptr, pc: 0, uid: 0, detached: true);
738 CHECK_EQ(tid, kMainTid);
739 ThreadStart(thr, tid, os_id: GetTid(), thread_type: ThreadType::Regular);
740#if TSAN_CONTAINS_UBSAN
741 __ubsan::InitAsPlugin();
742#endif
743
744#if !SANITIZER_GO
745 Symbolizer::LateInitialize();
746 if (InitializeMemoryProfiler() || flags()->force_background_thread)
747 MaybeSpawnBackgroundThread();
748#endif
749 ctx->initialized = true;
750
751 if (flags()->stop_on_start) {
752 Printf(format: "ThreadSanitizer is suspended at startup (pid %d)."
753 " Call __tsan_resume().\n",
754 (int)internal_getpid());
755 while (__tsan_resumed == 0) {}
756 }
757
758 OnInitialize();
759}
760
761void MaybeSpawnBackgroundThread() {
762 // On MIPS, TSan initialization is run before
763 // __pthread_initialize_minimal_internal() is finished, so we can not spawn
764 // new threads.
765#if !SANITIZER_GO && !defined(__mips__)
766 static atomic_uint32_t bg_thread = {};
767 if (atomic_load(a: &bg_thread, mo: memory_order_relaxed) == 0 &&
768 atomic_exchange(a: &bg_thread, v: 1, mo: memory_order_relaxed) == 0) {
769 StartBackgroundThread();
770 SetSandboxingCallback(StopBackgroundThread);
771 }
772#endif
773}
774
775int Finalize(ThreadState *thr) {
776 bool failed = false;
777
778#if !SANITIZER_GO
779 if (common_flags()->print_module_map == 1)
780 DumpProcessMap();
781#endif
782
783 if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
784 internal_usleep(useconds: u64(flags()->atexit_sleep_ms) * 1000);
785
786 {
787 // Wait for pending reports.
788 ScopedErrorReportLock lock;
789 }
790
791#if !SANITIZER_GO
792 if (Verbosity()) AllocatorPrintStats();
793#endif
794
795 ThreadFinalize(thr);
796
797 if (ctx->nreported) {
798 failed = true;
799#if !SANITIZER_GO
800 Printf(format: "ThreadSanitizer: reported %d warnings\n", ctx->nreported);
801#else
802 Printf("Found %d data race(s)\n", ctx->nreported);
803#endif
804 }
805
806 if (common_flags()->print_suppressions)
807 PrintMatchedSuppressions();
808
809 failed = OnFinalize(failed);
810
811 return failed ? common_flags()->exitcode : 0;
812}
813
814#if !SANITIZER_GO
815void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
816 VReport(2, "BeforeFork tid: %llu\n", GetTid());
817 GlobalProcessorLock();
818 // Detaching from the slot makes OnUserFree skip writing to the shadow.
819 // The slot will be locked so any attempts to use it will deadlock anyway.
820 SlotDetach(thr);
821 for (auto& slot : ctx->slots) slot.mtx.Lock();
822 ctx->thread_registry.Lock();
823 ctx->slot_mtx.Lock();
824 ScopedErrorReportLock::Lock();
825 AllocatorLockBeforeFork();
826 // Suppress all reports in the pthread_atfork callbacks.
827 // Reports will deadlock on the report_mtx.
828 // We could ignore sync operations as well,
829 // but so far it's unclear if it will do more good or harm.
830 // Unnecessarily ignoring things can lead to false positives later.
831 thr->suppress_reports++;
832 // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
833 // we'll assert in CheckNoLocks() unless we ignore interceptors.
834 // On OS X libSystem_atfork_prepare/parent/child callbacks are called
835 // after/before our callbacks and they call free.
836 thr->ignore_interceptors++;
837 // Disables memory write in OnUserAlloc/Free.
838 thr->ignore_reads_and_writes++;
839
840# if SANITIZER_APPLE
841 __tsan_test_only_on_fork();
842# endif
843}
844
845static void ForkAfter(ThreadState* thr,
846 bool child) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
847 thr->suppress_reports--; // Enabled in ForkBefore.
848 thr->ignore_interceptors--;
849 thr->ignore_reads_and_writes--;
850 AllocatorUnlockAfterFork(child);
851 ScopedErrorReportLock::Unlock();
852 ctx->slot_mtx.Unlock();
853 ctx->thread_registry.Unlock();
854 for (auto& slot : ctx->slots) slot.mtx.Unlock();
855 SlotAttachAndLock(thr);
856 SlotUnlock(thr);
857 GlobalProcessorUnlock();
858 VReport(2, "AfterFork tid: %llu\n", GetTid());
859}
860
861void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, child: false); }
862
863void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
864 ForkAfter(thr, child: true);
865 u32 nthread = ctx->thread_registry.OnFork(tid: thr->tid);
866 VPrintf(1,
867 "ThreadSanitizer: forked new process with pid %d,"
868 " parent had %d threads\n",
869 (int)internal_getpid(), (int)nthread);
870 if (nthread == 1) {
871 if (start_thread)
872 StartBackgroundThread();
873 } else {
874 // We've just forked a multi-threaded process. We cannot reasonably function
875 // after that (some mutexes may be locked before fork). So just enable
876 // ignores for everything in the hope that we will exec soon.
877 ctx->after_multithreaded_fork = true;
878 thr->ignore_interceptors++;
879 thr->suppress_reports++;
880 ThreadIgnoreBegin(thr, pc);
881 ThreadIgnoreSyncBegin(thr, pc);
882 }
883}
884#endif
885
886#if SANITIZER_GO
887NOINLINE
888void GrowShadowStack(ThreadState *thr) {
889 const int sz = thr->shadow_stack_end - thr->shadow_stack;
890 const int newsz = 2 * sz;
891 auto *newstack = (uptr *)Alloc(newsz * sizeof(uptr));
892 internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
893 Free(thr->shadow_stack);
894 thr->shadow_stack = newstack;
895 thr->shadow_stack_pos = newstack + sz;
896 thr->shadow_stack_end = newstack + newsz;
897}
898#endif
899
900StackID CurrentStackId(ThreadState *thr, uptr pc) {
901#if !SANITIZER_GO
902 if (!thr->is_inited) // May happen during bootstrap.
903 return kInvalidStackID;
904#endif
905 if (pc != 0) {
906#if !SANITIZER_GO
907 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
908#else
909 if (thr->shadow_stack_pos == thr->shadow_stack_end)
910 GrowShadowStack(thr);
911#endif
912 thr->shadow_stack_pos[0] = pc;
913 thr->shadow_stack_pos++;
914 }
915 StackID id = StackDepotPut(
916 stack: StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
917 if (pc != 0)
918 thr->shadow_stack_pos--;
919 return id;
920}
921
922static bool TraceSkipGap(ThreadState* thr) {
923 Trace *trace = &thr->tctx->trace;
924 Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(a: &thr->trace_pos));
925 DCHECK_EQ(reinterpret_cast<uptr>(pos + 1) & TracePart::kAlignment, 0);
926 auto *part = trace->parts.Back();
927 DPrintf("#%d: TraceSwitchPart enter trace=%p parts=%p-%p pos=%p\n", thr->tid,
928 trace, trace->parts.Front(), part, pos);
929 if (!part)
930 return false;
931 // We can get here when we still have space in the current trace part.
932 // The fast-path check in TraceAcquire has false positives in the middle of
933 // the part. Check if we are indeed at the end of the current part or not,
934 // and fill any gaps with NopEvent's.
935 Event* end = &part->events[TracePart::kSize];
936 DCHECK_GE(pos, &part->events[0]);
937 DCHECK_LE(pos, end);
938 if (pos + 1 < end) {
939 if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
940 TracePart::kAlignment)
941 *pos++ = NopEvent;
942 *pos++ = NopEvent;
943 DCHECK_LE(pos + 2, end);
944 atomic_store_relaxed(a: &thr->trace_pos, v: reinterpret_cast<uptr>(pos));
945 return true;
946 }
947 // We are indeed at the end.
948 for (; pos < end; pos++) *pos = NopEvent;
949 return false;
950}
951
952NOINLINE
953void TraceSwitchPart(ThreadState* thr) {
954 if (TraceSkipGap(thr))
955 return;
956#if !SANITIZER_GO
957 if (ctx->after_multithreaded_fork) {
958 // We just need to survive till exec.
959 TracePart* part = thr->tctx->trace.parts.Back();
960 if (part) {
961 atomic_store_relaxed(a: &thr->trace_pos,
962 v: reinterpret_cast<uptr>(&part->events[0]));
963 return;
964 }
965 }
966#endif
967 TraceSwitchPartImpl(thr);
968}
969
970void TraceSwitchPartImpl(ThreadState* thr) {
971 SlotLocker locker(thr, true);
972 Trace* trace = &thr->tctx->trace;
973 TracePart* part = TracePartAlloc(thr);
974 part->trace = trace;
975 thr->trace_prev_pc = 0;
976 TracePart* recycle = nullptr;
977 // Keep roughly half of parts local to the thread
978 // (not queued into the recycle queue).
979 uptr local_parts = (Trace::kMinParts + flags()->history_size + 1) / 2;
980 {
981 Lock lock(&trace->mtx);
982 if (trace->parts.Empty())
983 trace->local_head = part;
984 if (trace->parts.Size() >= local_parts) {
985 recycle = trace->local_head;
986 trace->local_head = trace->parts.Next(e: recycle);
987 }
988 trace->parts.PushBack(e: part);
989 atomic_store_relaxed(a: &thr->trace_pos,
990 v: reinterpret_cast<uptr>(&part->events[0]));
991 }
992 // Make this part self-sufficient by restoring the current stack
993 // and mutex set in the beginning of the trace.
994 TraceTime(thr);
995 {
996 // Pathologically large stacks may not fit into the part.
997 // In these cases we log only fixed number of top frames.
998 const uptr kMaxFrames = 1000;
999 // Check that kMaxFrames won't consume the whole part.
1000 static_assert(kMaxFrames < TracePart::kSize / 2, "kMaxFrames is too big");
1001 uptr* pos = Max(a: &thr->shadow_stack[0], b: thr->shadow_stack_pos - kMaxFrames);
1002 for (; pos < thr->shadow_stack_pos; pos++) {
1003 if (TryTraceFunc(thr, pc: *pos))
1004 continue;
1005 CHECK(TraceSkipGap(thr));
1006 CHECK(TryTraceFunc(thr, *pos));
1007 }
1008 }
1009 for (uptr i = 0; i < thr->mset.Size(); i++) {
1010 MutexSet::Desc d = thr->mset.Get(i);
1011 for (uptr i = 0; i < d.count; i++)
1012 TraceMutexLock(thr, type: d.write ? EventType::kLock : EventType::kRLock, pc: 0,
1013 addr: d.addr, stk: d.stack_id);
1014 }
1015 // Callers of TraceSwitchPart expect that TraceAcquire will always succeed
1016 // after the call. It's possible that TryTraceFunc/TraceMutexLock above
1017 // filled the trace part exactly up to the TracePart::kAlignment gap
1018 // and the next TraceAcquire won't succeed. Skip the gap to avoid that.
1019 EventFunc *ev;
1020 if (!TraceAcquire(thr, ev: &ev)) {
1021 CHECK(TraceSkipGap(thr));
1022 CHECK(TraceAcquire(thr, &ev));
1023 }
1024 {
1025 Lock lock(&ctx->slot_mtx);
1026 // There is a small chance that the slot may be not queued at this point.
1027 // This can happen if the slot has kEpochLast epoch and another thread
1028 // in FindSlotAndLock discovered that it's exhausted and removed it from
1029 // the slot queue. kEpochLast can happen in 2 cases: (1) if TraceSwitchPart
1030 // was called with the slot locked and epoch already at kEpochLast,
1031 // or (2) if we've acquired a new slot in SlotLock in the beginning
1032 // of the function and the slot was at kEpochLast - 1, so after increment
1033 // in SlotAttachAndLock it become kEpochLast.
1034 if (ctx->slot_queue.Queued(e: thr->slot)) {
1035 ctx->slot_queue.Remove(e: thr->slot);
1036 ctx->slot_queue.PushBack(e: thr->slot);
1037 }
1038 if (recycle)
1039 ctx->trace_part_recycle.PushBack(e: recycle);
1040 }
1041 DPrintf("#%d: TraceSwitchPart exit parts=%p-%p pos=0x%zx\n", thr->tid,
1042 trace->parts.Front(), trace->parts.Back(),
1043 atomic_load_relaxed(&thr->trace_pos));
1044}
1045
1046void ThreadIgnoreBegin(ThreadState* thr, uptr pc) {
1047 DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
1048 thr->ignore_reads_and_writes++;
1049 CHECK_GT(thr->ignore_reads_and_writes, 0);
1050 thr->fast_state.SetIgnoreBit();
1051#if !SANITIZER_GO
1052 if (pc && !ctx->after_multithreaded_fork)
1053 thr->mop_ignore_set.Add(stack_id: CurrentStackId(thr, pc));
1054#endif
1055}
1056
1057void ThreadIgnoreEnd(ThreadState *thr) {
1058 DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
1059 CHECK_GT(thr->ignore_reads_and_writes, 0);
1060 thr->ignore_reads_and_writes--;
1061 if (thr->ignore_reads_and_writes == 0) {
1062 thr->fast_state.ClearIgnoreBit();
1063#if !SANITIZER_GO
1064 thr->mop_ignore_set.Reset();
1065#endif
1066 }
1067}
1068
1069#if !SANITIZER_GO
1070extern "C" SANITIZER_INTERFACE_ATTRIBUTE
1071uptr __tsan_testonly_shadow_stack_current_size() {
1072 ThreadState *thr = cur_thread();
1073 return thr->shadow_stack_pos - thr->shadow_stack;
1074}
1075#endif
1076
1077void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
1078 DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
1079 thr->ignore_sync++;
1080 CHECK_GT(thr->ignore_sync, 0);
1081#if !SANITIZER_GO
1082 if (pc && !ctx->after_multithreaded_fork)
1083 thr->sync_ignore_set.Add(stack_id: CurrentStackId(thr, pc));
1084#endif
1085}
1086
1087void ThreadIgnoreSyncEnd(ThreadState *thr) {
1088 DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
1089 CHECK_GT(thr->ignore_sync, 0);
1090 thr->ignore_sync--;
1091#if !SANITIZER_GO
1092 if (thr->ignore_sync == 0)
1093 thr->sync_ignore_set.Reset();
1094#endif
1095}
1096
1097bool MD5Hash::operator==(const MD5Hash &other) const {
1098 return hash[0] == other.hash[0] && hash[1] == other.hash[1];
1099}
1100
1101#if SANITIZER_DEBUG
1102void build_consistency_debug() {}
1103#else
1104void build_consistency_release() {}
1105#endif
1106} // namespace __tsan
1107
1108#if SANITIZER_CHECK_DEADLOCKS
1109namespace __sanitizer {
1110using namespace __tsan;
1111MutexMeta mutex_meta[] = {
1112 {MutexInvalid, "Invalid", {}},
1113 {MutexThreadRegistry,
1114 "ThreadRegistry",
1115 {MutexTypeSlots, MutexTypeTrace, MutexTypeReport}},
1116 {MutexTypeReport, "Report", {MutexTypeTrace}},
1117 {MutexTypeSyncVar, "SyncVar", {MutexTypeReport, MutexTypeTrace}},
1118 {MutexTypeAnnotations, "Annotations", {}},
1119 {MutexTypeAtExit, "AtExit", {}},
1120 {MutexTypeFired, "Fired", {MutexLeaf}},
1121 {MutexTypeRacy, "Racy", {MutexLeaf}},
1122 {MutexTypeGlobalProc, "GlobalProc", {MutexTypeSlot, MutexTypeSlots}},
1123 {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
1124 {MutexTypeTrace, "Trace", {}},
1125 {MutexTypeSlot,
1126 "Slot",
1127 {MutexMulti, MutexTypeTrace, MutexTypeSyncVar, MutexThreadRegistry,
1128 MutexTypeSlots}},
1129 {MutexTypeSlots, "Slots", {MutexTypeTrace, MutexTypeReport}},
1130 {},
1131};
1132
1133void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
1134
1135} // namespace __sanitizer
1136#endif
1137

Provided by KDAB

Privacy Policy
Update your C++ knowledge – Modern C++11/14/17 Training
Find out more

source code of compiler-rt/lib/tsan/rtl/tsan_rtl.cpp