1//===-- tsan_interceptors_mac.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Mac-specific interceptors.
12//===----------------------------------------------------------------------===//
13
14#include "sanitizer_common/sanitizer_platform.h"
15#if SANITIZER_APPLE
16
17#include "interception/interception.h"
18#include "tsan_interceptors.h"
19#include "tsan_interface.h"
20#include "tsan_interface_ann.h"
21#include "tsan_spinlock_defs_mac.h"
22#include "sanitizer_common/sanitizer_addrhashmap.h"
23
24#include <errno.h>
25#include <libkern/OSAtomic.h>
26#include <objc/objc-sync.h>
27#include <os/lock.h>
28#include <sys/ucontext.h>
29
30#if defined(__has_include) && __has_include(<xpc/xpc.h>)
31#include <xpc/xpc.h>
32#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
33
34typedef long long_t;
35
36extern "C" {
37int getcontext(ucontext_t *ucp) __attribute__((returns_twice));
38int setcontext(const ucontext_t *ucp);
39}
40
41namespace __tsan {
42
43// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
44// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
45// actually aliases of each other, and we cannot have different interceptors for
46// them, because they're actually the same function. Thus, we have to stay
47// conservative and treat the non-barrier versions as mo_acq_rel.
48static constexpr morder kMacOrderBarrier = mo_acq_rel;
49static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
50static constexpr morder kMacFailureOrder = mo_relaxed;
51
52#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
53 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
54 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
55 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
56 }
57
58#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
59 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
60 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
61 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
62 }
63
64#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
65 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
66 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
67 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
68 }
69
70#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
71 mo) \
72 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
73 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
74 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
75 }
76
77#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
78 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
79 kMacOrderNonBarrier) \
80 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
81 kMacOrderBarrier) \
82 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
83 kMacOrderNonBarrier) \
84 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
85 kMacOrderBarrier)
86
87#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
88 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
89 kMacOrderNonBarrier) \
90 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
91 kMacOrderBarrier) \
92 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
93 kMacOrderNonBarrier) \
94 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
95 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
96
97OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
98 OSATOMIC_INTERCEPTOR_PLUS_X)
99OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
100 OSATOMIC_INTERCEPTOR_PLUS_1)
101OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
102 OSATOMIC_INTERCEPTOR_MINUS_1)
103OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
104 OSATOMIC_INTERCEPTOR)
105OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
106 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
107OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
108 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
109
110#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
111 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
112 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
113 return tsan_atomic_f##_compare_exchange_strong( \
114 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
115 kMacOrderNonBarrier, kMacFailureOrder); \
116 } \
117 \
118 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
119 t volatile *ptr) { \
120 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
121 return tsan_atomic_f##_compare_exchange_strong( \
122 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
123 kMacOrderBarrier, kMacFailureOrder); \
124 }
125
126OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
127OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
128 long_t)
129OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
130 void *)
131OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
132 int32_t)
133OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
134 int64_t)
135
136#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
137 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
138 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
139 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \
140 char bit = 0x80u >> (n & 7); \
141 char mask = clear ? ~bit : bit; \
142 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
143 return orig_byte & bit; \
144 }
145
146#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
147 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
148 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
149
150OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
151OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
152 true)
153
154TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
155 size_t offset) {
156 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
157 __tsan_release(item);
158 REAL(OSAtomicEnqueue)(list, item, offset);
159}
160
161TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
162 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
163 void *item = REAL(OSAtomicDequeue)(list, offset);
164 if (item) __tsan_acquire(item);
165 return item;
166}
167
168// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
169#if !SANITIZER_IOS
170
171TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
172 size_t offset) {
173 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
174 __tsan_release(item);
175 REAL(OSAtomicFifoEnqueue)(list, item, offset);
176}
177
178TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
179 size_t offset) {
180 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
181 void *item = REAL(OSAtomicFifoDequeue)(list, offset);
182 if (item) __tsan_acquire(item);
183 return item;
184}
185
186#endif
187
188TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
189 CHECK(!cur_thread()->is_dead);
190 if (!cur_thread()->is_inited) {
191 return REAL(OSSpinLockLock)(lock);
192 }
193 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
194 REAL(OSSpinLockLock)(lock);
195 Acquire(thr, pc, (uptr)lock);
196}
197
198TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
199 CHECK(!cur_thread()->is_dead);
200 if (!cur_thread()->is_inited) {
201 return REAL(OSSpinLockTry)(lock);
202 }
203 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
204 bool result = REAL(OSSpinLockTry)(lock);
205 if (result)
206 Acquire(thr, pc, (uptr)lock);
207 return result;
208}
209
210TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
211 CHECK(!cur_thread()->is_dead);
212 if (!cur_thread()->is_inited) {
213 return REAL(OSSpinLockUnlock)(lock);
214 }
215 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
216 Release(thr, pc, (uptr)lock);
217 REAL(OSSpinLockUnlock)(lock);
218}
219
220TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
221 CHECK(!cur_thread()->is_dead);
222 if (!cur_thread()->is_inited) {
223 return REAL(os_lock_lock)(lock);
224 }
225 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
226 REAL(os_lock_lock)(lock);
227 Acquire(thr, pc, (uptr)lock);
228}
229
230TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
231 CHECK(!cur_thread()->is_dead);
232 if (!cur_thread()->is_inited) {
233 return REAL(os_lock_trylock)(lock);
234 }
235 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
236 bool result = REAL(os_lock_trylock)(lock);
237 if (result)
238 Acquire(thr, pc, (uptr)lock);
239 return result;
240}
241
242TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
243 CHECK(!cur_thread()->is_dead);
244 if (!cur_thread()->is_inited) {
245 return REAL(os_lock_unlock)(lock);
246 }
247 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
248 Release(thr, pc, (uptr)lock);
249 REAL(os_lock_unlock)(lock);
250}
251
252TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
253 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
254 return REAL(os_unfair_lock_lock)(lock);
255 }
256 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);
257 REAL(os_unfair_lock_lock)(lock);
258 Acquire(thr, pc, (uptr)lock);
259}
260
261TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
262 u32 options) {
263 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
264 return REAL(os_unfair_lock_lock_with_options)(lock, options);
265 }
266 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);
267 REAL(os_unfair_lock_lock_with_options)(lock, options);
268 Acquire(thr, pc, (uptr)lock);
269}
270
271TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {
272 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
273 return REAL(os_unfair_lock_trylock)(lock);
274 }
275 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);
276 bool result = REAL(os_unfair_lock_trylock)(lock);
277 if (result)
278 Acquire(thr, pc, (uptr)lock);
279 return result;
280}
281
282TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
283 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
284 return REAL(os_unfair_lock_unlock)(lock);
285 }
286 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);
287 Release(thr, pc, (uptr)lock);
288 REAL(os_unfair_lock_unlock)(lock);
289}
290
291#if defined(__has_include) && __has_include(<xpc/xpc.h>)
292
293TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
294 xpc_connection_t connection, xpc_handler_t handler) {
295 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
296 handler);
297 Release(thr, pc, (uptr)connection);
298 xpc_handler_t new_handler = ^(xpc_object_t object) {
299 {
300 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
301 Acquire(thr, pc, (uptr)connection);
302 }
303 handler(object);
304 };
305 REAL(xpc_connection_set_event_handler)(connection, new_handler);
306}
307
308TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
309 dispatch_block_t barrier) {
310 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
311 Release(thr, pc, (uptr)connection);
312 dispatch_block_t new_barrier = ^() {
313 {
314 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
315 Acquire(thr, pc, (uptr)connection);
316 }
317 barrier();
318 };
319 REAL(xpc_connection_send_barrier)(connection, new_barrier);
320}
321
322TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
323 xpc_connection_t connection, xpc_object_t message,
324 dispatch_queue_t replyq, xpc_handler_t handler) {
325 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
326 message, replyq, handler);
327 Release(thr, pc, (uptr)connection);
328 xpc_handler_t new_handler = ^(xpc_object_t object) {
329 {
330 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
331 Acquire(thr, pc, (uptr)connection);
332 }
333 handler(object);
334 };
335 REAL(xpc_connection_send_message_with_reply)
336 (connection, message, replyq, new_handler);
337}
338
339TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
340 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
341 Release(thr, pc, (uptr)connection);
342 REAL(xpc_connection_cancel)(connection);
343}
344
345#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
346
347// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
348// pointers encode the object data directly in their pointer bits and do not
349// have an associated memory allocation. The Obj-C runtime uses tagged pointers
350// to transparently optimize small objects.
351static bool IsTaggedObjCPointer(id obj) {
352 const uptr kPossibleTaggedBits = 0x8000000000000001ull;
353 return ((uptr)obj & kPossibleTaggedBits) != 0;
354}
355
356// Returns an address which can be used to inform TSan about synchronization
357// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid
358// address in the process space. We do a small allocation here to obtain a
359// stable address (the array backing the hash map can change). The memory is
360// never free'd (leaked) and allocation and locking are slow, but this code only
361// runs for @synchronized with tagged pointers, which is very rare.
362static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
363 typedef AddrHashMap<uptr, 5> Map;
364 static Map Addresses;
365 Map::Handle h(&Addresses, addr);
366 if (h.created()) {
367 ThreadIgnoreBegin(thr, pc);
368 *h = (uptr) user_alloc(thr, pc, /*size=*/1);
369 ThreadIgnoreEnd(thr);
370 }
371 return *h;
372}
373
374// Returns an address on which we can synchronize given an Obj-C object pointer.
375// For normal object pointers, this is just the address of the object in memory.
376// Tagged pointers are not backed by an actual memory allocation, so we need to
377// synthesize a valid address.
378static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
379 if (IsTaggedObjCPointer(obj))
380 return GetOrCreateSyncAddress((uptr)obj, thr, pc);
381 return (uptr)obj;
382}
383
384TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
385 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
386 if (!obj) return REAL(objc_sync_enter)(obj);
387 uptr addr = SyncAddressForObjCObject(obj, thr, pc);
388 MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
389 int result = REAL(objc_sync_enter)(obj);
390 CHECK_EQ(result, OBJC_SYNC_SUCCESS);
391 MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);
392 return result;
393}
394
395TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
396 SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
397 if (!obj) return REAL(objc_sync_exit)(obj);
398 uptr addr = SyncAddressForObjCObject(obj, thr, pc);
399 MutexUnlock(thr, pc, addr);
400 int result = REAL(objc_sync_exit)(obj);
401 if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);
402 return result;
403}
404
405TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) {
406 {
407 SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
408 }
409 // Because of swapcontext() semantics we have no option but to copy its
410 // implementation here
411 if (!oucp || !ucp) {
412 errno = EINVAL;
413 return -1;
414 }
415 ThreadState *thr = cur_thread();
416 const int UCF_SWAPPED = 0x80000000;
417 oucp->uc_onstack &= ~UCF_SWAPPED;
418 thr->ignore_interceptors++;
419 int ret = getcontext(oucp);
420 if (!(oucp->uc_onstack & UCF_SWAPPED)) {
421 thr->ignore_interceptors--;
422 if (!ret) {
423 oucp->uc_onstack |= UCF_SWAPPED;
424 ret = setcontext(ucp);
425 }
426 }
427 return ret;
428}
429
430// On macOS, libc++ is always linked dynamically, so intercepting works the
431// usual way.
432#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
433
434namespace {
435struct fake_shared_weak_count {
436 volatile a64 shared_owners;
437 volatile a64 shared_weak_owners;
438 virtual void _unused_0x0() = 0;
439 virtual void _unused_0x8() = 0;
440 virtual void on_zero_shared() = 0;
441 virtual void _unused_0x18() = 0;
442 virtual void on_zero_shared_weak() = 0;
443 virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor
444};
445} // namespace
446
447// The following code adds libc++ interceptors for:
448// void __shared_weak_count::__release_shared() _NOEXCEPT;
449// bool __shared_count::__release_shared() _NOEXCEPT;
450// Shared and weak pointers in C++ maintain reference counts via atomics in
451// libc++.dylib, which are TSan-invisible, and this leads to false positives in
452// destructor code. These interceptors re-implements the whole functions so that
453// the mo_acq_rel semantics of the atomic decrement are visible.
454//
455// Unfortunately, the interceptors cannot simply Acquire/Release some sync
456// object and call the original function, because it would have a race between
457// the sync and the destruction of the object. Calling both under a lock will
458// not work because the destructor can invoke this interceptor again (and even
459// in a different thread, so recursive locks don't help).
460
461STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
462 fake_shared_weak_count *o) {
463 if (!flags()->shared_ptr_interceptor)
464 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
465
466 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
467 o);
468 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
469 Acquire(thr, pc, (uptr)&o->shared_owners);
470 o->on_zero_shared();
471 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
472 0) {
473 Acquire(thr, pc, (uptr)&o->shared_weak_owners);
474 o->on_zero_shared_weak();
475 }
476 }
477}
478
479STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
480 fake_shared_weak_count *o) {
481 if (!flags()->shared_ptr_interceptor)
482 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
483
484 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
485 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
486 Acquire(thr, pc, (uptr)&o->shared_owners);
487 o->on_zero_shared();
488 return true;
489 }
490 return false;
491}
492
493namespace {
494struct call_once_callback_args {
495 void (*orig_func)(void *arg);
496 void *orig_arg;
497 void *flag;
498};
499
500void call_once_callback_wrapper(void *arg) {
501 call_once_callback_args *new_args = (call_once_callback_args *)arg;
502 new_args->orig_func(new_args->orig_arg);
503 __tsan_release(new_args->flag);
504}
505} // namespace
506
507// This adds a libc++ interceptor for:
508// void __call_once(volatile unsigned long&, void*, void(*)(void*));
509// C++11 call_once is implemented via an internal function __call_once which is
510// inside libc++.dylib, and the atomic release store inside it is thus
511// TSan-invisible. To avoid false positives, this interceptor wraps the callback
512// function and performs an explicit Release after the user code has run.
513STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
514 void *arg, void (*func)(void *arg)) {
515 call_once_callback_args new_args = {func, arg, flag};
516 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
517 call_once_callback_wrapper);
518}
519
520} // namespace __tsan
521
522#endif // SANITIZER_APPLE
523

source code of compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp