1//===-- tsan_interceptors_libdispatch.cpp ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Support for intercepting libdispatch (GCD).
12//===----------------------------------------------------------------------===//
13
14#include "sanitizer_common/sanitizer_common.h"
15#include "interception/interception.h"
16#include "tsan_interceptors.h"
17#include "tsan_rtl.h"
18
19#include "BlocksRuntime/Block.h"
20#include "tsan_dispatch_defs.h"
21
22#if SANITIZER_APPLE
23# include <Availability.h>
24#endif
25
26namespace __tsan {
27 typedef u16 uint16_t;
28
29typedef struct {
30 dispatch_queue_t queue;
31 void *orig_context;
32 dispatch_function_t orig_work;
33 bool free_context_in_callback;
34 bool submitted_synchronously;
35 bool is_barrier_block;
36 uptr non_queue_sync_object;
37} block_context_t;
38
39// The offsets of different fields of the dispatch_queue_t structure, exported
40// by libdispatch.dylib.
41extern "C" struct dispatch_queue_offsets_s {
42 const uint16_t dqo_version;
43 const uint16_t dqo_label;
44 const uint16_t dqo_label_size;
45 const uint16_t dqo_flags;
46 const uint16_t dqo_flags_size;
47 const uint16_t dqo_serialnum;
48 const uint16_t dqo_serialnum_size;
49 const uint16_t dqo_width;
50 const uint16_t dqo_width_size;
51 const uint16_t dqo_running;
52 const uint16_t dqo_running_size;
53 const uint16_t dqo_suspend_cnt;
54 const uint16_t dqo_suspend_cnt_size;
55 const uint16_t dqo_target_queue;
56 const uint16_t dqo_target_queue_size;
57 const uint16_t dqo_priority;
58 const uint16_t dqo_priority_size;
59} dispatch_queue_offsets;
60
61static bool IsQueueSerial(dispatch_queue_t q) {
62 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
63 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
64 CHECK_NE(width, 0);
65 return width == 1;
66}
67
68static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
69 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
70 dispatch_queue_t tq = *(
71 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
72 return tq;
73}
74
75static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
76 dispatch_queue_t tq = GetTargetQueueFromQueue(q: (dispatch_queue_t)source);
77 CHECK_NE(tq, 0);
78 return tq;
79}
80
81static block_context_t *AllocContext(ThreadState *thr, uptr pc,
82 dispatch_queue_t queue, void *orig_context,
83 dispatch_function_t orig_work) {
84 block_context_t *new_context =
85 (block_context_t *)user_alloc_internal(thr, pc, sz: sizeof(block_context_t));
86 new_context->queue = queue;
87 new_context->orig_context = orig_context;
88 new_context->orig_work = orig_work;
89 new_context->free_context_in_callback = true;
90 new_context->submitted_synchronously = false;
91 new_context->is_barrier_block = false;
92 new_context->non_queue_sync_object = 0;
93 return new_context;
94}
95
96#define GET_QUEUE_SYNC_VARS(context, q) \
97 bool is_queue_serial = q && IsQueueSerial(q); \
98 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
99 uptr serial_sync = (uptr)sync_ptr; \
100 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
101 bool serial_task = context->is_barrier_block || is_queue_serial
102
103static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
104 block_context_t *context) {
105 uptr submit_sync = (uptr)context;
106 Acquire(thr, pc, addr: submit_sync);
107
108 dispatch_queue_t q = context->queue;
109 do {
110 GET_QUEUE_SYNC_VARS(context, q);
111 if (serial_sync) Acquire(thr, pc, addr: serial_sync);
112 if (serial_task && concurrent_sync) Acquire(thr, pc, addr: concurrent_sync);
113
114 if (q) q = GetTargetQueueFromQueue(q);
115 } while (q);
116}
117
118static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
119 block_context_t *context) {
120 uptr submit_sync = (uptr)context;
121 if (context->submitted_synchronously) Release(thr, pc, addr: submit_sync);
122
123 dispatch_queue_t q = context->queue;
124 do {
125 GET_QUEUE_SYNC_VARS(context, q);
126 if (serial_task && serial_sync) Release(thr, pc, addr: serial_sync);
127 if (!serial_task && concurrent_sync) Release(thr, pc, addr: concurrent_sync);
128
129 if (q) q = GetTargetQueueFromQueue(q);
130 } while (q);
131}
132
133static void dispatch_callback_wrap(void *param) {
134 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
135 block_context_t *context = (block_context_t *)param;
136
137 dispatch_sync_pre_execute(thr, pc, context);
138
139 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
140 context->orig_work(context->orig_context);
141 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
142
143 dispatch_sync_post_execute(thr, pc, context);
144
145 if (context->free_context_in_callback) user_free(thr, pc, p: context);
146}
147
148static void invoke_block(void *param) {
149 dispatch_block_t block = (dispatch_block_t)param;
150 block();
151}
152
153static void invoke_and_release_block(void *param) {
154 dispatch_block_t block = (dispatch_block_t)param;
155 block();
156 Block_release(block);
157}
158
159#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \
160 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
161 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
162 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
163 dispatch_block_t heap_block = Block_copy(block); \
164 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
165 block_context_t *new_context = \
166 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
167 new_context->is_barrier_block = barrier; \
168 Release(thr, pc, (uptr)new_context); \
169 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
170 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
171 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
172 }
173
174#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
175 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
176 DISPATCH_NOESCAPE dispatch_block_t block) { \
177 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
178 block_context_t new_context = { \
179 q, block, &invoke_block, false, true, barrier, 0}; \
180 Release(thr, pc, (uptr)&new_context); \
181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
182 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
183 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
184 Acquire(thr, pc, (uptr)&new_context); \
185 }
186
187#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \
188 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
189 dispatch_function_t work) { \
190 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
191 block_context_t *new_context = \
192 AllocContext(thr, pc, q, context, work); \
193 new_context->is_barrier_block = barrier; \
194 Release(thr, pc, (uptr)new_context); \
195 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
196 REAL(name)(q, new_context, dispatch_callback_wrap); \
197 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
198 }
199
200#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
201 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
202 dispatch_function_t work) { \
203 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
204 block_context_t new_context = { \
205 q, context, work, false, true, barrier, 0}; \
206 Release(thr, pc, (uptr)&new_context); \
207 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
208 REAL(name)(q, &new_context, dispatch_callback_wrap); \
209 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
210 Acquire(thr, pc, (uptr)&new_context); \
211 }
212
213#define DISPATCH_INTERCEPT(name, barrier) \
214 DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \
215 DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \
216 DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \
217 DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier)
218
219// We wrap dispatch_async, dispatch_sync and friends where we allocate a new
220// context, which is used to synchronize (we release the context before
221// submitting, and the callback acquires it before executing the original
222// callback).
223DISPATCH_INTERCEPT(dispatch, false)
224DISPATCH_INTERCEPT(dispatch_barrier, true)
225
226// dispatch_async_and_wait() and friends were introduced in macOS 10.14.
227// Linking of these interceptors fails when using an older SDK.
228#if !SANITIZER_APPLE || defined(__MAC_10_14)
229// macOS 10.14 is greater than our minimal deployment target. To ensure we
230// generate a weak reference so the TSan dylib continues to work on older
231// systems, we need to forward declare the intercepted functions as "weak
232// imports". Note that this file is multi-platform, so we cannot include the
233// actual header file (#include <dispatch/dispatch.h>).
234SANITIZER_WEAK_IMPORT void dispatch_async_and_wait(
235 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
236SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f(
237 dispatch_queue_t queue, void *context, dispatch_function_t work);
238SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait(
239 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
240SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f(
241 dispatch_queue_t queue, void *context, dispatch_function_t work);
242
243DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false)
244DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false)
245DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true)
246DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true)
247#endif
248
249
250DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,
251 dispatch_queue_t queue, void *context, dispatch_function_t work)
252
253TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
254 dispatch_queue_t queue, dispatch_block_t block) {
255 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
256 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
257 dispatch_block_t heap_block = Block_copy(block);
258 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
259 block_context_t *new_context =
260 AllocContext(thr, pc, queue, orig_context: heap_block, orig_work: &invoke_and_release_block);
261 Release(thr, pc, addr: (uptr)new_context);
262 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
263 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
264 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
265}
266
267TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
268 dispatch_queue_t queue, void *context,
269 dispatch_function_t work) {
270 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
271 WRAP(dispatch_after)(when, queue, block: ^(void) {
272 work(context);
273 });
274}
275
276// GCD's dispatch_once implementation has a fast path that contains a racy read
277// and it's inlined into user's code. Furthermore, this fast path doesn't
278// establish a proper happens-before relations between the initialization and
279// code following the call to dispatch_once. We could deal with this in
280// instrumented code, but there's not much we can do about it in system
281// libraries. Let's disable the fast path (by never storing the value ~0 to
282// predicate), so the interceptor is always called, and let's add proper release
283// and acquire semantics. Since TSan does not see its own atomic stores, the
284// race on predicate won't be reported - the only accesses to it that TSan sees
285// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
286// both a macro and a real function, we want to intercept the function, so we
287// need to undefine the macro.
288#undef dispatch_once
289TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
290 DISPATCH_NOESCAPE dispatch_block_t block) {
291 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
292 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
293 u32 v = atomic_load(a, mo: memory_order_acquire);
294 if (v == 0 &&
295 atomic_compare_exchange_strong(a, cmp: &v, xchg: 1, mo: memory_order_relaxed)) {
296 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
297 block();
298 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
299 Release(thr, pc, addr: (uptr)a);
300 atomic_store(a, v: 2, mo: memory_order_release);
301 } else {
302 while (v != 2) {
303 internal_sched_yield();
304 v = atomic_load(a, mo: memory_order_acquire);
305 }
306 Acquire(thr, pc, addr: (uptr)a);
307 }
308}
309
310#undef dispatch_once_f
311TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
312 void *context, dispatch_function_t function) {
313 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
314 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
315 WRAP(dispatch_once)(predicate, block: ^(void) {
316 function(context);
317 });
318 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
319}
320
321TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
322 dispatch_semaphore_t dsema) {
323 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
324 Release(thr, pc, addr: (uptr)dsema);
325 return REAL(dispatch_semaphore_signal)(dsema);
326}
327
328TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
329 dispatch_time_t timeout) {
330 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
331 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
332 if (result == 0) Acquire(thr, pc, addr: (uptr)dsema);
333 return result;
334}
335
336TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
337 dispatch_time_t timeout) {
338 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
339 long_t result = REAL(dispatch_group_wait)(group, timeout);
340 if (result == 0) Acquire(thr, pc, addr: (uptr)group);
341 return result;
342}
343
344// Used, but not intercepted.
345extern "C" void dispatch_group_enter(dispatch_group_t group);
346
347TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
348 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
349 // Acquired in the group notification callback in dispatch_group_notify[_f].
350 Release(thr, pc, addr: (uptr)group);
351 REAL(dispatch_group_leave)(group);
352}
353
354TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
355 dispatch_queue_t queue, dispatch_block_t block) {
356 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
357 dispatch_retain(object: group);
358 dispatch_group_enter(group);
359 __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block);
360 WRAP(dispatch_async)(q: queue, block: ^(void) {
361 block_copy();
362 Block_release(block_copy);
363 WRAP(dispatch_group_leave)(group);
364 dispatch_release(object: group);
365 });
366}
367
368TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
369 dispatch_queue_t queue, void *context,
370 dispatch_function_t work) {
371 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
372 dispatch_retain(object: group);
373 dispatch_group_enter(group);
374 WRAP(dispatch_async)(q: queue, block: ^(void) {
375 work(context);
376 WRAP(dispatch_group_leave)(group);
377 dispatch_release(object: group);
378 });
379}
380
381DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group,
382 dispatch_queue_t q, void *context, dispatch_function_t work)
383
384TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
385 dispatch_queue_t q, dispatch_block_t block) {
386 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
387
388 // To make sure the group is still available in the callback (otherwise
389 // it can be already destroyed). Will be released in the callback.
390 dispatch_retain(object: group);
391
392 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
393 dispatch_block_t heap_block = Block_copy(^(void) {
394 {
395 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
396 // Released when leaving the group (dispatch_group_leave).
397 Acquire(thr, pc, (uptr)group);
398 }
399 dispatch_release(group);
400 block();
401 });
402 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
403 block_context_t *new_context =
404 AllocContext(thr, pc, queue: q, orig_context: heap_block, orig_work: &invoke_and_release_block);
405 new_context->is_barrier_block = true;
406 Release(thr, pc, addr: (uptr)new_context);
407 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
408}
409
410TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
411 dispatch_queue_t q, void *context, dispatch_function_t work) {
412 WRAP(dispatch_group_notify)(group, q, block: ^(void) { work(context); });
413}
414
415TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
416 dispatch_source_t source, dispatch_block_t handler) {
417 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
418 if (handler == nullptr)
419 return REAL(dispatch_source_set_event_handler)(source, nullptr);
420 dispatch_queue_t q = GetTargetQueueFromSource(source);
421 __block block_context_t new_context = {
422 q, handler, &invoke_block, false, false, false, 0 };
423 dispatch_block_t new_handler = Block_copy(^(void) {
424 new_context.orig_context = handler; // To explicitly capture "handler".
425 dispatch_callback_wrap(&new_context);
426 });
427 uptr submit_sync = (uptr)&new_context;
428 Release(thr, pc, addr: submit_sync);
429 REAL(dispatch_source_set_event_handler)(source, new_handler);
430 Block_release(new_handler);
431}
432
433TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
434 dispatch_source_t source, dispatch_function_t handler) {
435 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
436 if (handler == nullptr)
437 return REAL(dispatch_source_set_event_handler)(source, nullptr);
438 dispatch_block_t block = ^(void) {
439 handler(dispatch_get_context(object: source));
440 };
441 WRAP(dispatch_source_set_event_handler)(source, handler: block);
442}
443
444TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
445 dispatch_source_t source, dispatch_block_t handler) {
446 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
447 if (handler == nullptr)
448 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
449 dispatch_queue_t q = GetTargetQueueFromSource(source);
450 __block block_context_t new_context = {
451 q, handler, &invoke_block, false, false, false, 0};
452 dispatch_block_t new_handler = Block_copy(^(void) {
453 new_context.orig_context = handler; // To explicitly capture "handler".
454 dispatch_callback_wrap(&new_context);
455 });
456 uptr submit_sync = (uptr)&new_context;
457 Release(thr, pc, addr: submit_sync);
458 REAL(dispatch_source_set_cancel_handler)(source, new_handler);
459 Block_release(new_handler);
460}
461
462TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
463 dispatch_source_t source, dispatch_function_t handler) {
464 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
465 handler);
466 if (handler == nullptr)
467 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
468 dispatch_block_t block = ^(void) {
469 handler(dispatch_get_context(object: source));
470 };
471 WRAP(dispatch_source_set_cancel_handler)(source, handler: block);
472}
473
474TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
475 dispatch_source_t source, dispatch_block_t handler) {
476 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
477 handler);
478 if (handler == nullptr)
479 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
480 dispatch_queue_t q = GetTargetQueueFromSource(source);
481 __block block_context_t new_context = {
482 q, handler, &invoke_block, false, false, false, 0};
483 dispatch_block_t new_handler = Block_copy(^(void) {
484 new_context.orig_context = handler; // To explicitly capture "handler".
485 dispatch_callback_wrap(&new_context);
486 });
487 uptr submit_sync = (uptr)&new_context;
488 Release(thr, pc, addr: submit_sync);
489 REAL(dispatch_source_set_registration_handler)(source, new_handler);
490 Block_release(new_handler);
491}
492
493TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
494 dispatch_source_t source, dispatch_function_t handler) {
495 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
496 handler);
497 if (handler == nullptr)
498 return REAL(dispatch_source_set_registration_handler)(source, nullptr);
499 dispatch_block_t block = ^(void) {
500 handler(dispatch_get_context(object: source));
501 };
502 WRAP(dispatch_source_set_registration_handler)(source, handler: block);
503}
504
505TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
506 dispatch_queue_t queue,
507 DISPATCH_NOESCAPE void (^block)(size_t)) {
508 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
509
510 u8 sync1, sync2;
511 uptr parent_to_child_sync = (uptr)&sync1;
512 uptr child_to_parent_sync = (uptr)&sync2;
513
514 Release(thr, pc, addr: parent_to_child_sync);
515 void (^new_block)(size_t) = ^(size_t iteration) {
516 SCOPED_INTERCEPTOR_RAW(dispatch_apply);
517 Acquire(thr, pc, addr: parent_to_child_sync);
518 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
519 block(iteration);
520 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
521 Release(thr, pc, addr: child_to_parent_sync);
522 };
523 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
524 REAL(dispatch_apply)(iterations, queue, new_block);
525 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
526 Acquire(thr, pc, addr: child_to_parent_sync);
527}
528
529static void invoke_block_iteration(void *param, size_t iteration) {
530 auto block = (void (^)(size_t)) param;
531 block(iteration);
532}
533
534TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
535 dispatch_queue_t queue, void *context,
536 void (*work)(void *, size_t)) {
537 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
538
539 // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch
540 // implements dispatch_apply in terms of dispatch_apply_f.
541 u8 sync1, sync2;
542 uptr parent_to_child_sync = (uptr)&sync1;
543 uptr child_to_parent_sync = (uptr)&sync2;
544
545 Release(thr, pc, addr: parent_to_child_sync);
546 void (^new_block)(size_t) = ^(size_t iteration) {
547 SCOPED_INTERCEPTOR_RAW(dispatch_apply_f);
548 Acquire(thr, pc, addr: parent_to_child_sync);
549 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
550 work(context, iteration);
551 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
552 Release(thr, pc, addr: child_to_parent_sync);
553 };
554 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
555 REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration);
556 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
557 Acquire(thr, pc, addr: child_to_parent_sync);
558}
559
560DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
561DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, SIZE_T sz)
562
563TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
564 size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
565 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
566 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
567 return REAL(dispatch_data_create)(buffer, size, q, destructor);
568
569 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
570 destructor = ^(void) { WRAP(free)(ptr: (void *)(uintptr_t)buffer); };
571 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
572 destructor = ^(void) { WRAP(munmap)(addr: (void *)(uintptr_t)buffer, sz: size); };
573
574 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
575 dispatch_block_t heap_block = Block_copy(destructor);
576 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
577 block_context_t *new_context =
578 AllocContext(thr, pc, queue: q, orig_context: heap_block, orig_work: &invoke_and_release_block);
579 uptr submit_sync = (uptr)new_context;
580 Release(thr, pc, addr: submit_sync);
581 return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
582 dispatch_callback_wrap(param: new_context);
583 });
584}
585
586typedef void (^fd_handler_t)(dispatch_data_t data, int error);
587typedef void (^cleanup_handler_t)(int error);
588
589TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
590 dispatch_queue_t q, fd_handler_t h) {
591 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
592 __block block_context_t new_context = {
593 q, nullptr, &invoke_block, false, false, false, 0};
594 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
595 new_context.orig_context = ^(void) {
596 h(data, error);
597 };
598 dispatch_callback_wrap(&new_context);
599 });
600 uptr submit_sync = (uptr)&new_context;
601 Release(thr, pc, addr: submit_sync);
602 REAL(dispatch_read)(fd, length, q, new_h);
603 Block_release(new_h);
604}
605
606TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
607 dispatch_queue_t q, fd_handler_t h) {
608 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
609 __block block_context_t new_context = {
610 q, nullptr, &invoke_block, false, false, false, 0};
611 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
612 new_context.orig_context = ^(void) {
613 h(data, error);
614 };
615 dispatch_callback_wrap(&new_context);
616 });
617 uptr submit_sync = (uptr)&new_context;
618 Release(thr, pc, addr: submit_sync);
619 REAL(dispatch_write)(fd, data, q, new_h);
620 Block_release(new_h);
621}
622
623TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
624 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
625 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
626 __block block_context_t new_context = {
627 q, nullptr, &invoke_block, false, false, false, 0};
628 dispatch_io_handler_t new_h =
629 Block_copy(^(bool done, dispatch_data_t data, int error) {
630 new_context.orig_context = ^(void) {
631 h(done, data, error);
632 };
633 dispatch_callback_wrap(&new_context);
634 });
635 uptr submit_sync = (uptr)&new_context;
636 Release(thr, pc, addr: submit_sync);
637 REAL(dispatch_io_read)(channel, offset, length, q, new_h);
638 Block_release(new_h);
639}
640
641TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
642 dispatch_data_t data, dispatch_queue_t q,
643 dispatch_io_handler_t h) {
644 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
645 __block block_context_t new_context = {
646 q, nullptr, &invoke_block, false, false, false, 0};
647 dispatch_io_handler_t new_h =
648 Block_copy(^(bool done, dispatch_data_t data, int error) {
649 new_context.orig_context = ^(void) {
650 h(done, data, error);
651 };
652 dispatch_callback_wrap(&new_context);
653 });
654 uptr submit_sync = (uptr)&new_context;
655 Release(thr, pc, addr: submit_sync);
656 REAL(dispatch_io_write)(channel, offset, data, q, new_h);
657 Block_release(new_h);
658}
659
660TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
661 dispatch_block_t barrier) {
662 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
663 __block block_context_t new_context = {
664 nullptr, nullptr, &invoke_block, false, false, false, 0};
665 new_context.non_queue_sync_object = (uptr)channel;
666 new_context.is_barrier_block = true;
667 dispatch_block_t new_block = Block_copy(^(void) {
668 new_context.orig_context = ^(void) {
669 barrier();
670 };
671 dispatch_callback_wrap(&new_context);
672 });
673 uptr submit_sync = (uptr)&new_context;
674 Release(thr, pc, addr: submit_sync);
675 REAL(dispatch_io_barrier)(channel, new_block);
676 Block_release(new_block);
677}
678
679TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
680 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
681 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
682 __block dispatch_io_t new_channel = nullptr;
683 __block block_context_t new_context = {
684 q, nullptr, &invoke_block, false, false, false, 0};
685 cleanup_handler_t new_h = Block_copy(^(int error) {
686 {
687 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
688 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
689 }
690 new_context.orig_context = ^(void) {
691 h(error);
692 };
693 dispatch_callback_wrap(&new_context);
694 });
695 uptr submit_sync = (uptr)&new_context;
696 Release(thr, pc, addr: submit_sync);
697 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
698 Block_release(new_h);
699 return new_channel;
700}
701
702TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
703 dispatch_io_type_t type, const char *path, int oflag,
704 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
705 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
706 q, h);
707 __block dispatch_io_t new_channel = nullptr;
708 __block block_context_t new_context = {
709 q, nullptr, &invoke_block, false, false, false, 0};
710 cleanup_handler_t new_h = Block_copy(^(int error) {
711 {
712 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
713 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
714 }
715 new_context.orig_context = ^(void) {
716 h(error);
717 };
718 dispatch_callback_wrap(&new_context);
719 });
720 uptr submit_sync = (uptr)&new_context;
721 Release(thr, pc, addr: submit_sync);
722 new_channel =
723 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
724 Block_release(new_h);
725 return new_channel;
726}
727
728TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
729 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
730 cleanup_handler_t h) {
731 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
732 __block dispatch_io_t new_channel = nullptr;
733 __block block_context_t new_context = {
734 q, nullptr, &invoke_block, false, false, false, 0};
735 cleanup_handler_t new_h = Block_copy(^(int error) {
736 {
737 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
738 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
739 }
740 new_context.orig_context = ^(void) {
741 h(error);
742 };
743 dispatch_callback_wrap(&new_context);
744 });
745 uptr submit_sync = (uptr)&new_context;
746 Release(thr, pc, addr: submit_sync);
747 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
748 Block_release(new_h);
749 return new_channel;
750}
751
752TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
753 dispatch_io_close_flags_t flags) {
754 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
755 Release(thr, pc, addr: (uptr)channel); // Acquire() in dispatch_io_create[_*].
756 return REAL(dispatch_io_close)(channel, flags);
757}
758
759// Resuming a suspended queue needs to synchronize with all subsequent
760// executions of blocks in that queue.
761TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
762 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
763 Release(thr, pc, addr: (uptr)o); // Synchronizes with the Acquire() on serial_sync
764 // in dispatch_sync_pre_execute
765 return REAL(dispatch_resume)(o);
766}
767
768void InitializeLibdispatchInterceptors() {
769 INTERCEPT_FUNCTION(dispatch_async);
770 INTERCEPT_FUNCTION(dispatch_async_f);
771 INTERCEPT_FUNCTION(dispatch_sync);
772 INTERCEPT_FUNCTION(dispatch_sync_f);
773 INTERCEPT_FUNCTION(dispatch_barrier_async);
774 INTERCEPT_FUNCTION(dispatch_barrier_async_f);
775 INTERCEPT_FUNCTION(dispatch_barrier_sync);
776 INTERCEPT_FUNCTION(dispatch_barrier_sync_f);
777 INTERCEPT_FUNCTION(dispatch_async_and_wait);
778 INTERCEPT_FUNCTION(dispatch_async_and_wait_f);
779 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait);
780 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f);
781 INTERCEPT_FUNCTION(dispatch_after);
782 INTERCEPT_FUNCTION(dispatch_after_f);
783 INTERCEPT_FUNCTION(dispatch_once);
784 INTERCEPT_FUNCTION(dispatch_once_f);
785 INTERCEPT_FUNCTION(dispatch_semaphore_signal);
786 INTERCEPT_FUNCTION(dispatch_semaphore_wait);
787 INTERCEPT_FUNCTION(dispatch_group_wait);
788 INTERCEPT_FUNCTION(dispatch_group_leave);
789 INTERCEPT_FUNCTION(dispatch_group_async);
790 INTERCEPT_FUNCTION(dispatch_group_async_f);
791 INTERCEPT_FUNCTION(dispatch_group_notify);
792 INTERCEPT_FUNCTION(dispatch_group_notify_f);
793 INTERCEPT_FUNCTION(dispatch_source_set_event_handler);
794 INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f);
795 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler);
796 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f);
797 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler);
798 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f);
799 INTERCEPT_FUNCTION(dispatch_apply);
800 INTERCEPT_FUNCTION(dispatch_apply_f);
801 INTERCEPT_FUNCTION(dispatch_data_create);
802 INTERCEPT_FUNCTION(dispatch_read);
803 INTERCEPT_FUNCTION(dispatch_write);
804 INTERCEPT_FUNCTION(dispatch_io_read);
805 INTERCEPT_FUNCTION(dispatch_io_write);
806 INTERCEPT_FUNCTION(dispatch_io_barrier);
807 INTERCEPT_FUNCTION(dispatch_io_create);
808 INTERCEPT_FUNCTION(dispatch_io_create_with_path);
809 INTERCEPT_FUNCTION(dispatch_io_create_with_io);
810 INTERCEPT_FUNCTION(dispatch_io_close);
811 INTERCEPT_FUNCTION(dispatch_resume);
812}
813
814} // namespace __tsan
815

source code of compiler-rt/lib/tsan/rtl/tsan_interceptors_libdispatch.cpp