1 //===-- tsan_interceptors_libdispatch.cpp ---------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Support for intercepting libdispatch (GCD). 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_common/sanitizer_common.h" 15 #include "interception/interception.h" 16 #include "tsan_interceptors.h" 17 #include "tsan_rtl.h" 18 19 #include "BlocksRuntime/Block.h" 20 #include "tsan_dispatch_defs.h" 21 22 #if SANITIZER_APPLE 23 # include <Availability.h> 24 #endif 25 26 namespace __tsan { 27 typedef u16 uint16_t; 28 29 typedef struct { 30 dispatch_queue_t queue; 31 void *orig_context; 32 dispatch_function_t orig_work; 33 bool free_context_in_callback; 34 bool submitted_synchronously; 35 bool is_barrier_block; 36 uptr non_queue_sync_object; 37 } block_context_t; 38 39 // The offsets of different fields of the dispatch_queue_t structure, exported 40 // by libdispatch.dylib. 41 extern "C" struct dispatch_queue_offsets_s { 42 const uint16_t dqo_version; 43 const uint16_t dqo_label; 44 const uint16_t dqo_label_size; 45 const uint16_t dqo_flags; 46 const uint16_t dqo_flags_size; 47 const uint16_t dqo_serialnum; 48 const uint16_t dqo_serialnum_size; 49 const uint16_t dqo_width; 50 const uint16_t dqo_width_size; 51 const uint16_t dqo_running; 52 const uint16_t dqo_running_size; 53 const uint16_t dqo_suspend_cnt; 54 const uint16_t dqo_suspend_cnt_size; 55 const uint16_t dqo_target_queue; 56 const uint16_t dqo_target_queue_size; 57 const uint16_t dqo_priority; 58 const uint16_t dqo_priority_size; 59 } dispatch_queue_offsets; 60 61 static bool IsQueueSerial(dispatch_queue_t q) { 62 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); 63 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); 64 CHECK_NE(width, 0); 65 return width == 1; 66 } 67 68 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) { 69 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); 70 dispatch_queue_t tq = *( 71 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue); 72 return tq; 73 } 74 75 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { 76 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source); 77 CHECK_NE(tq, 0); 78 return tq; 79 } 80 81 static block_context_t *AllocContext(ThreadState *thr, uptr pc, 82 dispatch_queue_t queue, void *orig_context, 83 dispatch_function_t orig_work) { 84 block_context_t *new_context = 85 (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); 86 new_context->queue = queue; 87 new_context->orig_context = orig_context; 88 new_context->orig_work = orig_work; 89 new_context->free_context_in_callback = true; 90 new_context->submitted_synchronously = false; 91 new_context->is_barrier_block = false; 92 new_context->non_queue_sync_object = 0; 93 return new_context; 94 } 95 96 #define GET_QUEUE_SYNC_VARS(context, q) \ 97 bool is_queue_serial = q && IsQueueSerial(q); \ 98 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \ 99 uptr serial_sync = (uptr)sync_ptr; \ 100 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \ 101 bool serial_task = context->is_barrier_block || is_queue_serial 102 103 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, 104 block_context_t *context) { 105 uptr submit_sync = (uptr)context; 106 Acquire(thr, pc, submit_sync); 107 108 dispatch_queue_t q = context->queue; 109 do { 110 GET_QUEUE_SYNC_VARS(context, q); 111 if (serial_sync) Acquire(thr, pc, serial_sync); 112 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync); 113 114 if (q) q = GetTargetQueueFromQueue(q); 115 } while (q); 116 } 117 118 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, 119 block_context_t *context) { 120 uptr submit_sync = (uptr)context; 121 if (context->submitted_synchronously) Release(thr, pc, submit_sync); 122 123 dispatch_queue_t q = context->queue; 124 do { 125 GET_QUEUE_SYNC_VARS(context, q); 126 if (serial_task && serial_sync) Release(thr, pc, serial_sync); 127 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync); 128 129 if (q) q = GetTargetQueueFromQueue(q); 130 } while (q); 131 } 132 133 static void dispatch_callback_wrap(void *param) { 134 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); 135 block_context_t *context = (block_context_t *)param; 136 137 dispatch_sync_pre_execute(thr, pc, context); 138 139 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 140 context->orig_work(context->orig_context); 141 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 142 143 dispatch_sync_post_execute(thr, pc, context); 144 145 if (context->free_context_in_callback) user_free(thr, pc, context); 146 } 147 148 static void invoke_block(void *param) { 149 dispatch_block_t block = (dispatch_block_t)param; 150 block(); 151 } 152 153 static void invoke_and_release_block(void *param) { 154 dispatch_block_t block = (dispatch_block_t)param; 155 block(); 156 Block_release(block); 157 } 158 159 #define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ 160 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 161 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 162 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 163 dispatch_block_t heap_block = Block_copy(block); \ 164 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 165 block_context_t *new_context = \ 166 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ 167 new_context->is_barrier_block = barrier; \ 168 Release(thr, pc, (uptr)new_context); \ 169 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 170 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ 171 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 172 } 173 174 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ 175 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ 176 DISPATCH_NOESCAPE dispatch_block_t block) { \ 177 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 178 block_context_t new_context = { \ 179 q, block, &invoke_block, false, true, barrier, 0}; \ 180 Release(thr, pc, (uptr)&new_context); \ 181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 182 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ 183 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 184 Acquire(thr, pc, (uptr)&new_context); \ 185 } 186 187 #define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ 188 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 189 dispatch_function_t work) { \ 190 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 191 block_context_t *new_context = \ 192 AllocContext(thr, pc, q, context, work); \ 193 new_context->is_barrier_block = barrier; \ 194 Release(thr, pc, (uptr)new_context); \ 195 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 196 REAL(name)(q, new_context, dispatch_callback_wrap); \ 197 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 198 } 199 200 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ 201 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 202 dispatch_function_t work) { \ 203 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 204 block_context_t new_context = { \ 205 q, context, work, false, true, barrier, 0}; \ 206 Release(thr, pc, (uptr)&new_context); \ 207 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 208 REAL(name)(q, &new_context, dispatch_callback_wrap); \ 209 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 210 Acquire(thr, pc, (uptr)&new_context); \ 211 } 212 213 #define DISPATCH_INTERCEPT(name, barrier) \ 214 DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ 215 DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ 216 DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ 217 DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) 218 219 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new 220 // context, which is used to synchronize (we release the context before 221 // submitting, and the callback acquires it before executing the original 222 // callback). 223 DISPATCH_INTERCEPT(dispatch, false) 224 DISPATCH_INTERCEPT(dispatch_barrier, true) 225 226 // dispatch_async_and_wait() and friends were introduced in macOS 10.14. 227 // Linking of these interceptors fails when using an older SDK. 228 #if !SANITIZER_APPLE || defined(__MAC_10_14) 229 // macOS 10.14 is greater than our minimal deployment target. To ensure we 230 // generate a weak reference so the TSan dylib continues to work on older 231 // systems, we need to forward declare the intercepted functions as "weak 232 // imports". Note that this file is multi-platform, so we cannot include the 233 // actual header file (#include <dispatch/dispatch.h>). 234 SANITIZER_WEAK_IMPORT void dispatch_async_and_wait( 235 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block); 236 SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f( 237 dispatch_queue_t queue, void *context, dispatch_function_t work); 238 SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait( 239 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block); 240 SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f( 241 dispatch_queue_t queue, void *context, dispatch_function_t work); 242 243 DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false) 244 DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false) 245 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true) 246 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true) 247 #endif 248 249 250 DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, 251 dispatch_queue_t queue, void *context, dispatch_function_t work) 252 253 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, 254 dispatch_queue_t queue, dispatch_block_t block) { 255 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); 256 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 257 dispatch_block_t heap_block = Block_copy(block); 258 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 259 block_context_t *new_context = 260 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); 261 Release(thr, pc, (uptr)new_context); 262 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 263 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); 264 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 265 } 266 267 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, 268 dispatch_queue_t queue, void *context, 269 dispatch_function_t work) { 270 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); 271 WRAP(dispatch_after)(when, queue, ^(void) { 272 work(context); 273 }); 274 } 275 276 // GCD's dispatch_once implementation has a fast path that contains a racy read 277 // and it's inlined into user's code. Furthermore, this fast path doesn't 278 // establish a proper happens-before relations between the initialization and 279 // code following the call to dispatch_once. We could deal with this in 280 // instrumented code, but there's not much we can do about it in system 281 // libraries. Let's disable the fast path (by never storing the value ~0 to 282 // predicate), so the interceptor is always called, and let's add proper release 283 // and acquire semantics. Since TSan does not see its own atomic stores, the 284 // race on predicate won't be reported - the only accesses to it that TSan sees 285 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is 286 // both a macro and a real function, we want to intercept the function, so we 287 // need to undefine the macro. 288 #undef dispatch_once 289 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, 290 DISPATCH_NOESCAPE dispatch_block_t block) { 291 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); 292 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); 293 u32 v = atomic_load(a, memory_order_acquire); 294 if (v == 0 && 295 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { 296 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 297 block(); 298 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 299 Release(thr, pc, (uptr)a); 300 atomic_store(a, 2, memory_order_release); 301 } else { 302 while (v != 2) { 303 internal_sched_yield(); 304 v = atomic_load(a, memory_order_acquire); 305 } 306 Acquire(thr, pc, (uptr)a); 307 } 308 } 309 310 #undef dispatch_once_f 311 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, 312 void *context, dispatch_function_t function) { 313 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); 314 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 315 WRAP(dispatch_once)(predicate, ^(void) { 316 function(context); 317 }); 318 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 319 } 320 321 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, 322 dispatch_semaphore_t dsema) { 323 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); 324 Release(thr, pc, (uptr)dsema); 325 return REAL(dispatch_semaphore_signal)(dsema); 326 } 327 328 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, 329 dispatch_time_t timeout) { 330 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); 331 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); 332 if (result == 0) Acquire(thr, pc, (uptr)dsema); 333 return result; 334 } 335 336 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, 337 dispatch_time_t timeout) { 338 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); 339 long_t result = REAL(dispatch_group_wait)(group, timeout); 340 if (result == 0) Acquire(thr, pc, (uptr)group); 341 return result; 342 } 343 344 // Used, but not intercepted. 345 extern "C" void dispatch_group_enter(dispatch_group_t group); 346 347 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { 348 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); 349 // Acquired in the group notification callback in dispatch_group_notify[_f]. 350 Release(thr, pc, (uptr)group); 351 REAL(dispatch_group_leave)(group); 352 } 353 354 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, 355 dispatch_queue_t queue, dispatch_block_t block) { 356 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); 357 dispatch_retain(group); 358 dispatch_group_enter(group); 359 __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); 360 WRAP(dispatch_async)(queue, ^(void) { 361 block_copy(); 362 Block_release(block_copy); 363 WRAP(dispatch_group_leave)(group); 364 dispatch_release(group); 365 }); 366 } 367 368 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, 369 dispatch_queue_t queue, void *context, 370 dispatch_function_t work) { 371 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); 372 dispatch_retain(group); 373 dispatch_group_enter(group); 374 WRAP(dispatch_async)(queue, ^(void) { 375 work(context); 376 WRAP(dispatch_group_leave)(group); 377 dispatch_release(group); 378 }); 379 } 380 381 DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, 382 dispatch_queue_t q, void *context, dispatch_function_t work) 383 384 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, 385 dispatch_queue_t q, dispatch_block_t block) { 386 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); 387 388 // To make sure the group is still available in the callback (otherwise 389 // it can be already destroyed). Will be released in the callback. 390 dispatch_retain(group); 391 392 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 393 dispatch_block_t heap_block = Block_copy(^(void) { 394 { 395 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); 396 // Released when leaving the group (dispatch_group_leave). 397 Acquire(thr, pc, (uptr)group); 398 } 399 dispatch_release(group); 400 block(); 401 }); 402 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 403 block_context_t *new_context = 404 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 405 new_context->is_barrier_block = true; 406 Release(thr, pc, (uptr)new_context); 407 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); 408 } 409 410 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, 411 dispatch_queue_t q, void *context, dispatch_function_t work) { 412 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); 413 } 414 415 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, 416 dispatch_source_t source, dispatch_block_t handler) { 417 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); 418 if (handler == nullptr) 419 return REAL(dispatch_source_set_event_handler)(source, nullptr); 420 dispatch_queue_t q = GetTargetQueueFromSource(source); 421 __block block_context_t new_context = { 422 q, handler, &invoke_block, false, false, false, 0 }; 423 dispatch_block_t new_handler = Block_copy(^(void) { 424 new_context.orig_context = handler; // To explicitly capture "handler". 425 dispatch_callback_wrap(&new_context); 426 }); 427 uptr submit_sync = (uptr)&new_context; 428 Release(thr, pc, submit_sync); 429 REAL(dispatch_source_set_event_handler)(source, new_handler); 430 Block_release(new_handler); 431 } 432 433 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, 434 dispatch_source_t source, dispatch_function_t handler) { 435 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); 436 if (handler == nullptr) 437 return REAL(dispatch_source_set_event_handler)(source, nullptr); 438 dispatch_block_t block = ^(void) { 439 handler(dispatch_get_context(source)); 440 }; 441 WRAP(dispatch_source_set_event_handler)(source, block); 442 } 443 444 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, 445 dispatch_source_t source, dispatch_block_t handler) { 446 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); 447 if (handler == nullptr) 448 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 449 dispatch_queue_t q = GetTargetQueueFromSource(source); 450 __block block_context_t new_context = { 451 q, handler, &invoke_block, false, false, false, 0}; 452 dispatch_block_t new_handler = Block_copy(^(void) { 453 new_context.orig_context = handler; // To explicitly capture "handler". 454 dispatch_callback_wrap(&new_context); 455 }); 456 uptr submit_sync = (uptr)&new_context; 457 Release(thr, pc, submit_sync); 458 REAL(dispatch_source_set_cancel_handler)(source, new_handler); 459 Block_release(new_handler); 460 } 461 462 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, 463 dispatch_source_t source, dispatch_function_t handler) { 464 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, 465 handler); 466 if (handler == nullptr) 467 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 468 dispatch_block_t block = ^(void) { 469 handler(dispatch_get_context(source)); 470 }; 471 WRAP(dispatch_source_set_cancel_handler)(source, block); 472 } 473 474 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, 475 dispatch_source_t source, dispatch_block_t handler) { 476 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, 477 handler); 478 if (handler == nullptr) 479 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 480 dispatch_queue_t q = GetTargetQueueFromSource(source); 481 __block block_context_t new_context = { 482 q, handler, &invoke_block, false, false, false, 0}; 483 dispatch_block_t new_handler = Block_copy(^(void) { 484 new_context.orig_context = handler; // To explicitly capture "handler". 485 dispatch_callback_wrap(&new_context); 486 }); 487 uptr submit_sync = (uptr)&new_context; 488 Release(thr, pc, submit_sync); 489 REAL(dispatch_source_set_registration_handler)(source, new_handler); 490 Block_release(new_handler); 491 } 492 493 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, 494 dispatch_source_t source, dispatch_function_t handler) { 495 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, 496 handler); 497 if (handler == nullptr) 498 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 499 dispatch_block_t block = ^(void) { 500 handler(dispatch_get_context(source)); 501 }; 502 WRAP(dispatch_source_set_registration_handler)(source, block); 503 } 504 505 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, 506 dispatch_queue_t queue, 507 DISPATCH_NOESCAPE void (^block)(size_t)) { 508 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); 509 510 u8 sync1, sync2; 511 uptr parent_to_child_sync = (uptr)&sync1; 512 uptr child_to_parent_sync = (uptr)&sync2; 513 514 Release(thr, pc, parent_to_child_sync); 515 void (^new_block)(size_t) = ^(size_t iteration) { 516 SCOPED_INTERCEPTOR_RAW(dispatch_apply); 517 Acquire(thr, pc, parent_to_child_sync); 518 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 519 block(iteration); 520 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 521 Release(thr, pc, child_to_parent_sync); 522 }; 523 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 524 REAL(dispatch_apply)(iterations, queue, new_block); 525 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 526 Acquire(thr, pc, child_to_parent_sync); 527 } 528 529 static void invoke_block_iteration(void *param, size_t iteration) { 530 auto block = (void (^)(size_t)) param; 531 block(iteration); 532 } 533 534 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, 535 dispatch_queue_t queue, void *context, 536 void (*work)(void *, size_t)) { 537 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); 538 539 // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch 540 // implements dispatch_apply in terms of dispatch_apply_f. 541 u8 sync1, sync2; 542 uptr parent_to_child_sync = (uptr)&sync1; 543 uptr child_to_parent_sync = (uptr)&sync2; 544 545 Release(thr, pc, parent_to_child_sync); 546 void (^new_block)(size_t) = ^(size_t iteration) { 547 SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); 548 Acquire(thr, pc, parent_to_child_sync); 549 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 550 work(context, iteration); 551 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 552 Release(thr, pc, child_to_parent_sync); 553 }; 554 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 555 REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); 556 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 557 Acquire(thr, pc, child_to_parent_sync); 558 } 559 560 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 561 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) 562 563 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, 564 size_t size, dispatch_queue_t q, dispatch_block_t destructor) { 565 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); 566 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) 567 return REAL(dispatch_data_create)(buffer, size, q, destructor); 568 569 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) 570 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; 571 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) 572 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; 573 574 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 575 dispatch_block_t heap_block = Block_copy(destructor); 576 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 577 block_context_t *new_context = 578 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 579 uptr submit_sync = (uptr)new_context; 580 Release(thr, pc, submit_sync); 581 return REAL(dispatch_data_create)(buffer, size, q, ^(void) { 582 dispatch_callback_wrap(new_context); 583 }); 584 } 585 586 typedef void (^fd_handler_t)(dispatch_data_t data, int error); 587 typedef void (^cleanup_handler_t)(int error); 588 589 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, 590 dispatch_queue_t q, fd_handler_t h) { 591 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); 592 __block block_context_t new_context = { 593 q, nullptr, &invoke_block, false, false, false, 0}; 594 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 595 new_context.orig_context = ^(void) { 596 h(data, error); 597 }; 598 dispatch_callback_wrap(&new_context); 599 }); 600 uptr submit_sync = (uptr)&new_context; 601 Release(thr, pc, submit_sync); 602 REAL(dispatch_read)(fd, length, q, new_h); 603 Block_release(new_h); 604 } 605 606 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, 607 dispatch_queue_t q, fd_handler_t h) { 608 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); 609 __block block_context_t new_context = { 610 q, nullptr, &invoke_block, false, false, false, 0}; 611 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 612 new_context.orig_context = ^(void) { 613 h(data, error); 614 }; 615 dispatch_callback_wrap(&new_context); 616 }); 617 uptr submit_sync = (uptr)&new_context; 618 Release(thr, pc, submit_sync); 619 REAL(dispatch_write)(fd, data, q, new_h); 620 Block_release(new_h); 621 } 622 623 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, 624 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { 625 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); 626 __block block_context_t new_context = { 627 q, nullptr, &invoke_block, false, false, false, 0}; 628 dispatch_io_handler_t new_h = 629 Block_copy(^(bool done, dispatch_data_t data, int error) { 630 new_context.orig_context = ^(void) { 631 h(done, data, error); 632 }; 633 dispatch_callback_wrap(&new_context); 634 }); 635 uptr submit_sync = (uptr)&new_context; 636 Release(thr, pc, submit_sync); 637 REAL(dispatch_io_read)(channel, offset, length, q, new_h); 638 Block_release(new_h); 639 } 640 641 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, 642 dispatch_data_t data, dispatch_queue_t q, 643 dispatch_io_handler_t h) { 644 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); 645 __block block_context_t new_context = { 646 q, nullptr, &invoke_block, false, false, false, 0}; 647 dispatch_io_handler_t new_h = 648 Block_copy(^(bool done, dispatch_data_t data, int error) { 649 new_context.orig_context = ^(void) { 650 h(done, data, error); 651 }; 652 dispatch_callback_wrap(&new_context); 653 }); 654 uptr submit_sync = (uptr)&new_context; 655 Release(thr, pc, submit_sync); 656 REAL(dispatch_io_write)(channel, offset, data, q, new_h); 657 Block_release(new_h); 658 } 659 660 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, 661 dispatch_block_t barrier) { 662 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); 663 __block block_context_t new_context = { 664 nullptr, nullptr, &invoke_block, false, false, false, 0}; 665 new_context.non_queue_sync_object = (uptr)channel; 666 new_context.is_barrier_block = true; 667 dispatch_block_t new_block = Block_copy(^(void) { 668 new_context.orig_context = ^(void) { 669 barrier(); 670 }; 671 dispatch_callback_wrap(&new_context); 672 }); 673 uptr submit_sync = (uptr)&new_context; 674 Release(thr, pc, submit_sync); 675 REAL(dispatch_io_barrier)(channel, new_block); 676 Block_release(new_block); 677 } 678 679 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, 680 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { 681 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); 682 __block dispatch_io_t new_channel = nullptr; 683 __block block_context_t new_context = { 684 q, nullptr, &invoke_block, false, false, false, 0}; 685 cleanup_handler_t new_h = Block_copy(^(int error) { 686 { 687 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 688 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 689 } 690 new_context.orig_context = ^(void) { 691 h(error); 692 }; 693 dispatch_callback_wrap(&new_context); 694 }); 695 uptr submit_sync = (uptr)&new_context; 696 Release(thr, pc, submit_sync); 697 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); 698 Block_release(new_h); 699 return new_channel; 700 } 701 702 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, 703 dispatch_io_type_t type, const char *path, int oflag, 704 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { 705 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, 706 q, h); 707 __block dispatch_io_t new_channel = nullptr; 708 __block block_context_t new_context = { 709 q, nullptr, &invoke_block, false, false, false, 0}; 710 cleanup_handler_t new_h = Block_copy(^(int error) { 711 { 712 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 713 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 714 } 715 new_context.orig_context = ^(void) { 716 h(error); 717 }; 718 dispatch_callback_wrap(&new_context); 719 }); 720 uptr submit_sync = (uptr)&new_context; 721 Release(thr, pc, submit_sync); 722 new_channel = 723 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); 724 Block_release(new_h); 725 return new_channel; 726 } 727 728 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, 729 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, 730 cleanup_handler_t h) { 731 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); 732 __block dispatch_io_t new_channel = nullptr; 733 __block block_context_t new_context = { 734 q, nullptr, &invoke_block, false, false, false, 0}; 735 cleanup_handler_t new_h = Block_copy(^(int error) { 736 { 737 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 738 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 739 } 740 new_context.orig_context = ^(void) { 741 h(error); 742 }; 743 dispatch_callback_wrap(&new_context); 744 }); 745 uptr submit_sync = (uptr)&new_context; 746 Release(thr, pc, submit_sync); 747 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); 748 Block_release(new_h); 749 return new_channel; 750 } 751 752 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, 753 dispatch_io_close_flags_t flags) { 754 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); 755 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. 756 return REAL(dispatch_io_close)(channel, flags); 757 } 758 759 // Resuming a suspended queue needs to synchronize with all subsequent 760 // executions of blocks in that queue. 761 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { 762 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o); 763 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync 764 // in dispatch_sync_pre_execute 765 return REAL(dispatch_resume)(o); 766 } 767 768 void InitializeLibdispatchInterceptors() { 769 INTERCEPT_FUNCTION(dispatch_async); 770 INTERCEPT_FUNCTION(dispatch_async_f); 771 INTERCEPT_FUNCTION(dispatch_sync); 772 INTERCEPT_FUNCTION(dispatch_sync_f); 773 INTERCEPT_FUNCTION(dispatch_barrier_async); 774 INTERCEPT_FUNCTION(dispatch_barrier_async_f); 775 INTERCEPT_FUNCTION(dispatch_barrier_sync); 776 INTERCEPT_FUNCTION(dispatch_barrier_sync_f); 777 INTERCEPT_FUNCTION(dispatch_async_and_wait); 778 INTERCEPT_FUNCTION(dispatch_async_and_wait_f); 779 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait); 780 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f); 781 INTERCEPT_FUNCTION(dispatch_after); 782 INTERCEPT_FUNCTION(dispatch_after_f); 783 INTERCEPT_FUNCTION(dispatch_once); 784 INTERCEPT_FUNCTION(dispatch_once_f); 785 INTERCEPT_FUNCTION(dispatch_semaphore_signal); 786 INTERCEPT_FUNCTION(dispatch_semaphore_wait); 787 INTERCEPT_FUNCTION(dispatch_group_wait); 788 INTERCEPT_FUNCTION(dispatch_group_leave); 789 INTERCEPT_FUNCTION(dispatch_group_async); 790 INTERCEPT_FUNCTION(dispatch_group_async_f); 791 INTERCEPT_FUNCTION(dispatch_group_notify); 792 INTERCEPT_FUNCTION(dispatch_group_notify_f); 793 INTERCEPT_FUNCTION(dispatch_source_set_event_handler); 794 INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); 795 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); 796 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); 797 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); 798 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); 799 INTERCEPT_FUNCTION(dispatch_apply); 800 INTERCEPT_FUNCTION(dispatch_apply_f); 801 INTERCEPT_FUNCTION(dispatch_data_create); 802 INTERCEPT_FUNCTION(dispatch_read); 803 INTERCEPT_FUNCTION(dispatch_write); 804 INTERCEPT_FUNCTION(dispatch_io_read); 805 INTERCEPT_FUNCTION(dispatch_io_write); 806 INTERCEPT_FUNCTION(dispatch_io_barrier); 807 INTERCEPT_FUNCTION(dispatch_io_create); 808 INTERCEPT_FUNCTION(dispatch_io_create_with_path); 809 INTERCEPT_FUNCTION(dispatch_io_create_with_io); 810 INTERCEPT_FUNCTION(dispatch_io_close); 811 INTERCEPT_FUNCTION(dispatch_resume); 812 } 813 814 } // namespace __tsan 815