1 //===-- tsan_interceptors_mac.cc ------------------------------------------===//
2 //
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
5 //
6 //===----------------------------------------------------------------------===//
7 //
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
9 //
10 // Mac-specific interceptors.
11 //===----------------------------------------------------------------------===//
12
13 #include "sanitizer_common/sanitizer_platform.h"
14 #if SANITIZER_MAC
15
16 #include "interception/interception.h"
17 #include "tsan_interceptors.h"
18 #include "tsan_interface.h"
19 #include "tsan_interface_ann.h"
20
21 #include <libkern/OSAtomic.h>
22
23 #if defined(__has_include) && __has_include(<xpc/xpc.h>)
24 #include <xpc/xpc.h>
25 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
26
27 typedef long long_t; // NOLINT
28
29 namespace __tsan {
30
31 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
32 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
33 // actually aliases of each other, and we cannot have different interceptors for
34 // them, because they're actually the same function. Thus, we have to stay
35 // conservative and treat the non-barrier versions as mo_acq_rel.
36 static const morder kMacOrderBarrier = mo_acq_rel;
37 static const morder kMacOrderNonBarrier = mo_acq_rel;
38
39 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
40 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
41 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
42 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
43 }
44
45 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
46 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
47 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
48 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
49 }
50
51 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
52 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
53 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
54 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
55 }
56
57 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
58 mo) \
59 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
60 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
61 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
62 }
63
64 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
65 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
66 kMacOrderNonBarrier) \
67 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
68 kMacOrderBarrier) \
69 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
70 kMacOrderNonBarrier) \
71 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
72 kMacOrderBarrier)
73
74 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
75 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
76 kMacOrderNonBarrier) \
77 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
78 kMacOrderBarrier) \
79 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
80 kMacOrderNonBarrier) \
81 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
82 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
83
OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd,fetch_add,OSATOMIC_INTERCEPTOR_PLUS_X)84 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
85 OSATOMIC_INTERCEPTOR_PLUS_X)
86 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
87 OSATOMIC_INTERCEPTOR_PLUS_1)
88 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
89 OSATOMIC_INTERCEPTOR_MINUS_1)
90 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
91 OSATOMIC_INTERCEPTOR)
92 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
93 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
94 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
95 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
96
97 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
98 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
99 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
100 return tsan_atomic_f##_compare_exchange_strong( \
101 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
102 kMacOrderNonBarrier, kMacOrderNonBarrier); \
103 } \
104 \
105 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
106 t volatile *ptr) { \
107 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
108 return tsan_atomic_f##_compare_exchange_strong( \
109 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
110 kMacOrderBarrier, kMacOrderNonBarrier); \
111 }
112
113 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
114 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
115 long_t)
116 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
117 void *)
118 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
119 int32_t)
120 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
121 int64_t)
122
123 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
124 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
125 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
126 char *byte_ptr = ((char *)ptr) + (n >> 3); \
127 char bit = 0x80u >> (n & 7); \
128 char mask = clear ? ~bit : bit; \
129 char orig_byte = op((a8 *)byte_ptr, mask, mo); \
130 return orig_byte & bit; \
131 }
132
133 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
134 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
135 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
136
137 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
138 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
139 true)
140
141 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
142 size_t offset) {
143 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
144 __tsan_release(item);
145 REAL(OSAtomicEnqueue)(list, item, offset);
146 }
147
TSAN_INTERCEPTOR(void *,OSAtomicDequeue,OSQueueHead * list,size_t offset)148 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
149 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
150 void *item = REAL(OSAtomicDequeue)(list, offset);
151 if (item) __tsan_acquire(item);
152 return item;
153 }
154
155 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
156 #if !SANITIZER_IOS
157
TSAN_INTERCEPTOR(void,OSAtomicFifoEnqueue,OSFifoQueueHead * list,void * item,size_t offset)158 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
159 size_t offset) {
160 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
161 __tsan_release(item);
162 REAL(OSAtomicFifoEnqueue)(list, item, offset);
163 }
164
TSAN_INTERCEPTOR(void *,OSAtomicFifoDequeue,OSFifoQueueHead * list,size_t offset)165 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
166 size_t offset) {
167 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
168 void *item = REAL(OSAtomicFifoDequeue)(list, offset);
169 if (item) __tsan_acquire(item);
170 return item;
171 }
172
173 #endif
174
TSAN_INTERCEPTOR(void,OSSpinLockLock,volatile OSSpinLock * lock)175 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
176 CHECK(!cur_thread()->is_dead);
177 if (!cur_thread()->is_inited) {
178 return REAL(OSSpinLockLock)(lock);
179 }
180 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
181 REAL(OSSpinLockLock)(lock);
182 Acquire(thr, pc, (uptr)lock);
183 }
184
TSAN_INTERCEPTOR(bool,OSSpinLockTry,volatile OSSpinLock * lock)185 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
186 CHECK(!cur_thread()->is_dead);
187 if (!cur_thread()->is_inited) {
188 return REAL(OSSpinLockTry)(lock);
189 }
190 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
191 bool result = REAL(OSSpinLockTry)(lock);
192 if (result)
193 Acquire(thr, pc, (uptr)lock);
194 return result;
195 }
196
TSAN_INTERCEPTOR(void,OSSpinLockUnlock,volatile OSSpinLock * lock)197 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
198 CHECK(!cur_thread()->is_dead);
199 if (!cur_thread()->is_inited) {
200 return REAL(OSSpinLockUnlock)(lock);
201 }
202 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
203 Release(thr, pc, (uptr)lock);
204 REAL(OSSpinLockUnlock)(lock);
205 }
206
TSAN_INTERCEPTOR(void,os_lock_lock,void * lock)207 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
208 CHECK(!cur_thread()->is_dead);
209 if (!cur_thread()->is_inited) {
210 return REAL(os_lock_lock)(lock);
211 }
212 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
213 REAL(os_lock_lock)(lock);
214 Acquire(thr, pc, (uptr)lock);
215 }
216
TSAN_INTERCEPTOR(bool,os_lock_trylock,void * lock)217 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
218 CHECK(!cur_thread()->is_dead);
219 if (!cur_thread()->is_inited) {
220 return REAL(os_lock_trylock)(lock);
221 }
222 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
223 bool result = REAL(os_lock_trylock)(lock);
224 if (result)
225 Acquire(thr, pc, (uptr)lock);
226 return result;
227 }
228
TSAN_INTERCEPTOR(void,os_lock_unlock,void * lock)229 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
230 CHECK(!cur_thread()->is_dead);
231 if (!cur_thread()->is_inited) {
232 return REAL(os_lock_unlock)(lock);
233 }
234 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
235 Release(thr, pc, (uptr)lock);
236 REAL(os_lock_unlock)(lock);
237 }
238
239 #if defined(__has_include) && __has_include(<xpc/xpc.h>)
240
TSAN_INTERCEPTOR(void,xpc_connection_set_event_handler,xpc_connection_t connection,xpc_handler_t handler)241 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
242 xpc_connection_t connection, xpc_handler_t handler) {
243 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
244 handler);
245 Release(thr, pc, (uptr)connection);
246 xpc_handler_t new_handler = ^(xpc_object_t object) {
247 {
248 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
249 Acquire(thr, pc, (uptr)connection);
250 }
251 handler(object);
252 };
253 REAL(xpc_connection_set_event_handler)(connection, new_handler);
254 }
255
TSAN_INTERCEPTOR(void,xpc_connection_send_barrier,xpc_connection_t connection,dispatch_block_t barrier)256 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
257 dispatch_block_t barrier) {
258 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
259 Release(thr, pc, (uptr)connection);
260 dispatch_block_t new_barrier = ^() {
261 {
262 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
263 Acquire(thr, pc, (uptr)connection);
264 }
265 barrier();
266 };
267 REAL(xpc_connection_send_barrier)(connection, new_barrier);
268 }
269
TSAN_INTERCEPTOR(void,xpc_connection_send_message_with_reply,xpc_connection_t connection,xpc_object_t message,dispatch_queue_t replyq,xpc_handler_t handler)270 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
271 xpc_connection_t connection, xpc_object_t message,
272 dispatch_queue_t replyq, xpc_handler_t handler) {
273 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
274 message, replyq, handler);
275 Release(thr, pc, (uptr)connection);
276 xpc_handler_t new_handler = ^(xpc_object_t object) {
277 {
278 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
279 Acquire(thr, pc, (uptr)connection);
280 }
281 handler(object);
282 };
283 REAL(xpc_connection_send_message_with_reply)
284 (connection, message, replyq, new_handler);
285 }
286
TSAN_INTERCEPTOR(void,xpc_connection_cancel,xpc_connection_t connection)287 TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
288 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
289 Release(thr, pc, (uptr)connection);
290 REAL(xpc_connection_cancel)(connection);
291 }
292
293 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
294
295 // On macOS, libc++ is always linked dynamically, so intercepting works the
296 // usual way.
297 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
298
299 namespace {
300 struct fake_shared_weak_count {
301 volatile a64 shared_owners;
302 volatile a64 shared_weak_owners;
303 virtual void _unused_0x0() = 0;
304 virtual void _unused_0x8() = 0;
305 virtual void on_zero_shared() = 0;
306 virtual void _unused_0x18() = 0;
307 virtual void on_zero_shared_weak() = 0;
308 };
309 } // namespace
310
311 // The following code adds libc++ interceptors for:
312 // void __shared_weak_count::__release_shared() _NOEXCEPT;
313 // bool __shared_count::__release_shared() _NOEXCEPT;
314 // Shared and weak pointers in C++ maintain reference counts via atomics in
315 // libc++.dylib, which are TSan-invisible, and this leads to false positives in
316 // destructor code. These interceptors re-implements the whole functions so that
317 // the mo_acq_rel semantics of the atomic decrement are visible.
318 //
319 // Unfortunately, the interceptors cannot simply Acquire/Release some sync
320 // object and call the original function, because it would have a race between
321 // the sync and the destruction of the object. Calling both under a lock will
322 // not work because the destructor can invoke this interceptor again (and even
323 // in a different thread, so recursive locks don't help).
324
STDCXX_INTERCEPTOR(void,_ZNSt3__119__shared_weak_count16__release_sharedEv,fake_shared_weak_count * o)325 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
326 fake_shared_weak_count *o) {
327 if (!flags()->shared_ptr_interceptor)
328 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
329
330 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
331 o);
332 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
333 Acquire(thr, pc, (uptr)&o->shared_owners);
334 o->on_zero_shared();
335 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
336 0) {
337 Acquire(thr, pc, (uptr)&o->shared_weak_owners);
338 o->on_zero_shared_weak();
339 }
340 }
341 }
342
STDCXX_INTERCEPTOR(bool,_ZNSt3__114__shared_count16__release_sharedEv,fake_shared_weak_count * o)343 STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
344 fake_shared_weak_count *o) {
345 if (!flags()->shared_ptr_interceptor)
346 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
347
348 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
349 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
350 Acquire(thr, pc, (uptr)&o->shared_owners);
351 o->on_zero_shared();
352 return true;
353 }
354 return false;
355 }
356
357 namespace {
358 struct call_once_callback_args {
359 void (*orig_func)(void *arg);
360 void *orig_arg;
361 void *flag;
362 };
363
call_once_callback_wrapper(void * arg)364 void call_once_callback_wrapper(void *arg) {
365 call_once_callback_args *new_args = (call_once_callback_args *)arg;
366 new_args->orig_func(new_args->orig_arg);
367 __tsan_release(new_args->flag);
368 }
369 } // namespace
370
371 // This adds a libc++ interceptor for:
372 // void __call_once(volatile unsigned long&, void*, void(*)(void*));
373 // C++11 call_once is implemented via an internal function __call_once which is
374 // inside libc++.dylib, and the atomic release store inside it is thus
375 // TSan-invisible. To avoid false positives, this interceptor wraps the callback
376 // function and performs an explicit Release after the user code has run.
STDCXX_INTERCEPTOR(void,_ZNSt3__111__call_onceERVmPvPFvS2_E,void * flag,void * arg,void (* func)(void * arg))377 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
378 void *arg, void (*func)(void *arg)) {
379 call_once_callback_args new_args = {func, arg, flag};
380 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
381 call_once_callback_wrapper);
382 }
383
384 } // namespace __tsan
385
386 #endif // SANITIZER_MAC
387