1 //===-- tsan_platform_linux.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Linux- and FreeBSD-specific code. 12 //===----------------------------------------------------------------------===// 13 14 15 #include "sanitizer_common/sanitizer_platform.h" 16 #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD 17 18 #include "sanitizer_common/sanitizer_common.h" 19 #include "sanitizer_common/sanitizer_libc.h" 20 #include "sanitizer_common/sanitizer_linux.h" 21 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h" 22 #include "sanitizer_common/sanitizer_platform_limits_posix.h" 23 #include "sanitizer_common/sanitizer_posix.h" 24 #include "sanitizer_common/sanitizer_procmaps.h" 25 #include "sanitizer_common/sanitizer_stoptheworld.h" 26 #include "sanitizer_common/sanitizer_stackdepot.h" 27 #include "tsan_platform.h" 28 #include "tsan_rtl.h" 29 #include "tsan_flags.h" 30 31 #include <fcntl.h> 32 #include <pthread.h> 33 #include <signal.h> 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <string.h> 37 #include <stdarg.h> 38 #include <sys/mman.h> 39 #if SANITIZER_LINUX 40 #include <sys/personality.h> 41 #include <setjmp.h> 42 #endif 43 #include <sys/syscall.h> 44 #include <sys/socket.h> 45 #include <sys/time.h> 46 #include <sys/types.h> 47 #include <sys/resource.h> 48 #include <sys/stat.h> 49 #include <unistd.h> 50 #include <sched.h> 51 #include <dlfcn.h> 52 #if SANITIZER_LINUX 53 #define __need_res_state 54 #include <resolv.h> 55 #endif 56 57 #ifdef sa_handler 58 # undef sa_handler 59 #endif 60 61 #ifdef sa_sigaction 62 # undef sa_sigaction 63 #endif 64 65 #if SANITIZER_FREEBSD 66 extern "C" void *__libc_stack_end; 67 void *__libc_stack_end = 0; 68 #endif 69 70 #if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO 71 # define INIT_LONGJMP_XOR_KEY 1 72 #else 73 # define INIT_LONGJMP_XOR_KEY 0 74 #endif 75 76 #if INIT_LONGJMP_XOR_KEY 77 #include "interception/interception.h" 78 // Must be declared outside of other namespaces. 79 DECLARE_REAL(int, _setjmp, void *env) 80 #endif 81 82 namespace __tsan { 83 84 #if INIT_LONGJMP_XOR_KEY 85 static void InitializeLongjmpXorKey(); 86 static uptr longjmp_xor_key; 87 #endif 88 89 #ifdef TSAN_RUNTIME_VMA 90 // Runtime detected VMA size. 91 uptr vmaSize; 92 #endif 93 94 enum { 95 MemTotal = 0, 96 MemShadow = 1, 97 MemMeta = 2, 98 MemFile = 3, 99 MemMmap = 4, 100 MemTrace = 5, 101 MemHeap = 6, 102 MemOther = 7, 103 MemCount = 8, 104 }; 105 106 void FillProfileCallback(uptr p, uptr rss, bool file, 107 uptr *mem, uptr stats_size) { 108 mem[MemTotal] += rss; 109 if (p >= ShadowBeg() && p < ShadowEnd()) 110 mem[MemShadow] += rss; 111 else if (p >= MetaShadowBeg() && p < MetaShadowEnd()) 112 mem[MemMeta] += rss; 113 #if !SANITIZER_GO 114 else if (p >= HeapMemBeg() && p < HeapMemEnd()) 115 mem[MemHeap] += rss; 116 else if (p >= LoAppMemBeg() && p < LoAppMemEnd()) 117 mem[file ? MemFile : MemMmap] += rss; 118 else if (p >= HiAppMemBeg() && p < HiAppMemEnd()) 119 mem[file ? MemFile : MemMmap] += rss; 120 #else 121 else if (p >= AppMemBeg() && p < AppMemEnd()) 122 mem[file ? MemFile : MemMmap] += rss; 123 #endif 124 else if (p >= TraceMemBeg() && p < TraceMemEnd()) 125 mem[MemTrace] += rss; 126 else 127 mem[MemOther] += rss; 128 } 129 130 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { 131 uptr mem[MemCount]; 132 internal_memset(mem, 0, sizeof(mem[0]) * MemCount); 133 __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7); 134 StackDepotStats *stacks = StackDepotGetStats(); 135 internal_snprintf(buf, buf_size, 136 "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd" 137 " trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n", 138 mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20, 139 mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20, 140 mem[MemHeap] >> 20, mem[MemOther] >> 20, 141 stacks->allocated >> 20, stacks->n_uniq_ids, 142 nlive, nthread); 143 } 144 145 #if SANITIZER_LINUX 146 void FlushShadowMemoryCallback( 147 const SuspendedThreadsList &suspended_threads_list, 148 void *argument) { 149 ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd()); 150 } 151 #endif 152 153 void FlushShadowMemory() { 154 #if SANITIZER_LINUX 155 StopTheWorld(FlushShadowMemoryCallback, 0); 156 #endif 157 } 158 159 #if !SANITIZER_GO 160 // Mark shadow for .rodata sections with the special kShadowRodata marker. 161 // Accesses to .rodata can't race, so this saves time, memory and trace space. 162 static void MapRodata() { 163 // First create temp file. 164 const char *tmpdir = GetEnv("TMPDIR"); 165 if (tmpdir == 0) 166 tmpdir = GetEnv("TEST_TMPDIR"); 167 #ifdef P_tmpdir 168 if (tmpdir == 0) 169 tmpdir = P_tmpdir; 170 #endif 171 if (tmpdir == 0) 172 return; 173 char name[256]; 174 internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d", 175 tmpdir, (int)internal_getpid()); 176 uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); 177 if (internal_iserror(openrv)) 178 return; 179 internal_unlink(name); // Unlink it now, so that we can reuse the buffer. 180 fd_t fd = openrv; 181 // Fill the file with kShadowRodata. 182 const uptr kMarkerSize = 512 * 1024 / sizeof(u64); 183 InternalMmapVector<u64> marker(kMarkerSize); 184 // volatile to prevent insertion of memset 185 for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++) 186 *p = kShadowRodata; 187 internal_write(fd, marker.data(), marker.size() * sizeof(u64)); 188 // Map the file into memory. 189 uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE, 190 MAP_PRIVATE | MAP_ANONYMOUS, fd, 0); 191 if (internal_iserror(page)) { 192 internal_close(fd); 193 return; 194 } 195 // Map the file into shadow of .rodata sections. 196 MemoryMappingLayout proc_maps(/*cache_enabled*/true); 197 // Reusing the buffer 'name'. 198 MemoryMappedSegment segment(name, ARRAY_SIZE(name)); 199 while (proc_maps.Next(&segment)) { 200 if (segment.filename[0] != 0 && segment.filename[0] != '[' && 201 segment.IsReadable() && segment.IsExecutable() && 202 !segment.IsWritable() && IsAppMem(segment.start)) { 203 // Assume it's .rodata 204 char *shadow_start = (char *)MemToShadow(segment.start); 205 char *shadow_end = (char *)MemToShadow(segment.end); 206 for (char *p = shadow_start; p < shadow_end; 207 p += marker.size() * sizeof(u64)) { 208 internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p), 209 PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); 210 } 211 } 212 } 213 internal_close(fd); 214 } 215 216 void InitializeShadowMemoryPlatform() { 217 MapRodata(); 218 } 219 220 #endif // #if !SANITIZER_GO 221 222 void InitializePlatformEarly() { 223 #ifdef TSAN_RUNTIME_VMA 224 vmaSize = 225 (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); 226 #if defined(__aarch64__) 227 # if !SANITIZER_GO 228 if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) { 229 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 230 Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize); 231 Die(); 232 } 233 #else 234 if (vmaSize != 48) { 235 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 236 Printf("FATAL: Found %zd - Supported 48\n", vmaSize); 237 Die(); 238 } 239 #endif 240 #elif defined(__powerpc64__) 241 # if !SANITIZER_GO 242 if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { 243 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 244 Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize); 245 Die(); 246 } 247 # else 248 if (vmaSize != 46 && vmaSize != 47) { 249 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 250 Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize); 251 Die(); 252 } 253 # endif 254 #endif 255 #endif 256 } 257 258 void InitializePlatform() { 259 DisableCoreDumperIfNecessary(); 260 261 // Go maps shadow memory lazily and works fine with limited address space. 262 // Unlimited stack is not a problem as well, because the executable 263 // is not compiled with -pie. 264 #if !SANITIZER_GO 265 { 266 bool reexec = false; 267 // TSan doesn't play well with unlimited stack size (as stack 268 // overlaps with shadow memory). If we detect unlimited stack size, 269 // we re-exec the program with limited stack size as a best effort. 270 if (StackSizeIsUnlimited()) { 271 const uptr kMaxStackSize = 32 * 1024 * 1024; 272 VReport(1, "Program is run with unlimited stack size, which wouldn't " 273 "work with ThreadSanitizer.\n" 274 "Re-execing with stack size limited to %zd bytes.\n", 275 kMaxStackSize); 276 SetStackSizeLimitInBytes(kMaxStackSize); 277 reexec = true; 278 } 279 280 if (!AddressSpaceIsUnlimited()) { 281 Report("WARNING: Program is run with limited virtual address space," 282 " which wouldn't work with ThreadSanitizer.\n"); 283 Report("Re-execing with unlimited virtual address space.\n"); 284 SetAddressSpaceUnlimited(); 285 reexec = true; 286 } 287 #if SANITIZER_LINUX && defined(__aarch64__) 288 // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in 289 // linux kernel, the random gap between stack and mapped area is increased 290 // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover 291 // this big range, we should disable randomized virtual space on aarch64. 292 int old_personality = personality(0xffffffff); 293 if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { 294 VReport(1, "WARNING: Program is run with randomized virtual address " 295 "space, which wouldn't work with ThreadSanitizer.\n" 296 "Re-execing with fixed virtual address space.\n"); 297 CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); 298 reexec = true; 299 } 300 // Initialize the xor key used in {sig}{set,long}jump. 301 InitializeLongjmpXorKey(); 302 #endif 303 if (reexec) 304 ReExec(); 305 } 306 307 CheckAndProtect(); 308 InitTlsSize(); 309 #endif // !SANITIZER_GO 310 } 311 312 #if !SANITIZER_GO 313 // Extract file descriptors passed to glibc internal __res_iclose function. 314 // This is required to properly "close" the fds, because we do not see internal 315 // closes within glibc. The code is a pure hack. 316 int ExtractResolvFDs(void *state, int *fds, int nfd) { 317 #if SANITIZER_LINUX && !SANITIZER_ANDROID 318 int cnt = 0; 319 struct __res_state *statp = (struct __res_state*)state; 320 for (int i = 0; i < MAXNS && cnt < nfd; i++) { 321 if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1) 322 fds[cnt++] = statp->_u._ext.nssocks[i]; 323 } 324 return cnt; 325 #else 326 return 0; 327 #endif 328 } 329 330 // Extract file descriptors passed via UNIX domain sockets. 331 // This is requried to properly handle "open" of these fds. 332 // see 'man recvmsg' and 'man 3 cmsg'. 333 int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { 334 int res = 0; 335 msghdr *msg = (msghdr*)msgp; 336 struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg); 337 for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { 338 if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) 339 continue; 340 int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]); 341 for (int i = 0; i < n; i++) { 342 fds[res++] = ((int*)CMSG_DATA(cmsg))[i]; 343 if (res == nfd) 344 return res; 345 } 346 } 347 return res; 348 } 349 350 // Reverse operation of libc stack pointer mangling 351 static uptr UnmangleLongJmpSp(uptr mangled_sp) { 352 #if defined(__x86_64__) 353 # if SANITIZER_LINUX 354 // Reverse of: 355 // xor %fs:0x30, %rsi 356 // rol $0x11, %rsi 357 uptr sp; 358 asm("ror $0x11, %0 \n" 359 "xor %%fs:0x30, %0 \n" 360 : "=r" (sp) 361 : "0" (mangled_sp)); 362 return sp; 363 # else 364 return mangled_sp; 365 # endif 366 #elif defined(__aarch64__) 367 # if SANITIZER_LINUX 368 return mangled_sp ^ longjmp_xor_key; 369 # else 370 return mangled_sp; 371 # endif 372 #elif defined(__powerpc64__) 373 // Reverse of: 374 // ld r4, -28696(r13) 375 // xor r4, r3, r4 376 uptr xor_key; 377 asm("ld %0, -28696(%%r13)" : "=r" (xor_key)); 378 return mangled_sp ^ xor_key; 379 #elif defined(__mips__) 380 return mangled_sp; 381 #else 382 #error "Unknown platform" 383 #endif 384 } 385 386 #ifdef __powerpc__ 387 # define LONG_JMP_SP_ENV_SLOT 0 388 #elif SANITIZER_FREEBSD 389 # define LONG_JMP_SP_ENV_SLOT 2 390 #elif SANITIZER_NETBSD 391 # define LONG_JMP_SP_ENV_SLOT 6 392 #elif SANITIZER_LINUX 393 # ifdef __aarch64__ 394 # define LONG_JMP_SP_ENV_SLOT 13 395 # elif defined(__mips64) 396 # define LONG_JMP_SP_ENV_SLOT 1 397 # else 398 # define LONG_JMP_SP_ENV_SLOT 6 399 # endif 400 #endif 401 402 uptr ExtractLongJmpSp(uptr *env) { 403 uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT]; 404 return UnmangleLongJmpSp(mangled_sp); 405 } 406 407 #if INIT_LONGJMP_XOR_KEY 408 // GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp 409 // functions) by XORing them with a random key. For AArch64 it is a global 410 // variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by 411 // issuing a setjmp and XORing the SP pointer values to derive the key. 412 static void InitializeLongjmpXorKey() { 413 // 1. Call REAL(setjmp), which stores the mangled SP in env. 414 jmp_buf env; 415 REAL(_setjmp)(env); 416 417 // 2. Retrieve vanilla/mangled SP. 418 uptr sp; 419 asm("mov %0, sp" : "=r" (sp)); 420 uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT]; 421 422 // 3. xor SPs to obtain key. 423 longjmp_xor_key = mangled_sp ^ sp; 424 } 425 #endif 426 427 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) { 428 // Check that the thr object is in tls; 429 const uptr thr_beg = (uptr)thr; 430 const uptr thr_end = (uptr)thr + sizeof(*thr); 431 CHECK_GE(thr_beg, tls_addr); 432 CHECK_LE(thr_beg, tls_addr + tls_size); 433 CHECK_GE(thr_end, tls_addr); 434 CHECK_LE(thr_end, tls_addr + tls_size); 435 // Since the thr object is huge, skip it. 436 MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, thr_beg - tls_addr); 437 MemoryRangeImitateWrite(thr, /*pc=*/2, thr_end, 438 tls_addr + tls_size - thr_end); 439 } 440 441 // Note: this function runs with async signals enabled, 442 // so it must not touch any tsan state. 443 int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, 444 void *abstime), void *c, void *m, void *abstime, 445 void(*cleanup)(void *arg), void *arg) { 446 // pthread_cleanup_push/pop are hardcore macros mess. 447 // We can't intercept nor call them w/o including pthread.h. 448 int res; 449 pthread_cleanup_push(cleanup, arg); 450 res = fn(c, m, abstime); 451 pthread_cleanup_pop(0); 452 return res; 453 } 454 #endif // !SANITIZER_GO 455 456 #if !SANITIZER_GO 457 void ReplaceSystemMalloc() { } 458 #endif 459 460 #if !SANITIZER_GO 461 #if SANITIZER_ANDROID 462 // On Android, one thread can call intercepted functions after 463 // DestroyThreadState(), so add a fake thread state for "dead" threads. 464 static ThreadState *dead_thread_state = nullptr; 465 466 ThreadState *cur_thread() { 467 ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr()); 468 if (thr == nullptr) { 469 __sanitizer_sigset_t emptyset; 470 internal_sigfillset(&emptyset); 471 __sanitizer_sigset_t oldset; 472 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); 473 thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr()); 474 if (thr == nullptr) { 475 thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState), 476 "ThreadState")); 477 *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); 478 if (dead_thread_state == nullptr) { 479 dead_thread_state = reinterpret_cast<ThreadState*>( 480 MmapOrDie(sizeof(ThreadState), "ThreadState")); 481 dead_thread_state->fast_state.SetIgnoreBit(); 482 dead_thread_state->ignore_interceptors = 1; 483 dead_thread_state->is_dead = true; 484 *const_cast<int*>(&dead_thread_state->tid) = -1; 485 CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState), 486 PROT_READ)); 487 } 488 } 489 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); 490 } 491 return thr; 492 } 493 494 void set_cur_thread(ThreadState *thr) { 495 *get_android_tls_ptr() = reinterpret_cast<uptr>(thr); 496 } 497 498 void cur_thread_finalize() { 499 __sanitizer_sigset_t emptyset; 500 internal_sigfillset(&emptyset); 501 __sanitizer_sigset_t oldset; 502 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); 503 ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr()); 504 if (thr != dead_thread_state) { 505 *get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state); 506 UnmapOrDie(thr, sizeof(ThreadState)); 507 } 508 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); 509 } 510 #endif // SANITIZER_ANDROID 511 #endif // if !SANITIZER_GO 512 513 } // namespace __tsan 514 515 #endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD 516