1 //===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and 11 /// FreeBSD-specific code. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_platform.h" 16 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD 17 18 # include <dlfcn.h> 19 # include <elf.h> 20 # include <errno.h> 21 # include <link.h> 22 # include <pthread.h> 23 # include <signal.h> 24 # include <stdio.h> 25 # include <stdlib.h> 26 # include <sys/prctl.h> 27 # include <sys/resource.h> 28 # include <sys/time.h> 29 # include <unistd.h> 30 # include <unwind.h> 31 32 # include "hwasan.h" 33 # include "hwasan_dynamic_shadow.h" 34 # include "hwasan_interface_internal.h" 35 # include "hwasan_mapping.h" 36 # include "hwasan_report.h" 37 # include "hwasan_thread.h" 38 # include "hwasan_thread_list.h" 39 # include "sanitizer_common/sanitizer_common.h" 40 # include "sanitizer_common/sanitizer_procmaps.h" 41 # include "sanitizer_common/sanitizer_stackdepot.h" 42 43 // Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID. 44 // 45 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF 46 // Not currently tested. 47 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON 48 // Integration tests downstream exist. 49 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF 50 // Tested with check-hwasan on x86_64-linux. 51 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON 52 // Tested with check-hwasan on aarch64-linux-android. 53 # if !SANITIZER_ANDROID 54 SANITIZER_INTERFACE_ATTRIBUTE 55 THREADLOCAL uptr __hwasan_tls; 56 # endif 57 58 namespace __hwasan { 59 60 // With the zero shadow base we can not actually map pages starting from 0. 61 // This constant is somewhat arbitrary. 62 constexpr uptr kZeroBaseShadowStart = 0; 63 constexpr uptr kZeroBaseMaxShadowStart = 1 << 18; 64 65 static void ProtectGap(uptr addr, uptr size) { 66 __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart, 67 kZeroBaseMaxShadowStart); 68 } 69 70 uptr kLowMemStart; 71 uptr kLowMemEnd; 72 uptr kHighMemStart; 73 uptr kHighMemEnd; 74 75 static void PrintRange(uptr start, uptr end, const char *name) { 76 Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name); 77 } 78 79 static void PrintAddressSpaceLayout() { 80 PrintRange(kHighMemStart, kHighMemEnd, "HighMem"); 81 if (kHighShadowEnd + 1 < kHighMemStart) 82 PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap"); 83 else 84 CHECK_EQ(kHighShadowEnd + 1, kHighMemStart); 85 PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow"); 86 if (kLowShadowEnd + 1 < kHighShadowStart) 87 PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap"); 88 else 89 CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); 90 PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); 91 if (kLowMemEnd + 1 < kLowShadowStart) 92 PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap"); 93 else 94 CHECK_EQ(kLowMemEnd + 1, kLowShadowStart); 95 PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); 96 CHECK_EQ(0, kLowMemStart); 97 } 98 99 static uptr GetHighMemEnd() { 100 // HighMem covers the upper part of the address space. 101 uptr max_address = GetMaxUserVirtualAddress(); 102 // Adjust max address to make sure that kHighMemEnd and kHighMemStart are 103 // properly aligned: 104 max_address |= (GetMmapGranularity() << kShadowScale) - 1; 105 return max_address; 106 } 107 108 static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { 109 __hwasan_shadow_memory_dynamic_address = 110 FindDynamicShadowStart(shadow_size_bytes); 111 } 112 113 static void MaybeDieIfNoTaggingAbi(const char *message) { 114 if (!flags()->fail_without_syscall_abi) 115 return; 116 Printf("FATAL: %s\n", message); 117 Die(); 118 } 119 120 # define PR_SET_TAGGED_ADDR_CTRL 55 121 # define PR_GET_TAGGED_ADDR_CTRL 56 122 # define PR_TAGGED_ADDR_ENABLE (1UL << 0) 123 # define ARCH_GET_UNTAG_MASK 0x4001 124 # define ARCH_ENABLE_TAGGED_ADDR 0x4002 125 # define ARCH_GET_MAX_TAG_BITS 0x4003 126 127 static bool CanUseTaggingAbi() { 128 # if defined(__x86_64__) 129 unsigned long num_bits = 0; 130 // Check for x86 LAM support. This API is based on a currently unsubmitted 131 // patch to the Linux kernel (as of August 2022) and is thus subject to 132 // change. The patch is here: 133 // https://lore.kernel.org/all/20220815041803.17954-1-kirill.shutemov@linux.intel.com/ 134 // 135 // arch_prctl(ARCH_GET_MAX_TAG_BITS, &bits) returns the maximum number of tag 136 // bits the user can request, or zero if LAM is not supported by the hardware. 137 if (internal_iserror(internal_arch_prctl(ARCH_GET_MAX_TAG_BITS, 138 reinterpret_cast<uptr>(&num_bits)))) 139 return false; 140 // The platform must provide enough bits for HWASan tags. 141 if (num_bits < kTagBits) 142 return false; 143 return true; 144 # else 145 // Check for ARM TBI support. 146 return !internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)); 147 # endif // __x86_64__ 148 } 149 150 static bool EnableTaggingAbi() { 151 # if defined(__x86_64__) 152 // Enable x86 LAM tagging for the process. 153 // 154 // arch_prctl(ARCH_ENABLE_TAGGED_ADDR, bits) enables tagging if the number of 155 // tag bits requested by the user does not exceed that provided by the system. 156 // arch_prctl(ARCH_GET_UNTAG_MASK, &mask) returns the mask of significant 157 // address bits. It is ~0ULL if either LAM is disabled for the process or LAM 158 // is not supported by the hardware. 159 if (internal_iserror(internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits))) 160 return false; 161 unsigned long mask = 0; 162 // Make sure the tag bits are where we expect them to be. 163 if (internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK, 164 reinterpret_cast<uptr>(&mask)))) 165 return false; 166 // @mask has ones for non-tag bits, whereas @kAddressTagMask has ones for tag 167 // bits. Therefore these masks must not overlap. 168 if (mask & kAddressTagMask) 169 return false; 170 return true; 171 # else 172 // Enable ARM TBI tagging for the process. If for some reason tagging is not 173 // supported, prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE) returns 174 // -EINVAL. 175 if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, 176 PR_TAGGED_ADDR_ENABLE, 0, 0, 0))) 177 return false; 178 // Ensure that TBI is enabled. 179 if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) != 180 PR_TAGGED_ADDR_ENABLE) 181 return false; 182 return true; 183 # endif // __x86_64__ 184 } 185 186 void InitializeOsSupport() { 187 // Check we're running on a kernel that can use the tagged address ABI. 188 bool has_abi = CanUseTaggingAbi(); 189 190 if (!has_abi) { 191 # if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE) 192 // Some older Android kernels have the tagged pointer ABI on 193 // unconditionally, and hence don't have the tagged-addr prctl while still 194 // allow the ABI. 195 // If targeting Android and the prctl is not around we assume this is the 196 // case. 197 return; 198 # else 199 MaybeDieIfNoTaggingAbi( 200 "HWAddressSanitizer requires a kernel with tagged address ABI."); 201 # endif 202 } 203 204 if (EnableTaggingAbi()) 205 return; 206 207 # if SANITIZER_ANDROID 208 MaybeDieIfNoTaggingAbi( 209 "HWAddressSanitizer failed to enable tagged address syscall ABI.\n" 210 "Check the `sysctl abi.tagged_addr_disabled` configuration."); 211 # else 212 MaybeDieIfNoTaggingAbi( 213 "HWAddressSanitizer failed to enable tagged address syscall ABI.\n"); 214 # endif 215 } 216 217 bool InitShadow() { 218 // Define the entire memory range. 219 kHighMemEnd = GetHighMemEnd(); 220 221 // Determine shadow memory base offset. 222 InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd)); 223 224 // Place the low memory first. 225 kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1; 226 kLowMemStart = 0; 227 228 // Define the low shadow based on the already placed low memory. 229 kLowShadowEnd = MemToShadow(kLowMemEnd); 230 kLowShadowStart = __hwasan_shadow_memory_dynamic_address; 231 232 // High shadow takes whatever memory is left up there (making sure it is not 233 // interfering with low memory in the fixed case). 234 kHighShadowEnd = MemToShadow(kHighMemEnd); 235 kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1; 236 237 // High memory starts where allocated shadow allows. 238 kHighMemStart = ShadowToMem(kHighShadowStart); 239 240 // Check the sanity of the defined memory ranges (there might be gaps). 241 CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0); 242 CHECK_GT(kHighMemStart, kHighShadowEnd); 243 CHECK_GT(kHighShadowEnd, kHighShadowStart); 244 CHECK_GT(kHighShadowStart, kLowMemEnd); 245 CHECK_GT(kLowMemEnd, kLowMemStart); 246 CHECK_GT(kLowShadowEnd, kLowShadowStart); 247 CHECK_GT(kLowShadowStart, kLowMemEnd); 248 249 if (Verbosity()) 250 PrintAddressSpaceLayout(); 251 252 // Reserve shadow memory. 253 ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow"); 254 ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow"); 255 256 // Protect all the gaps. 257 ProtectGap(0, Min(kLowMemStart, kLowShadowStart)); 258 if (kLowMemEnd + 1 < kLowShadowStart) 259 ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1); 260 if (kLowShadowEnd + 1 < kHighShadowStart) 261 ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1); 262 if (kHighShadowEnd + 1 < kHighMemStart) 263 ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1); 264 265 return true; 266 } 267 268 void InitThreads() { 269 CHECK(__hwasan_shadow_memory_dynamic_address); 270 uptr guard_page_size = GetMmapGranularity(); 271 uptr thread_space_start = 272 __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment); 273 uptr thread_space_end = 274 __hwasan_shadow_memory_dynamic_address - guard_page_size; 275 ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1, 276 "hwasan threads", /*madvise_shadow*/ false); 277 ProtectGap(thread_space_end, 278 __hwasan_shadow_memory_dynamic_address - thread_space_end); 279 InitThreadList(thread_space_start, thread_space_end - thread_space_start); 280 hwasanThreadList().CreateCurrentThread(); 281 } 282 283 bool MemIsApp(uptr p) { 284 // Memory outside the alias range has non-zero tags. 285 # if !defined(HWASAN_ALIASING_MODE) 286 CHECK_EQ(GetTagFromPointer(p), 0); 287 # endif 288 289 return (p >= kHighMemStart && p <= kHighMemEnd) || 290 (p >= kLowMemStart && p <= kLowMemEnd); 291 } 292 293 void InstallAtExitHandler() { atexit(HwasanAtExit); } 294 295 // ---------------------- TSD ---------------- {{{1 296 297 extern "C" void __hwasan_thread_enter() { 298 hwasanThreadList().CreateCurrentThread()->EnsureRandomStateInited(); 299 } 300 301 extern "C" void __hwasan_thread_exit() { 302 Thread *t = GetCurrentThread(); 303 // Make sure that signal handler can not see a stale current thread pointer. 304 atomic_signal_fence(memory_order_seq_cst); 305 if (t) { 306 // Block async signals on the thread as the handler can be instrumented. 307 // After this point instrumented code can't access essential data from TLS 308 // and will crash. 309 // Bionic already calls __hwasan_thread_exit with blocked signals. 310 if (SANITIZER_GLIBC) 311 BlockSignals(); 312 hwasanThreadList().ReleaseThread(t); 313 } 314 } 315 316 # if HWASAN_WITH_INTERCEPTORS 317 static pthread_key_t tsd_key; 318 static bool tsd_key_inited = false; 319 320 void HwasanTSDThreadInit() { 321 if (tsd_key_inited) 322 CHECK_EQ(0, pthread_setspecific(tsd_key, 323 (void *)GetPthreadDestructorIterations())); 324 } 325 326 void HwasanTSDDtor(void *tsd) { 327 uptr iterations = (uptr)tsd; 328 if (iterations > 1) { 329 CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1))); 330 return; 331 } 332 __hwasan_thread_exit(); 333 } 334 335 void HwasanTSDInit() { 336 CHECK(!tsd_key_inited); 337 tsd_key_inited = true; 338 CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor)); 339 } 340 # else 341 void HwasanTSDInit() {} 342 void HwasanTSDThreadInit() {} 343 # endif 344 345 # if SANITIZER_ANDROID 346 uptr *GetCurrentThreadLongPtr() { return (uptr *)get_android_tls_ptr(); } 347 # else 348 uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; } 349 # endif 350 351 # if SANITIZER_ANDROID 352 void AndroidTestTlsSlot() { 353 uptr kMagicValue = 0x010203040A0B0C0D; 354 uptr *tls_ptr = GetCurrentThreadLongPtr(); 355 uptr old_value = *tls_ptr; 356 *tls_ptr = kMagicValue; 357 dlerror(); 358 if (*(uptr *)get_android_tls_ptr() != kMagicValue) { 359 Printf( 360 "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used " 361 "for dlerror().\n"); 362 Die(); 363 } 364 *tls_ptr = old_value; 365 } 366 # else 367 void AndroidTestTlsSlot() {} 368 # endif 369 370 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { 371 // Access type is passed in a platform dependent way (see below) and encoded 372 // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is 373 // recoverable. Valid values of Y are 0 to 4, which are interpreted as 374 // log2(access_size), and 0xF, which means that access size is passed via 375 // platform dependent register (see below). 376 # if defined(__aarch64__) 377 // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF, 378 // access size is stored in X1 register. Access address is always in X0 379 // register. 380 uptr pc = (uptr)info->si_addr; 381 const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff; 382 if ((code & 0xff00) != 0x900) 383 return AccessInfo{}; // Not ours. 384 385 const bool is_store = code & 0x10; 386 const bool recover = code & 0x20; 387 const uptr addr = uc->uc_mcontext.regs[0]; 388 const unsigned size_log = code & 0xf; 389 if (size_log > 4 && size_log != 0xf) 390 return AccessInfo{}; // Not ours. 391 const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log; 392 393 # elif defined(__x86_64__) 394 // Access type is encoded in the instruction following INT3 as 395 // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in 396 // RSI register. Access address is always in RDI register. 397 uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP]; 398 uint8_t *nop = (uint8_t *)pc; 399 if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 || 400 *(nop + 3) < 0x40) 401 return AccessInfo{}; // Not ours. 402 const unsigned code = *(nop + 3); 403 404 const bool is_store = code & 0x10; 405 const bool recover = code & 0x20; 406 const uptr addr = uc->uc_mcontext.gregs[REG_RDI]; 407 const unsigned size_log = code & 0xf; 408 if (size_log > 4 && size_log != 0xf) 409 return AccessInfo{}; // Not ours. 410 const uptr size = 411 size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log; 412 413 # elif SANITIZER_RISCV64 414 // Access type is encoded in the instruction following EBREAK as 415 // ADDI x0, x0, [0x40 + 0xXY]. For Y == 0xF, access size is stored in 416 // X11 register. Access address is always in X10 register. 417 uptr pc = (uptr)uc->uc_mcontext.__gregs[REG_PC]; 418 uint8_t byte1 = *((u8 *)(pc + 0)); 419 uint8_t byte2 = *((u8 *)(pc + 1)); 420 uint8_t byte3 = *((u8 *)(pc + 2)); 421 uint8_t byte4 = *((u8 *)(pc + 3)); 422 uint32_t ebreak = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24)); 423 bool isFaultShort = false; 424 bool isEbreak = (ebreak == 0x100073); 425 bool isShortEbreak = false; 426 # if defined(__riscv_compressed) 427 isFaultShort = ((ebreak & 0x3) != 0x3); 428 isShortEbreak = ((ebreak & 0xffff) == 0x9002); 429 # endif 430 // faulted insn is not ebreak, not our case 431 if (!(isEbreak || isShortEbreak)) 432 return AccessInfo{}; 433 // advance pc to point after ebreak and reconstruct addi instruction 434 pc += isFaultShort ? 2 : 4; 435 byte1 = *((u8 *)(pc + 0)); 436 byte2 = *((u8 *)(pc + 1)); 437 byte3 = *((u8 *)(pc + 2)); 438 byte4 = *((u8 *)(pc + 3)); 439 // reconstruct instruction 440 uint32_t instr = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24)); 441 // check if this is really 32 bit instruction 442 // code is encoded in top 12 bits, since instruction is supposed to be with 443 // imm 444 const unsigned code = (instr >> 20) & 0xffff; 445 const uptr addr = uc->uc_mcontext.__gregs[10]; 446 const bool is_store = code & 0x10; 447 const bool recover = code & 0x20; 448 const unsigned size_log = code & 0xf; 449 if (size_log > 4 && size_log != 0xf) 450 return AccessInfo{}; // Not our case 451 const uptr size = 452 size_log == 0xf ? uc->uc_mcontext.__gregs[11] : 1U << size_log; 453 454 # else 455 # error Unsupported architecture 456 # endif 457 458 return AccessInfo{addr, size, is_store, !is_store, recover}; 459 } 460 461 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { 462 AccessInfo ai = GetAccessInfo(info, uc); 463 if (!ai.is_store && !ai.is_load) 464 return false; 465 466 SignalContext sig{info, uc}; 467 HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc); 468 469 # if defined(__aarch64__) 470 uc->uc_mcontext.pc += 4; 471 # elif defined(__x86_64__) 472 # elif SANITIZER_RISCV64 473 // pc points to EBREAK which is 2 bytes long 474 uint8_t *exception_source = (uint8_t *)(uc->uc_mcontext.__gregs[REG_PC]); 475 uint8_t byte1 = (uint8_t)(*(exception_source + 0)); 476 uint8_t byte2 = (uint8_t)(*(exception_source + 1)); 477 uint8_t byte3 = (uint8_t)(*(exception_source + 2)); 478 uint8_t byte4 = (uint8_t)(*(exception_source + 3)); 479 uint32_t faulted = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24)); 480 bool isFaultShort = false; 481 # if defined(__riscv_compressed) 482 isFaultShort = ((faulted & 0x3) != 0x3); 483 # endif 484 uc->uc_mcontext.__gregs[REG_PC] += isFaultShort ? 2 : 4; 485 # else 486 # error Unsupported architecture 487 # endif 488 return true; 489 } 490 491 static void OnStackUnwind(const SignalContext &sig, const void *, 492 BufferedStackTrace *stack) { 493 stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, 494 common_flags()->fast_unwind_on_fatal); 495 } 496 497 void HwasanOnDeadlySignal(int signo, void *info, void *context) { 498 // Probably a tag mismatch. 499 if (signo == SIGTRAP) 500 if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t *)context)) 501 return; 502 503 HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); 504 } 505 506 void Thread::InitStackAndTls(const InitState *) { 507 uptr tls_size; 508 uptr stack_size; 509 GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, 510 &tls_size); 511 stack_top_ = stack_bottom_ + stack_size; 512 tls_end_ = tls_begin_ + tls_size; 513 } 514 515 uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { 516 CHECK(IsAligned(p, kShadowAlignment)); 517 CHECK(IsAligned(size, kShadowAlignment)); 518 uptr shadow_start = MemToShadow(p); 519 uptr shadow_size = MemToShadowSize(size); 520 521 uptr page_size = GetPageSizeCached(); 522 uptr page_start = RoundUpTo(shadow_start, page_size); 523 uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); 524 uptr threshold = common_flags()->clear_shadow_mmap_threshold; 525 if (SANITIZER_LINUX && 526 UNLIKELY(page_end >= page_start + threshold && tag == 0)) { 527 internal_memset((void *)shadow_start, tag, page_start - shadow_start); 528 internal_memset((void *)page_end, tag, 529 shadow_start + shadow_size - page_end); 530 // For an anonymous private mapping MADV_DONTNEED will return a zero page on 531 // Linux. 532 ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); 533 } else { 534 internal_memset((void *)shadow_start, tag, shadow_size); 535 } 536 return AddTagToPointer(p, tag); 537 } 538 539 void HwasanInstallAtForkHandler() { 540 auto before = []() { 541 HwasanAllocatorLock(); 542 StackDepotLockAll(); 543 }; 544 auto after = []() { 545 StackDepotUnlockAll(); 546 HwasanAllocatorUnlock(); 547 }; 548 pthread_atfork(before, after, after); 549 } 550 551 void InstallAtExitCheckLeaks() { 552 if (CAN_SANITIZE_LEAKS) { 553 if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) { 554 if (flags()->halt_on_error) 555 Atexit(__lsan::DoLeakCheck); 556 else 557 Atexit(__lsan::DoRecoverableLeakCheckVoid); 558 } 559 } 560 } 561 562 } // namespace __hwasan 563 564 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD 565