1e8d8bef9SDimitry Andric //===-- memprof_allocator.cpp --------------------------------------------===// 2e8d8bef9SDimitry Andric // 3e8d8bef9SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4e8d8bef9SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5e8d8bef9SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6e8d8bef9SDimitry Andric // 7e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===// 8e8d8bef9SDimitry Andric // 9e8d8bef9SDimitry Andric // This file is a part of MemProfiler, a memory profiler. 10e8d8bef9SDimitry Andric // 11e8d8bef9SDimitry Andric // Implementation of MemProf's memory allocator, which uses the allocator 12e8d8bef9SDimitry Andric // from sanitizer_common. 13e8d8bef9SDimitry Andric // 14e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===// 15e8d8bef9SDimitry Andric 16e8d8bef9SDimitry Andric #include "memprof_allocator.h" 17e8d8bef9SDimitry Andric #include "memprof_mapping.h" 18349cc55cSDimitry Andric #include "memprof_meminfoblock.h" 19349cc55cSDimitry Andric #include "memprof_mibmap.h" 20349cc55cSDimitry Andric #include "memprof_rawprofile.h" 21e8d8bef9SDimitry Andric #include "memprof_stack.h" 22e8d8bef9SDimitry Andric #include "memprof_thread.h" 23e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_allocator_checks.h" 24e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_allocator_interface.h" 25e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_allocator_report.h" 26e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_errno.h" 27e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_file.h" 28e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_flags.h" 29e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_internal_defs.h" 30e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_list.h" 31349cc55cSDimitry Andric #include "sanitizer_common/sanitizer_procmaps.h" 32e8d8bef9SDimitry Andric #include "sanitizer_common/sanitizer_stackdepot.h" 33349cc55cSDimitry Andric #include "sanitizer_common/sanitizer_vector.h" 34e8d8bef9SDimitry Andric 35e8d8bef9SDimitry Andric #include <sched.h> 36e8d8bef9SDimitry Andric #include <time.h> 37e8d8bef9SDimitry Andric 38e8d8bef9SDimitry Andric namespace __memprof { 39e8d8bef9SDimitry Andric 40e8d8bef9SDimitry Andric static int GetCpuId(void) { 41e8d8bef9SDimitry Andric // _memprof_preinit is called via the preinit_array, which subsequently calls 42e8d8bef9SDimitry Andric // malloc. Since this is before _dl_init calls VDSO_SETUP, sched_getcpu 43e8d8bef9SDimitry Andric // will seg fault as the address of __vdso_getcpu will be null. 44e8d8bef9SDimitry Andric if (!memprof_init_done) 45e8d8bef9SDimitry Andric return -1; 46e8d8bef9SDimitry Andric return sched_getcpu(); 47e8d8bef9SDimitry Andric } 48e8d8bef9SDimitry Andric 49e8d8bef9SDimitry Andric // Compute the timestamp in ms. 50e8d8bef9SDimitry Andric static int GetTimestamp(void) { 51e8d8bef9SDimitry Andric // timespec_get will segfault if called from dl_init 52e8d8bef9SDimitry Andric if (!memprof_timestamp_inited) { 53e8d8bef9SDimitry Andric // By returning 0, this will be effectively treated as being 54e8d8bef9SDimitry Andric // timestamped at memprof init time (when memprof_init_timestamp_s 55e8d8bef9SDimitry Andric // is initialized). 56e8d8bef9SDimitry Andric return 0; 57e8d8bef9SDimitry Andric } 58e8d8bef9SDimitry Andric timespec ts; 59e8d8bef9SDimitry Andric clock_gettime(CLOCK_REALTIME, &ts); 60e8d8bef9SDimitry Andric return (ts.tv_sec - memprof_init_timestamp_s) * 1000 + ts.tv_nsec / 1000000; 61e8d8bef9SDimitry Andric } 62e8d8bef9SDimitry Andric 63e8d8bef9SDimitry Andric static MemprofAllocator &get_allocator(); 64e8d8bef9SDimitry Andric 65e8d8bef9SDimitry Andric // The memory chunk allocated from the underlying allocator looks like this: 66e8d8bef9SDimitry Andric // H H U U U U U U 67e8d8bef9SDimitry Andric // H -- ChunkHeader (32 bytes) 68e8d8bef9SDimitry Andric // U -- user memory. 69e8d8bef9SDimitry Andric 70e8d8bef9SDimitry Andric // If there is left padding before the ChunkHeader (due to use of memalign), 71e8d8bef9SDimitry Andric // we store a magic value in the first uptr word of the memory block and 72e8d8bef9SDimitry Andric // store the address of ChunkHeader in the next uptr. 73e8d8bef9SDimitry Andric // M B L L L L L L L L L H H U U U U U U 74e8d8bef9SDimitry Andric // | ^ 75e8d8bef9SDimitry Andric // ---------------------| 76e8d8bef9SDimitry Andric // M -- magic value kAllocBegMagic 77e8d8bef9SDimitry Andric // B -- address of ChunkHeader pointing to the first 'H' 78e8d8bef9SDimitry Andric 79e8d8bef9SDimitry Andric constexpr uptr kMaxAllowedMallocBits = 40; 80e8d8bef9SDimitry Andric 81e8d8bef9SDimitry Andric // Should be no more than 32-bytes 82e8d8bef9SDimitry Andric struct ChunkHeader { 83e8d8bef9SDimitry Andric // 1-st 4 bytes. 84e8d8bef9SDimitry Andric u32 alloc_context_id; 85e8d8bef9SDimitry Andric // 2-nd 4 bytes 86e8d8bef9SDimitry Andric u32 cpu_id; 87e8d8bef9SDimitry Andric // 3-rd 4 bytes 88e8d8bef9SDimitry Andric u32 timestamp_ms; 89e8d8bef9SDimitry Andric // 4-th 4 bytes 90e8d8bef9SDimitry Andric // Note only 1 bit is needed for this flag if we need space in the future for 91e8d8bef9SDimitry Andric // more fields. 92e8d8bef9SDimitry Andric u32 from_memalign; 93e8d8bef9SDimitry Andric // 5-th and 6-th 4 bytes 94e8d8bef9SDimitry Andric // The max size of an allocation is 2^40 (kMaxAllowedMallocSize), so this 95e8d8bef9SDimitry Andric // could be shrunk to kMaxAllowedMallocBits if we need space in the future for 96e8d8bef9SDimitry Andric // more fields. 97e8d8bef9SDimitry Andric atomic_uint64_t user_requested_size; 98e8d8bef9SDimitry Andric // 23 bits available 99e8d8bef9SDimitry Andric // 7-th and 8-th 4 bytes 100e8d8bef9SDimitry Andric u64 data_type_id; // TODO: hash of type name 101e8d8bef9SDimitry Andric }; 102e8d8bef9SDimitry Andric 103e8d8bef9SDimitry Andric static const uptr kChunkHeaderSize = sizeof(ChunkHeader); 104e8d8bef9SDimitry Andric COMPILER_CHECK(kChunkHeaderSize == 32); 105e8d8bef9SDimitry Andric 106e8d8bef9SDimitry Andric struct MemprofChunk : ChunkHeader { 107e8d8bef9SDimitry Andric uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; } 108e8d8bef9SDimitry Andric uptr UsedSize() { 109e8d8bef9SDimitry Andric return atomic_load(&user_requested_size, memory_order_relaxed); 110e8d8bef9SDimitry Andric } 111e8d8bef9SDimitry Andric void *AllocBeg() { 112e8d8bef9SDimitry Andric if (from_memalign) 113e8d8bef9SDimitry Andric return get_allocator().GetBlockBegin(reinterpret_cast<void *>(this)); 114e8d8bef9SDimitry Andric return reinterpret_cast<void *>(this); 115e8d8bef9SDimitry Andric } 116e8d8bef9SDimitry Andric }; 117e8d8bef9SDimitry Andric 118e8d8bef9SDimitry Andric class LargeChunkHeader { 119e8d8bef9SDimitry Andric static constexpr uptr kAllocBegMagic = 120e8d8bef9SDimitry Andric FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL); 121e8d8bef9SDimitry Andric atomic_uintptr_t magic; 122e8d8bef9SDimitry Andric MemprofChunk *chunk_header; 123e8d8bef9SDimitry Andric 124e8d8bef9SDimitry Andric public: 125e8d8bef9SDimitry Andric MemprofChunk *Get() const { 126e8d8bef9SDimitry Andric return atomic_load(&magic, memory_order_acquire) == kAllocBegMagic 127e8d8bef9SDimitry Andric ? chunk_header 128e8d8bef9SDimitry Andric : nullptr; 129e8d8bef9SDimitry Andric } 130e8d8bef9SDimitry Andric 131e8d8bef9SDimitry Andric void Set(MemprofChunk *p) { 132e8d8bef9SDimitry Andric if (p) { 133e8d8bef9SDimitry Andric chunk_header = p; 134e8d8bef9SDimitry Andric atomic_store(&magic, kAllocBegMagic, memory_order_release); 135e8d8bef9SDimitry Andric return; 136e8d8bef9SDimitry Andric } 137e8d8bef9SDimitry Andric 138e8d8bef9SDimitry Andric uptr old = kAllocBegMagic; 139e8d8bef9SDimitry Andric if (!atomic_compare_exchange_strong(&magic, &old, 0, 140e8d8bef9SDimitry Andric memory_order_release)) { 141e8d8bef9SDimitry Andric CHECK_EQ(old, kAllocBegMagic); 142e8d8bef9SDimitry Andric } 143e8d8bef9SDimitry Andric } 144e8d8bef9SDimitry Andric }; 145e8d8bef9SDimitry Andric 146e8d8bef9SDimitry Andric void FlushUnneededMemProfShadowMemory(uptr p, uptr size) { 147e8d8bef9SDimitry Andric // Since memprof's mapping is compacting, the shadow chunk may be 148e8d8bef9SDimitry Andric // not page-aligned, so we only flush the page-aligned portion. 149e8d8bef9SDimitry Andric ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size)); 150e8d8bef9SDimitry Andric } 151e8d8bef9SDimitry Andric 152e8d8bef9SDimitry Andric void MemprofMapUnmapCallback::OnMap(uptr p, uptr size) const { 153e8d8bef9SDimitry Andric // Statistics. 154e8d8bef9SDimitry Andric MemprofStats &thread_stats = GetCurrentThreadStats(); 155e8d8bef9SDimitry Andric thread_stats.mmaps++; 156e8d8bef9SDimitry Andric thread_stats.mmaped += size; 157e8d8bef9SDimitry Andric } 158e8d8bef9SDimitry Andric void MemprofMapUnmapCallback::OnUnmap(uptr p, uptr size) const { 159e8d8bef9SDimitry Andric // We are about to unmap a chunk of user memory. 160e8d8bef9SDimitry Andric // Mark the corresponding shadow memory as not needed. 161e8d8bef9SDimitry Andric FlushUnneededMemProfShadowMemory(p, size); 162e8d8bef9SDimitry Andric // Statistics. 163e8d8bef9SDimitry Andric MemprofStats &thread_stats = GetCurrentThreadStats(); 164e8d8bef9SDimitry Andric thread_stats.munmaps++; 165e8d8bef9SDimitry Andric thread_stats.munmaped += size; 166e8d8bef9SDimitry Andric } 167e8d8bef9SDimitry Andric 168e8d8bef9SDimitry Andric AllocatorCache *GetAllocatorCache(MemprofThreadLocalMallocStorage *ms) { 169e8d8bef9SDimitry Andric CHECK(ms); 170e8d8bef9SDimitry Andric return &ms->allocator_cache; 171e8d8bef9SDimitry Andric } 172e8d8bef9SDimitry Andric 173e8d8bef9SDimitry Andric // Accumulates the access count from the shadow for the given pointer and size. 174e8d8bef9SDimitry Andric u64 GetShadowCount(uptr p, u32 size) { 175e8d8bef9SDimitry Andric u64 *shadow = (u64 *)MEM_TO_SHADOW(p); 176e8d8bef9SDimitry Andric u64 *shadow_end = (u64 *)MEM_TO_SHADOW(p + size); 177e8d8bef9SDimitry Andric u64 count = 0; 178e8d8bef9SDimitry Andric for (; shadow <= shadow_end; shadow++) 179e8d8bef9SDimitry Andric count += *shadow; 180e8d8bef9SDimitry Andric return count; 181e8d8bef9SDimitry Andric } 182e8d8bef9SDimitry Andric 183e8d8bef9SDimitry Andric // Clears the shadow counters (when memory is allocated). 184e8d8bef9SDimitry Andric void ClearShadow(uptr addr, uptr size) { 185e8d8bef9SDimitry Andric CHECK(AddrIsAlignedByGranularity(addr)); 186e8d8bef9SDimitry Andric CHECK(AddrIsInMem(addr)); 187e8d8bef9SDimitry Andric CHECK(AddrIsAlignedByGranularity(addr + size)); 188e8d8bef9SDimitry Andric CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY)); 189e8d8bef9SDimitry Andric CHECK(REAL(memset)); 190e8d8bef9SDimitry Andric uptr shadow_beg = MEM_TO_SHADOW(addr); 191e8d8bef9SDimitry Andric uptr shadow_end = MEM_TO_SHADOW(addr + size - SHADOW_GRANULARITY) + 1; 192e8d8bef9SDimitry Andric if (shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { 193e8d8bef9SDimitry Andric REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); 194e8d8bef9SDimitry Andric } else { 195e8d8bef9SDimitry Andric uptr page_size = GetPageSizeCached(); 196e8d8bef9SDimitry Andric uptr page_beg = RoundUpTo(shadow_beg, page_size); 197e8d8bef9SDimitry Andric uptr page_end = RoundDownTo(shadow_end, page_size); 198e8d8bef9SDimitry Andric 199e8d8bef9SDimitry Andric if (page_beg >= page_end) { 200e8d8bef9SDimitry Andric REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); 201e8d8bef9SDimitry Andric } else { 202e8d8bef9SDimitry Andric if (page_beg != shadow_beg) { 203e8d8bef9SDimitry Andric REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg); 204e8d8bef9SDimitry Andric } 205e8d8bef9SDimitry Andric if (page_end != shadow_end) { 206e8d8bef9SDimitry Andric REAL(memset)((void *)page_end, 0, shadow_end - page_end); 207e8d8bef9SDimitry Andric } 208e8d8bef9SDimitry Andric ReserveShadowMemoryRange(page_beg, page_end - 1, nullptr); 209e8d8bef9SDimitry Andric } 210e8d8bef9SDimitry Andric } 211e8d8bef9SDimitry Andric } 212e8d8bef9SDimitry Andric 213e8d8bef9SDimitry Andric struct Allocator { 214e8d8bef9SDimitry Andric static const uptr kMaxAllowedMallocSize = 1ULL << kMaxAllowedMallocBits; 215e8d8bef9SDimitry Andric 216e8d8bef9SDimitry Andric MemprofAllocator allocator; 217e8d8bef9SDimitry Andric StaticSpinMutex fallback_mutex; 218e8d8bef9SDimitry Andric AllocatorCache fallback_allocator_cache; 219e8d8bef9SDimitry Andric 220e8d8bef9SDimitry Andric uptr max_user_defined_malloc_size; 221e8d8bef9SDimitry Andric atomic_uint8_t rss_limit_exceeded; 222e8d8bef9SDimitry Andric 223349cc55cSDimitry Andric // Holds the mapping of stack ids to MemInfoBlocks. 224349cc55cSDimitry Andric MIBMapTy MIBMap; 225349cc55cSDimitry Andric 226349cc55cSDimitry Andric atomic_uint8_t destructing; 227349cc55cSDimitry Andric atomic_uint8_t constructed; 228349cc55cSDimitry Andric bool print_text; 229e8d8bef9SDimitry Andric 230e8d8bef9SDimitry Andric // ------------------- Initialization ------------------------ 231349cc55cSDimitry Andric explicit Allocator(LinkerInitialized) : print_text(flags()->print_text) { 232349cc55cSDimitry Andric atomic_store_relaxed(&destructing, 0); 233349cc55cSDimitry Andric atomic_store_relaxed(&constructed, 1); 234349cc55cSDimitry Andric } 235e8d8bef9SDimitry Andric 236349cc55cSDimitry Andric ~Allocator() { 237349cc55cSDimitry Andric atomic_store_relaxed(&destructing, 1); 238349cc55cSDimitry Andric FinishAndWrite(); 239349cc55cSDimitry Andric } 240e8d8bef9SDimitry Andric 241349cc55cSDimitry Andric static void PrintCallback(const uptr Key, LockedMemInfoBlock *const &Value, 242349cc55cSDimitry Andric void *Arg) { 243349cc55cSDimitry Andric SpinMutexLock(&Value->mutex); 244349cc55cSDimitry Andric Value->mib.Print(Key, bool(Arg)); 245349cc55cSDimitry Andric } 246349cc55cSDimitry Andric 247349cc55cSDimitry Andric void FinishAndWrite() { 248349cc55cSDimitry Andric if (print_text && common_flags()->print_module_map) 249349cc55cSDimitry Andric DumpProcessMap(); 250349cc55cSDimitry Andric 251e8d8bef9SDimitry Andric allocator.ForceLock(); 252349cc55cSDimitry Andric 253349cc55cSDimitry Andric InsertLiveBlocks(); 254349cc55cSDimitry Andric if (print_text) { 255*4824e7fdSDimitry Andric if (!flags()->print_terse) 256*4824e7fdSDimitry Andric Printf("Recorded MIBs (incl. live on exit):\n"); 257349cc55cSDimitry Andric MIBMap.ForEach(PrintCallback, 258349cc55cSDimitry Andric reinterpret_cast<void *>(flags()->print_terse)); 259349cc55cSDimitry Andric StackDepotPrintAll(); 260349cc55cSDimitry Andric } else { 261349cc55cSDimitry Andric // Serialize the contents to a raw profile. Format documented in 262349cc55cSDimitry Andric // memprof_rawprofile.h. 263349cc55cSDimitry Andric char *Buffer = nullptr; 264349cc55cSDimitry Andric 265349cc55cSDimitry Andric MemoryMappingLayout Layout(/*cache_enabled=*/true); 266349cc55cSDimitry Andric u64 BytesSerialized = SerializeToRawProfile(MIBMap, Layout, Buffer); 267349cc55cSDimitry Andric CHECK(Buffer && BytesSerialized && "could not serialize to buffer"); 268349cc55cSDimitry Andric report_file.Write(Buffer, BytesSerialized); 269349cc55cSDimitry Andric } 270349cc55cSDimitry Andric 271349cc55cSDimitry Andric allocator.ForceUnlock(); 272349cc55cSDimitry Andric } 273349cc55cSDimitry Andric 274349cc55cSDimitry Andric // Inserts any blocks which have been allocated but not yet deallocated. 275349cc55cSDimitry Andric void InsertLiveBlocks() { 276e8d8bef9SDimitry Andric allocator.ForEachChunk( 277e8d8bef9SDimitry Andric [](uptr chunk, void *alloc) { 278e8d8bef9SDimitry Andric u64 user_requested_size; 279349cc55cSDimitry Andric Allocator *A = (Allocator *)alloc; 280e8d8bef9SDimitry Andric MemprofChunk *m = 281349cc55cSDimitry Andric A->GetMemprofChunk((void *)chunk, user_requested_size); 282e8d8bef9SDimitry Andric if (!m) 283e8d8bef9SDimitry Andric return; 284e8d8bef9SDimitry Andric uptr user_beg = ((uptr)m) + kChunkHeaderSize; 285e8d8bef9SDimitry Andric u64 c = GetShadowCount(user_beg, user_requested_size); 286e8d8bef9SDimitry Andric long curtime = GetTimestamp(); 287e8d8bef9SDimitry Andric MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime, 288e8d8bef9SDimitry Andric m->cpu_id, GetCpuId()); 289349cc55cSDimitry Andric InsertOrMerge(m->alloc_context_id, newMIB, A->MIBMap); 290e8d8bef9SDimitry Andric }, 291e8d8bef9SDimitry Andric this); 292e8d8bef9SDimitry Andric } 293e8d8bef9SDimitry Andric 294e8d8bef9SDimitry Andric void InitLinkerInitialized() { 295e8d8bef9SDimitry Andric SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); 296e8d8bef9SDimitry Andric allocator.InitLinkerInitialized( 297e8d8bef9SDimitry Andric common_flags()->allocator_release_to_os_interval_ms); 298e8d8bef9SDimitry Andric max_user_defined_malloc_size = common_flags()->max_allocation_size_mb 299e8d8bef9SDimitry Andric ? common_flags()->max_allocation_size_mb 300e8d8bef9SDimitry Andric << 20 301e8d8bef9SDimitry Andric : kMaxAllowedMallocSize; 302e8d8bef9SDimitry Andric } 303e8d8bef9SDimitry Andric 304e8d8bef9SDimitry Andric bool RssLimitExceeded() { 305e8d8bef9SDimitry Andric return atomic_load(&rss_limit_exceeded, memory_order_relaxed); 306e8d8bef9SDimitry Andric } 307e8d8bef9SDimitry Andric 308e8d8bef9SDimitry Andric void SetRssLimitExceeded(bool limit_exceeded) { 309e8d8bef9SDimitry Andric atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed); 310e8d8bef9SDimitry Andric } 311e8d8bef9SDimitry Andric 312e8d8bef9SDimitry Andric // -------------------- Allocation/Deallocation routines --------------- 313e8d8bef9SDimitry Andric void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, 314e8d8bef9SDimitry Andric AllocType alloc_type) { 315e8d8bef9SDimitry Andric if (UNLIKELY(!memprof_inited)) 316e8d8bef9SDimitry Andric MemprofInitFromRtl(); 317e8d8bef9SDimitry Andric if (RssLimitExceeded()) { 318e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 319e8d8bef9SDimitry Andric return nullptr; 320e8d8bef9SDimitry Andric ReportRssLimitExceeded(stack); 321e8d8bef9SDimitry Andric } 322e8d8bef9SDimitry Andric CHECK(stack); 323e8d8bef9SDimitry Andric const uptr min_alignment = MEMPROF_ALIGNMENT; 324e8d8bef9SDimitry Andric if (alignment < min_alignment) 325e8d8bef9SDimitry Andric alignment = min_alignment; 326e8d8bef9SDimitry Andric if (size == 0) { 327e8d8bef9SDimitry Andric // We'd be happy to avoid allocating memory for zero-size requests, but 328e8d8bef9SDimitry Andric // some programs/tests depend on this behavior and assume that malloc 329e8d8bef9SDimitry Andric // would not return NULL even for zero-size allocations. Moreover, it 330e8d8bef9SDimitry Andric // looks like operator new should never return NULL, and results of 331e8d8bef9SDimitry Andric // consecutive "new" calls must be different even if the allocated size 332e8d8bef9SDimitry Andric // is zero. 333e8d8bef9SDimitry Andric size = 1; 334e8d8bef9SDimitry Andric } 335e8d8bef9SDimitry Andric CHECK(IsPowerOfTwo(alignment)); 336e8d8bef9SDimitry Andric uptr rounded_size = RoundUpTo(size, alignment); 337e8d8bef9SDimitry Andric uptr needed_size = rounded_size + kChunkHeaderSize; 338e8d8bef9SDimitry Andric if (alignment > min_alignment) 339e8d8bef9SDimitry Andric needed_size += alignment; 340e8d8bef9SDimitry Andric CHECK(IsAligned(needed_size, min_alignment)); 341e8d8bef9SDimitry Andric if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize || 342e8d8bef9SDimitry Andric size > max_user_defined_malloc_size) { 343e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) { 344349cc55cSDimitry Andric Report("WARNING: MemProfiler failed to allocate 0x%zx bytes\n", size); 345e8d8bef9SDimitry Andric return nullptr; 346e8d8bef9SDimitry Andric } 347e8d8bef9SDimitry Andric uptr malloc_limit = 348e8d8bef9SDimitry Andric Min(kMaxAllowedMallocSize, max_user_defined_malloc_size); 349e8d8bef9SDimitry Andric ReportAllocationSizeTooBig(size, malloc_limit, stack); 350e8d8bef9SDimitry Andric } 351e8d8bef9SDimitry Andric 352e8d8bef9SDimitry Andric MemprofThread *t = GetCurrentThread(); 353e8d8bef9SDimitry Andric void *allocated; 354e8d8bef9SDimitry Andric if (t) { 355e8d8bef9SDimitry Andric AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 356e8d8bef9SDimitry Andric allocated = allocator.Allocate(cache, needed_size, 8); 357e8d8bef9SDimitry Andric } else { 358e8d8bef9SDimitry Andric SpinMutexLock l(&fallback_mutex); 359e8d8bef9SDimitry Andric AllocatorCache *cache = &fallback_allocator_cache; 360e8d8bef9SDimitry Andric allocated = allocator.Allocate(cache, needed_size, 8); 361e8d8bef9SDimitry Andric } 362e8d8bef9SDimitry Andric if (UNLIKELY(!allocated)) { 363e8d8bef9SDimitry Andric SetAllocatorOutOfMemory(); 364e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 365e8d8bef9SDimitry Andric return nullptr; 366e8d8bef9SDimitry Andric ReportOutOfMemory(size, stack); 367e8d8bef9SDimitry Andric } 368e8d8bef9SDimitry Andric 369e8d8bef9SDimitry Andric uptr alloc_beg = reinterpret_cast<uptr>(allocated); 370e8d8bef9SDimitry Andric uptr alloc_end = alloc_beg + needed_size; 371e8d8bef9SDimitry Andric uptr beg_plus_header = alloc_beg + kChunkHeaderSize; 372e8d8bef9SDimitry Andric uptr user_beg = beg_plus_header; 373e8d8bef9SDimitry Andric if (!IsAligned(user_beg, alignment)) 374e8d8bef9SDimitry Andric user_beg = RoundUpTo(user_beg, alignment); 375e8d8bef9SDimitry Andric uptr user_end = user_beg + size; 376e8d8bef9SDimitry Andric CHECK_LE(user_end, alloc_end); 377e8d8bef9SDimitry Andric uptr chunk_beg = user_beg - kChunkHeaderSize; 378e8d8bef9SDimitry Andric MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg); 379e8d8bef9SDimitry Andric m->from_memalign = alloc_beg != chunk_beg; 380e8d8bef9SDimitry Andric CHECK(size); 381e8d8bef9SDimitry Andric 382e8d8bef9SDimitry Andric m->cpu_id = GetCpuId(); 383e8d8bef9SDimitry Andric m->timestamp_ms = GetTimestamp(); 384e8d8bef9SDimitry Andric m->alloc_context_id = StackDepotPut(*stack); 385e8d8bef9SDimitry Andric 386e8d8bef9SDimitry Andric uptr size_rounded_down_to_granularity = 387e8d8bef9SDimitry Andric RoundDownTo(size, SHADOW_GRANULARITY); 388e8d8bef9SDimitry Andric if (size_rounded_down_to_granularity) 389e8d8bef9SDimitry Andric ClearShadow(user_beg, size_rounded_down_to_granularity); 390e8d8bef9SDimitry Andric 391e8d8bef9SDimitry Andric MemprofStats &thread_stats = GetCurrentThreadStats(); 392e8d8bef9SDimitry Andric thread_stats.mallocs++; 393e8d8bef9SDimitry Andric thread_stats.malloced += size; 394e8d8bef9SDimitry Andric thread_stats.malloced_overhead += needed_size - size; 395e8d8bef9SDimitry Andric if (needed_size > SizeClassMap::kMaxSize) 396e8d8bef9SDimitry Andric thread_stats.malloc_large++; 397e8d8bef9SDimitry Andric else 398e8d8bef9SDimitry Andric thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++; 399e8d8bef9SDimitry Andric 400e8d8bef9SDimitry Andric void *res = reinterpret_cast<void *>(user_beg); 401e8d8bef9SDimitry Andric atomic_store(&m->user_requested_size, size, memory_order_release); 402e8d8bef9SDimitry Andric if (alloc_beg != chunk_beg) { 403e8d8bef9SDimitry Andric CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg); 404e8d8bef9SDimitry Andric reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m); 405e8d8bef9SDimitry Andric } 406e8d8bef9SDimitry Andric MEMPROF_MALLOC_HOOK(res, size); 407e8d8bef9SDimitry Andric return res; 408e8d8bef9SDimitry Andric } 409e8d8bef9SDimitry Andric 410e8d8bef9SDimitry Andric void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment, 411e8d8bef9SDimitry Andric BufferedStackTrace *stack, AllocType alloc_type) { 412e8d8bef9SDimitry Andric uptr p = reinterpret_cast<uptr>(ptr); 413e8d8bef9SDimitry Andric if (p == 0) 414e8d8bef9SDimitry Andric return; 415e8d8bef9SDimitry Andric 416e8d8bef9SDimitry Andric MEMPROF_FREE_HOOK(ptr); 417e8d8bef9SDimitry Andric 418e8d8bef9SDimitry Andric uptr chunk_beg = p - kChunkHeaderSize; 419e8d8bef9SDimitry Andric MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg); 420e8d8bef9SDimitry Andric 421e8d8bef9SDimitry Andric u64 user_requested_size = 422e8d8bef9SDimitry Andric atomic_exchange(&m->user_requested_size, 0, memory_order_acquire); 423349cc55cSDimitry Andric if (memprof_inited && memprof_init_done && 424349cc55cSDimitry Andric atomic_load_relaxed(&constructed) && 425349cc55cSDimitry Andric !atomic_load_relaxed(&destructing)) { 426e8d8bef9SDimitry Andric u64 c = GetShadowCount(p, user_requested_size); 427e8d8bef9SDimitry Andric long curtime = GetTimestamp(); 428e8d8bef9SDimitry Andric 429e8d8bef9SDimitry Andric MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime, 430e8d8bef9SDimitry Andric m->cpu_id, GetCpuId()); 431349cc55cSDimitry Andric InsertOrMerge(m->alloc_context_id, newMIB, MIBMap); 432e8d8bef9SDimitry Andric } 433e8d8bef9SDimitry Andric 434e8d8bef9SDimitry Andric MemprofStats &thread_stats = GetCurrentThreadStats(); 435e8d8bef9SDimitry Andric thread_stats.frees++; 436e8d8bef9SDimitry Andric thread_stats.freed += user_requested_size; 437e8d8bef9SDimitry Andric 438e8d8bef9SDimitry Andric void *alloc_beg = m->AllocBeg(); 439e8d8bef9SDimitry Andric if (alloc_beg != m) { 440e8d8bef9SDimitry Andric // Clear the magic value, as allocator internals may overwrite the 441e8d8bef9SDimitry Andric // contents of deallocated chunk, confusing GetMemprofChunk lookup. 442e8d8bef9SDimitry Andric reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(nullptr); 443e8d8bef9SDimitry Andric } 444e8d8bef9SDimitry Andric 445e8d8bef9SDimitry Andric MemprofThread *t = GetCurrentThread(); 446e8d8bef9SDimitry Andric if (t) { 447e8d8bef9SDimitry Andric AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 448e8d8bef9SDimitry Andric allocator.Deallocate(cache, alloc_beg); 449e8d8bef9SDimitry Andric } else { 450e8d8bef9SDimitry Andric SpinMutexLock l(&fallback_mutex); 451e8d8bef9SDimitry Andric AllocatorCache *cache = &fallback_allocator_cache; 452e8d8bef9SDimitry Andric allocator.Deallocate(cache, alloc_beg); 453e8d8bef9SDimitry Andric } 454e8d8bef9SDimitry Andric } 455e8d8bef9SDimitry Andric 456e8d8bef9SDimitry Andric void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) { 457e8d8bef9SDimitry Andric CHECK(old_ptr && new_size); 458e8d8bef9SDimitry Andric uptr p = reinterpret_cast<uptr>(old_ptr); 459e8d8bef9SDimitry Andric uptr chunk_beg = p - kChunkHeaderSize; 460e8d8bef9SDimitry Andric MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg); 461e8d8bef9SDimitry Andric 462e8d8bef9SDimitry Andric MemprofStats &thread_stats = GetCurrentThreadStats(); 463e8d8bef9SDimitry Andric thread_stats.reallocs++; 464e8d8bef9SDimitry Andric thread_stats.realloced += new_size; 465e8d8bef9SDimitry Andric 466e8d8bef9SDimitry Andric void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC); 467e8d8bef9SDimitry Andric if (new_ptr) { 468e8d8bef9SDimitry Andric CHECK_NE(REAL(memcpy), nullptr); 469e8d8bef9SDimitry Andric uptr memcpy_size = Min(new_size, m->UsedSize()); 470e8d8bef9SDimitry Andric REAL(memcpy)(new_ptr, old_ptr, memcpy_size); 471e8d8bef9SDimitry Andric Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC); 472e8d8bef9SDimitry Andric } 473e8d8bef9SDimitry Andric return new_ptr; 474e8d8bef9SDimitry Andric } 475e8d8bef9SDimitry Andric 476e8d8bef9SDimitry Andric void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { 477e8d8bef9SDimitry Andric if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 478e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 479e8d8bef9SDimitry Andric return nullptr; 480e8d8bef9SDimitry Andric ReportCallocOverflow(nmemb, size, stack); 481e8d8bef9SDimitry Andric } 482e8d8bef9SDimitry Andric void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC); 483e8d8bef9SDimitry Andric // If the memory comes from the secondary allocator no need to clear it 484e8d8bef9SDimitry Andric // as it comes directly from mmap. 485e8d8bef9SDimitry Andric if (ptr && allocator.FromPrimary(ptr)) 486e8d8bef9SDimitry Andric REAL(memset)(ptr, 0, nmemb * size); 487e8d8bef9SDimitry Andric return ptr; 488e8d8bef9SDimitry Andric } 489e8d8bef9SDimitry Andric 490e8d8bef9SDimitry Andric void CommitBack(MemprofThreadLocalMallocStorage *ms, 491e8d8bef9SDimitry Andric BufferedStackTrace *stack) { 492e8d8bef9SDimitry Andric AllocatorCache *ac = GetAllocatorCache(ms); 493e8d8bef9SDimitry Andric allocator.SwallowCache(ac); 494e8d8bef9SDimitry Andric } 495e8d8bef9SDimitry Andric 496e8d8bef9SDimitry Andric // -------------------------- Chunk lookup ---------------------- 497e8d8bef9SDimitry Andric 498e8d8bef9SDimitry Andric // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). 499e8d8bef9SDimitry Andric MemprofChunk *GetMemprofChunk(void *alloc_beg, u64 &user_requested_size) { 500e8d8bef9SDimitry Andric if (!alloc_beg) 501e8d8bef9SDimitry Andric return nullptr; 502e8d8bef9SDimitry Andric MemprofChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get(); 503e8d8bef9SDimitry Andric if (!p) { 504e8d8bef9SDimitry Andric if (!allocator.FromPrimary(alloc_beg)) 505e8d8bef9SDimitry Andric return nullptr; 506e8d8bef9SDimitry Andric p = reinterpret_cast<MemprofChunk *>(alloc_beg); 507e8d8bef9SDimitry Andric } 508e8d8bef9SDimitry Andric // The size is reset to 0 on deallocation (and a min of 1 on 509e8d8bef9SDimitry Andric // allocation). 510e8d8bef9SDimitry Andric user_requested_size = 511e8d8bef9SDimitry Andric atomic_load(&p->user_requested_size, memory_order_acquire); 512e8d8bef9SDimitry Andric if (user_requested_size) 513e8d8bef9SDimitry Andric return p; 514e8d8bef9SDimitry Andric return nullptr; 515e8d8bef9SDimitry Andric } 516e8d8bef9SDimitry Andric 517e8d8bef9SDimitry Andric MemprofChunk *GetMemprofChunkByAddr(uptr p, u64 &user_requested_size) { 518e8d8bef9SDimitry Andric void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast<void *>(p)); 519e8d8bef9SDimitry Andric return GetMemprofChunk(alloc_beg, user_requested_size); 520e8d8bef9SDimitry Andric } 521e8d8bef9SDimitry Andric 522e8d8bef9SDimitry Andric uptr AllocationSize(uptr p) { 523e8d8bef9SDimitry Andric u64 user_requested_size; 524e8d8bef9SDimitry Andric MemprofChunk *m = GetMemprofChunkByAddr(p, user_requested_size); 525e8d8bef9SDimitry Andric if (!m) 526e8d8bef9SDimitry Andric return 0; 527e8d8bef9SDimitry Andric if (m->Beg() != p) 528e8d8bef9SDimitry Andric return 0; 529e8d8bef9SDimitry Andric return user_requested_size; 530e8d8bef9SDimitry Andric } 531e8d8bef9SDimitry Andric 532e8d8bef9SDimitry Andric void Purge(BufferedStackTrace *stack) { allocator.ForceReleaseToOS(); } 533e8d8bef9SDimitry Andric 534e8d8bef9SDimitry Andric void PrintStats() { allocator.PrintStats(); } 535e8d8bef9SDimitry Andric 536fe6060f1SDimitry Andric void ForceLock() NO_THREAD_SAFETY_ANALYSIS { 537e8d8bef9SDimitry Andric allocator.ForceLock(); 538e8d8bef9SDimitry Andric fallback_mutex.Lock(); 539e8d8bef9SDimitry Andric } 540e8d8bef9SDimitry Andric 541fe6060f1SDimitry Andric void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS { 542e8d8bef9SDimitry Andric fallback_mutex.Unlock(); 543e8d8bef9SDimitry Andric allocator.ForceUnlock(); 544e8d8bef9SDimitry Andric } 545e8d8bef9SDimitry Andric }; 546e8d8bef9SDimitry Andric 547e8d8bef9SDimitry Andric static Allocator instance(LINKER_INITIALIZED); 548e8d8bef9SDimitry Andric 549e8d8bef9SDimitry Andric static MemprofAllocator &get_allocator() { return instance.allocator; } 550e8d8bef9SDimitry Andric 551e8d8bef9SDimitry Andric void InitializeAllocator() { instance.InitLinkerInitialized(); } 552e8d8bef9SDimitry Andric 553e8d8bef9SDimitry Andric void MemprofThreadLocalMallocStorage::CommitBack() { 554e8d8bef9SDimitry Andric GET_STACK_TRACE_MALLOC; 555e8d8bef9SDimitry Andric instance.CommitBack(this, &stack); 556e8d8bef9SDimitry Andric } 557e8d8bef9SDimitry Andric 558e8d8bef9SDimitry Andric void PrintInternalAllocatorStats() { instance.PrintStats(); } 559e8d8bef9SDimitry Andric 560e8d8bef9SDimitry Andric void memprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { 561e8d8bef9SDimitry Andric instance.Deallocate(ptr, 0, 0, stack, alloc_type); 562e8d8bef9SDimitry Andric } 563e8d8bef9SDimitry Andric 564e8d8bef9SDimitry Andric void memprof_delete(void *ptr, uptr size, uptr alignment, 565e8d8bef9SDimitry Andric BufferedStackTrace *stack, AllocType alloc_type) { 566e8d8bef9SDimitry Andric instance.Deallocate(ptr, size, alignment, stack, alloc_type); 567e8d8bef9SDimitry Andric } 568e8d8bef9SDimitry Andric 569e8d8bef9SDimitry Andric void *memprof_malloc(uptr size, BufferedStackTrace *stack) { 570e8d8bef9SDimitry Andric return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC)); 571e8d8bef9SDimitry Andric } 572e8d8bef9SDimitry Andric 573e8d8bef9SDimitry Andric void *memprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { 574e8d8bef9SDimitry Andric return SetErrnoOnNull(instance.Calloc(nmemb, size, stack)); 575e8d8bef9SDimitry Andric } 576e8d8bef9SDimitry Andric 577e8d8bef9SDimitry Andric void *memprof_reallocarray(void *p, uptr nmemb, uptr size, 578e8d8bef9SDimitry Andric BufferedStackTrace *stack) { 579e8d8bef9SDimitry Andric if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 580e8d8bef9SDimitry Andric errno = errno_ENOMEM; 581e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 582e8d8bef9SDimitry Andric return nullptr; 583e8d8bef9SDimitry Andric ReportReallocArrayOverflow(nmemb, size, stack); 584e8d8bef9SDimitry Andric } 585e8d8bef9SDimitry Andric return memprof_realloc(p, nmemb * size, stack); 586e8d8bef9SDimitry Andric } 587e8d8bef9SDimitry Andric 588e8d8bef9SDimitry Andric void *memprof_realloc(void *p, uptr size, BufferedStackTrace *stack) { 589e8d8bef9SDimitry Andric if (!p) 590e8d8bef9SDimitry Andric return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC)); 591e8d8bef9SDimitry Andric if (size == 0) { 592e8d8bef9SDimitry Andric if (flags()->allocator_frees_and_returns_null_on_realloc_zero) { 593e8d8bef9SDimitry Andric instance.Deallocate(p, 0, 0, stack, FROM_MALLOC); 594e8d8bef9SDimitry Andric return nullptr; 595e8d8bef9SDimitry Andric } 596e8d8bef9SDimitry Andric // Allocate a size of 1 if we shouldn't free() on Realloc to 0 597e8d8bef9SDimitry Andric size = 1; 598e8d8bef9SDimitry Andric } 599e8d8bef9SDimitry Andric return SetErrnoOnNull(instance.Reallocate(p, size, stack)); 600e8d8bef9SDimitry Andric } 601e8d8bef9SDimitry Andric 602e8d8bef9SDimitry Andric void *memprof_valloc(uptr size, BufferedStackTrace *stack) { 603e8d8bef9SDimitry Andric return SetErrnoOnNull( 604e8d8bef9SDimitry Andric instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC)); 605e8d8bef9SDimitry Andric } 606e8d8bef9SDimitry Andric 607e8d8bef9SDimitry Andric void *memprof_pvalloc(uptr size, BufferedStackTrace *stack) { 608e8d8bef9SDimitry Andric uptr PageSize = GetPageSizeCached(); 609e8d8bef9SDimitry Andric if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { 610e8d8bef9SDimitry Andric errno = errno_ENOMEM; 611e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 612e8d8bef9SDimitry Andric return nullptr; 613e8d8bef9SDimitry Andric ReportPvallocOverflow(size, stack); 614e8d8bef9SDimitry Andric } 615e8d8bef9SDimitry Andric // pvalloc(0) should allocate one page. 616e8d8bef9SDimitry Andric size = size ? RoundUpTo(size, PageSize) : PageSize; 617e8d8bef9SDimitry Andric return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC)); 618e8d8bef9SDimitry Andric } 619e8d8bef9SDimitry Andric 620e8d8bef9SDimitry Andric void *memprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, 621e8d8bef9SDimitry Andric AllocType alloc_type) { 622e8d8bef9SDimitry Andric if (UNLIKELY(!IsPowerOfTwo(alignment))) { 623e8d8bef9SDimitry Andric errno = errno_EINVAL; 624e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 625e8d8bef9SDimitry Andric return nullptr; 626e8d8bef9SDimitry Andric ReportInvalidAllocationAlignment(alignment, stack); 627e8d8bef9SDimitry Andric } 628e8d8bef9SDimitry Andric return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type)); 629e8d8bef9SDimitry Andric } 630e8d8bef9SDimitry Andric 631e8d8bef9SDimitry Andric void *memprof_aligned_alloc(uptr alignment, uptr size, 632e8d8bef9SDimitry Andric BufferedStackTrace *stack) { 633e8d8bef9SDimitry Andric if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { 634e8d8bef9SDimitry Andric errno = errno_EINVAL; 635e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 636e8d8bef9SDimitry Andric return nullptr; 637e8d8bef9SDimitry Andric ReportInvalidAlignedAllocAlignment(size, alignment, stack); 638e8d8bef9SDimitry Andric } 639e8d8bef9SDimitry Andric return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC)); 640e8d8bef9SDimitry Andric } 641e8d8bef9SDimitry Andric 642e8d8bef9SDimitry Andric int memprof_posix_memalign(void **memptr, uptr alignment, uptr size, 643e8d8bef9SDimitry Andric BufferedStackTrace *stack) { 644e8d8bef9SDimitry Andric if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { 645e8d8bef9SDimitry Andric if (AllocatorMayReturnNull()) 646e8d8bef9SDimitry Andric return errno_EINVAL; 647e8d8bef9SDimitry Andric ReportInvalidPosixMemalignAlignment(alignment, stack); 648e8d8bef9SDimitry Andric } 649e8d8bef9SDimitry Andric void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC); 650e8d8bef9SDimitry Andric if (UNLIKELY(!ptr)) 651e8d8bef9SDimitry Andric // OOM error is already taken care of by Allocate. 652e8d8bef9SDimitry Andric return errno_ENOMEM; 653e8d8bef9SDimitry Andric CHECK(IsAligned((uptr)ptr, alignment)); 654e8d8bef9SDimitry Andric *memptr = ptr; 655e8d8bef9SDimitry Andric return 0; 656e8d8bef9SDimitry Andric } 657e8d8bef9SDimitry Andric 658e8d8bef9SDimitry Andric uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp) { 659e8d8bef9SDimitry Andric if (!ptr) 660e8d8bef9SDimitry Andric return 0; 661e8d8bef9SDimitry Andric uptr usable_size = instance.AllocationSize(reinterpret_cast<uptr>(ptr)); 662e8d8bef9SDimitry Andric return usable_size; 663e8d8bef9SDimitry Andric } 664e8d8bef9SDimitry Andric 665e8d8bef9SDimitry Andric void MemprofSoftRssLimitExceededCallback(bool limit_exceeded) { 666e8d8bef9SDimitry Andric instance.SetRssLimitExceeded(limit_exceeded); 667e8d8bef9SDimitry Andric } 668e8d8bef9SDimitry Andric 669e8d8bef9SDimitry Andric } // namespace __memprof 670e8d8bef9SDimitry Andric 671e8d8bef9SDimitry Andric // ---------------------- Interface ---------------- {{{1 672e8d8bef9SDimitry Andric using namespace __memprof; 673e8d8bef9SDimitry Andric 674e8d8bef9SDimitry Andric #if !SANITIZER_SUPPORTS_WEAK_HOOKS 675e8d8bef9SDimitry Andric // Provide default (no-op) implementation of malloc hooks. 676e8d8bef9SDimitry Andric SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, void *ptr, 677e8d8bef9SDimitry Andric uptr size) { 678e8d8bef9SDimitry Andric (void)ptr; 679e8d8bef9SDimitry Andric (void)size; 680e8d8bef9SDimitry Andric } 681e8d8bef9SDimitry Andric 682e8d8bef9SDimitry Andric SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) { 683e8d8bef9SDimitry Andric (void)ptr; 684e8d8bef9SDimitry Andric } 685e8d8bef9SDimitry Andric #endif 686e8d8bef9SDimitry Andric 687e8d8bef9SDimitry Andric uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } 688e8d8bef9SDimitry Andric 689e8d8bef9SDimitry Andric int __sanitizer_get_ownership(const void *p) { 690e8d8bef9SDimitry Andric return memprof_malloc_usable_size(p, 0, 0) != 0; 691e8d8bef9SDimitry Andric } 692e8d8bef9SDimitry Andric 693e8d8bef9SDimitry Andric uptr __sanitizer_get_allocated_size(const void *p) { 694e8d8bef9SDimitry Andric return memprof_malloc_usable_size(p, 0, 0); 695e8d8bef9SDimitry Andric } 696e8d8bef9SDimitry Andric 697e8d8bef9SDimitry Andric int __memprof_profile_dump() { 698349cc55cSDimitry Andric instance.FinishAndWrite(); 699e8d8bef9SDimitry Andric // In the future we may want to return non-zero if there are any errors 700e8d8bef9SDimitry Andric // detected during the dumping process. 701e8d8bef9SDimitry Andric return 0; 702e8d8bef9SDimitry Andric } 703