1 // -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
2 // Copyright (c) 2005, Google Inc.
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // ---
32 // Author: Sanjay Ghemawat <opensource@google.com>
33 //
34 // A malloc that uses a per-thread cache to satisfy small malloc requests.
35 // (The time for malloc/free of a small object drops from 300 ns to 50 ns.)
36 //
37 // See docs/tcmalloc.html for a high-level
38 // description of how this malloc works.
39 //
40 // SYNCHRONIZATION
41 // 1. The thread-specific lists are accessed without acquiring any locks.
42 // This is safe because each such list is only accessed by one thread.
43 // 2. We have a lock per central free-list, and hold it while manipulating
44 // the central free list for a particular size.
45 // 3. The central page allocator is protected by "pageheap_lock".
46 // 4. The pagemap (which maps from page-number to descriptor),
47 // can be read without holding any locks, and written while holding
48 // the "pageheap_lock".
49 // 5. To improve performance, a subset of the information one can get
50 // from the pagemap is cached in a data structure, pagemap_cache_,
51 // that atomically reads and writes its entries. This cache can be
52 // read and written without locking.
53 //
54 // This multi-threaded access to the pagemap is safe for fairly
55 // subtle reasons. We basically assume that when an object X is
56 // allocated by thread A and deallocated by thread B, there must
57 // have been appropriate synchronization in the handoff of object
58 // X from thread A to thread B. The same logic applies to pagemap_cache_.
59 //
60 // THE PAGEID-TO-SIZECLASS CACHE
61 // Hot PageID-to-sizeclass mappings are held by pagemap_cache_. If this cache
62 // returns 0 for a particular PageID then that means "no information," not that
63 // the sizeclass is 0. The cache may have stale information for pages that do
64 // not hold the beginning of any free()'able object. Staleness is eliminated
65 // in Populate() for pages with sizeclass > 0 objects, and in do_malloc() and
66 // do_memalign() for all other relevant pages.
67 //
68 // PAGEMAP
69 // -------
70 // Page map contains a mapping from page id to Span.
71 //
72 // If Span s occupies pages [p..q],
73 // pagemap[p] == s
74 // pagemap[q] == s
75 // pagemap[p+1..q-1] are undefined
76 // pagemap[p-1] and pagemap[q+1] are defined:
77 // NULL if the corresponding page is not yet in the address space.
78 // Otherwise it points to a Span. This span may be free
79 // or allocated. If free, it is in one of pageheap's freelist.
80 //
81 // TODO: Bias reclamation to larger addresses
82 // TODO: implement mallinfo/mallopt
83 // TODO: Better testing
84 //
85 // 9/28/2003 (new page-level allocator replaces ptmalloc2):
86 // * malloc/free of small objects goes from ~300 ns to ~50 ns.
87 // * allocation of a reasonably complicated struct
88 // goes from about 1100 ns to about 300 ns.
89
90 #include "config.h"
91 // At least for gcc on Linux/i386 and Linux/amd64 not adding throw()
92 // to tc_xxx functions actually ends up generating better code.
93 #define PERFTOOLS_NOTHROW
94 #include <gperftools/tcmalloc.h>
95
96 #include <errno.h> // for ENOMEM, EINVAL, errno
97 #if defined HAVE_STDINT_H
98 #include <stdint.h>
99 #elif defined HAVE_INTTYPES_H
100 #include <inttypes.h>
101 #else
102 #include <sys/types.h>
103 #endif
104 #include <stddef.h> // for size_t, NULL
105 #include <stdlib.h> // for getenv
106 #include <string.h> // for strcmp, memset, strlen, etc
107 #ifdef HAVE_UNISTD_H
108 #include <unistd.h> // for getpagesize, write, etc
109 #endif
110 #include <algorithm> // for max, min
111 #include <limits> // for numeric_limits
112 #include <new> // for nothrow_t (ptr only), etc
113 #include <vector> // for vector
114
115 #include <gperftools/malloc_extension.h>
116 #include <gperftools/malloc_hook.h> // for MallocHook
117 #include <gperftools/nallocx.h>
118 #include "base/basictypes.h" // for int64
119 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc
120 #include "base/dynamic_annotations.h" // for RunningOnValgrind
121 #include "base/spinlock.h" // for SpinLockHolder
122 #include "central_freelist.h" // for CentralFreeListPadded
123 #include "common.h" // for StackTrace, kPageShift, etc
124 #include "free_list.h" // for FL_Init
125 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc
126 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc
127 #include "page_heap.h" // for PageHeap, PageHeap::Stats
128 #include "page_heap_allocator.h" // for PageHeapAllocator
129 #include "sampler.h" // for Sampler
130 #include "span.h" // for Span, DLL_Prepend, etc
131 #include "stack_trace_table.h" // for StackTraceTable
132 #include "static_vars.h" // for Static
133 #include "system-alloc.h" // for DumpSystemAllocatorStats, etc
134 #include "tcmalloc_guard.h" // for TCMallocGuard
135 #include "thread_cache.h" // for ThreadCache
136
137 #include "maybe_emergency_malloc.h"
138
139 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defined(WIN32_OVERRIDE_ALLOCATORS)
140 # define WIN32_DO_PATCHING 1
141 #endif
142
143 // Some windows file somewhere (at least on cygwin) #define's small (!)
144 #undef small
145
146 using STL_NAMESPACE::max;
147 using STL_NAMESPACE::min;
148 using STL_NAMESPACE::numeric_limits;
149 using STL_NAMESPACE::vector;
150
151 #include "libc_override.h"
152
153 using tcmalloc::AlignmentForSize;
154 using tcmalloc::kCrash;
155 using tcmalloc::kCrashWithStats;
156 using tcmalloc::kLog;
157 using tcmalloc::Log;
158 using tcmalloc::PageHeap;
159 using tcmalloc::PageHeapAllocator;
160 using tcmalloc::Sampler;
161 using tcmalloc::SizeMap;
162 using tcmalloc::Span;
163 using tcmalloc::StackTrace;
164 using tcmalloc::Static;
165 using tcmalloc::ThreadCache;
166
167 DECLARE_double(tcmalloc_release_rate);
168
169 // Those common architectures are known to be safe w.r.t. aliasing function
170 // with "extra" unused args to function with fewer arguments (e.g.
171 // tc_delete_nothrow being aliased to tc_delete).
172 //
173 // Benefit of aliasing is relatively moderate. It reduces instruction
174 // cache pressure a bit (not relevant for largely unused
175 // tc_delete_nothrow, but is potentially relevant for
176 // tc_delete_aligned (or sized)). It also used to be the case that gcc
177 // 5+ optimization for merging identical functions kicked in and
178 // "screwed" one of the otherwise identical functions with extra
179 // jump. I am not able to reproduce that anymore.
180 #if !defined(__i386__) && !defined(__x86_64__) && \
181 !defined(__ppc__) && !defined(__PPC__) && \
182 !defined(__aarch64__) && !defined(__mips__) && !defined(__arm__)
183 #undef TCMALLOC_NO_ALIASES
184 #define TCMALLOC_NO_ALIASES
185 #endif
186
187 #if defined(__GNUC__) && defined(__ELF__) && !defined(TCMALLOC_NO_ALIASES)
188 #define TC_ALIAS(name) __attribute__((alias(#name)))
189 #endif
190
191 // For windows, the printf we use to report large allocs is
192 // potentially dangerous: it could cause a malloc that would cause an
193 // infinite loop. So by default we set the threshold to a huge number
194 // on windows, so this bad situation will never trigger. You can
195 // always set TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD manually if you
196 // want this functionality.
197 #ifdef _WIN32
198 const int64 kDefaultLargeAllocReportThreshold = static_cast<int64>(1) << 62;
199 #else
200 const int64 kDefaultLargeAllocReportThreshold = static_cast<int64>(1) << 30;
201 #endif
202 DEFINE_int64(tcmalloc_large_alloc_report_threshold,
203 EnvToInt64("TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD",
204 kDefaultLargeAllocReportThreshold),
205 "Allocations larger than this value cause a stack "
206 "trace to be dumped to stderr. The threshold for "
207 "dumping stack traces is increased by a factor of 1.125 "
208 "every time we print a message so that the threshold "
209 "automatically goes up by a factor of ~1000 every 60 "
210 "messages. This bounds the amount of extra logging "
211 "generated by this flag. Default value of this flag "
212 "is very large and therefore you should see no extra "
213 "logging unless the flag is overridden. Set to 0 to "
214 "disable reporting entirely.");
215
216
217 // We already declared these functions in tcmalloc.h, but we have to
218 // declare them again to give them an ATTRIBUTE_SECTION: we want to
219 // put all callers of MallocHook::Invoke* in this module into
220 // ATTRIBUTE_SECTION(google_malloc) section, so that
221 // MallocHook::GetCallerStackTrace can function accurately.
222 #ifndef _WIN32 // windows doesn't have attribute_section, so don't bother
223 extern "C" {
224 void* tc_malloc(size_t size) PERFTOOLS_NOTHROW
225 ATTRIBUTE_SECTION(google_malloc);
226 void tc_free(void* ptr) PERFTOOLS_NOTHROW
227 ATTRIBUTE_SECTION(google_malloc);
228 void tc_free_sized(void* ptr, size_t size) PERFTOOLS_NOTHROW
229 ATTRIBUTE_SECTION(google_malloc);
230 void* tc_realloc(void* ptr, size_t size) PERFTOOLS_NOTHROW
231 ATTRIBUTE_SECTION(google_malloc);
232 void* tc_calloc(size_t nmemb, size_t size) PERFTOOLS_NOTHROW
233 ATTRIBUTE_SECTION(google_malloc);
234 void tc_cfree(void* ptr) PERFTOOLS_NOTHROW
235 ATTRIBUTE_SECTION(google_malloc);
236
237 void* tc_memalign(size_t __alignment, size_t __size) PERFTOOLS_NOTHROW
238 ATTRIBUTE_SECTION(google_malloc);
239 int tc_posix_memalign(void** ptr, size_t align, size_t size) PERFTOOLS_NOTHROW
240 ATTRIBUTE_SECTION(google_malloc);
241 void* tc_valloc(size_t __size) PERFTOOLS_NOTHROW
242 ATTRIBUTE_SECTION(google_malloc);
243 void* tc_pvalloc(size_t __size) PERFTOOLS_NOTHROW
244 ATTRIBUTE_SECTION(google_malloc);
245
246 void tc_malloc_stats(void) PERFTOOLS_NOTHROW
247 ATTRIBUTE_SECTION(google_malloc);
248 int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW
249 ATTRIBUTE_SECTION(google_malloc);
250 #ifdef HAVE_STRUCT_MALLINFO
251 struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW
252 ATTRIBUTE_SECTION(google_malloc);
253 #endif
254
255 void* tc_new(size_t size)
256 ATTRIBUTE_SECTION(google_malloc);
257 void tc_delete(void* p) PERFTOOLS_NOTHROW
258 ATTRIBUTE_SECTION(google_malloc);
259 void tc_delete_sized(void* p, size_t size) PERFTOOLS_NOTHROW
260 ATTRIBUTE_SECTION(google_malloc);
261 void* tc_newarray(size_t size)
262 ATTRIBUTE_SECTION(google_malloc);
263 void tc_deletearray(void* p) PERFTOOLS_NOTHROW
264 ATTRIBUTE_SECTION(google_malloc);
265 void tc_deletearray_sized(void* p, size_t size) PERFTOOLS_NOTHROW
266 ATTRIBUTE_SECTION(google_malloc);
267
268 // And the nothrow variants of these:
269 void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW
270 ATTRIBUTE_SECTION(google_malloc);
271 void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW
272 ATTRIBUTE_SECTION(google_malloc);
273 // Surprisingly, standard C++ library implementations use a
274 // nothrow-delete internally. See, eg:
275 // http://www.dinkumware.com/manuals/?manual=compleat&page=new.html
276 void tc_delete_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_NOTHROW
277 ATTRIBUTE_SECTION(google_malloc);
278 void tc_deletearray_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_NOTHROW
279 ATTRIBUTE_SECTION(google_malloc);
280
281 #if defined(ENABLE_ALIGNED_NEW_DELETE)
282
283 void* tc_new_aligned(size_t size, std::align_val_t al)
284 ATTRIBUTE_SECTION(google_malloc);
285 void tc_delete_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW
286 ATTRIBUTE_SECTION(google_malloc);
287 void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW
288 ATTRIBUTE_SECTION(google_malloc);
289 void* tc_newarray_aligned(size_t size, std::align_val_t al)
290 ATTRIBUTE_SECTION(google_malloc);
291 void tc_deletearray_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW
292 ATTRIBUTE_SECTION(google_malloc);
293 void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW
294 ATTRIBUTE_SECTION(google_malloc);
295
296 // And the nothrow variants of these:
297 void* tc_new_aligned_nothrow(size_t size, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
298 ATTRIBUTE_SECTION(google_malloc);
299 void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
300 ATTRIBUTE_SECTION(google_malloc);
301 void tc_delete_aligned_nothrow(void* ptr, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
302 ATTRIBUTE_SECTION(google_malloc);
303 void tc_deletearray_aligned_nothrow(void* ptr, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
304 ATTRIBUTE_SECTION(google_malloc);
305
306 #endif // defined(ENABLE_ALIGNED_NEW_DELETE)
307
308 // Some non-standard extensions that we support.
309
310 // This is equivalent to
311 // OS X: malloc_size()
312 // glibc: malloc_usable_size()
313 // Windows: _msize()
314 size_t tc_malloc_size(void* p) PERFTOOLS_NOTHROW
315 ATTRIBUTE_SECTION(google_malloc);
316
317 void* tc_malloc_skip_new_handler(size_t size)
318 ATTRIBUTE_SECTION(google_malloc);
319 } // extern "C"
320 #endif // #ifndef _WIN32
321
322 // ----------------------- IMPLEMENTATION -------------------------------
323
324 static int tc_new_mode = 0; // See tc_set_new_mode().
325
326 // Routines such as free() and realloc() catch some erroneous pointers
327 // passed to them, and invoke the below when they do. (An erroneous pointer
328 // won't be caught if it's within a valid span or a stale span for which
329 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing
330 // required) kind of exception handling for these routines.
331 namespace {
InvalidFree(void * ptr)332 ATTRIBUTE_NOINLINE void InvalidFree(void* ptr) {
333 if (tcmalloc::IsEmergencyPtr(ptr)) {
334 tcmalloc::EmergencyFree(ptr);
335 return;
336 }
337 Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr);
338 }
339
InvalidGetSizeForRealloc(const void * old_ptr)340 size_t InvalidGetSizeForRealloc(const void* old_ptr) {
341 Log(kCrash, __FILE__, __LINE__,
342 "Attempt to realloc invalid pointer", old_ptr);
343 return 0;
344 }
345
InvalidGetAllocatedSize(const void * ptr)346 size_t InvalidGetAllocatedSize(const void* ptr) {
347 Log(kCrash, __FILE__, __LINE__,
348 "Attempt to get the size of an invalid pointer", ptr);
349 return 0;
350 }
351
352 #if defined(TCMALLOC_DISABLE_HUGE_ALLOCATIONS)
353 // For security reasons, we want to limit the size of allocations.
354 // See crbug.com/169327.
IsAllocSizePermitted(size_t alloc_size)355 inline bool IsAllocSizePermitted(size_t alloc_size) {
356 // Never allow an allocation larger than what can be indexed via an int.
357 // Remove kPageSize to account for various rounding, padding and to have a
358 // small margin.
359 return alloc_size <= ((std::numeric_limits<int>::max)() - kPageSize);
360 }
361 #endif
362
363 } // unnamed namespace
364
365 // Extract interesting stats
366 struct TCMallocStats {
367 uint64_t thread_bytes; // Bytes in thread caches
368 uint64_t central_bytes; // Bytes in central cache
369 uint64_t transfer_bytes; // Bytes in central transfer cache
370 uint64_t metadata_bytes; // Bytes alloced for metadata
371 PageHeap::Stats pageheap; // Stats from page heap
372 };
373
374 // Get stats into "r". Also, if class_count != NULL, class_count[k]
375 // will be set to the total number of objects of size class k in the
376 // central cache, transfer cache, and per-thread caches. If small_spans
377 // is non-NULL, it is filled. Same for large_spans.
ExtractStats(TCMallocStats * r,uint64_t * class_count,PageHeap::SmallSpanStats * small_spans,PageHeap::LargeSpanStats * large_spans)378 static void ExtractStats(TCMallocStats* r, uint64_t* class_count,
379 PageHeap::SmallSpanStats* small_spans,
380 PageHeap::LargeSpanStats* large_spans) {
381 r->central_bytes = 0;
382 r->transfer_bytes = 0;
383 for (int cl = 0; cl < Static::num_size_classes(); ++cl) {
384 const int length = Static::central_cache()[cl].length();
385 const int tc_length = Static::central_cache()[cl].tc_length();
386 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes();
387 const size_t size = static_cast<uint64_t>(
388 Static::sizemap()->ByteSizeForClass(cl));
389 r->central_bytes += (size * length) + cache_overhead;
390 r->transfer_bytes += (size * tc_length);
391 if (class_count) {
392 // Sum the lengths of all per-class freelists, except the per-thread
393 // freelists, which get counted when we call GetThreadStats(), below.
394 class_count[cl] = length + tc_length;
395 }
396
397 }
398
399 // Add stats from per-thread heaps
400 r->thread_bytes = 0;
401 { // scope
402 SpinLockHolder h(Static::pageheap_lock());
403 ThreadCache::GetThreadStats(&r->thread_bytes, class_count);
404 r->metadata_bytes = tcmalloc::metadata_system_bytes();
405 r->pageheap = Static::pageheap()->stats();
406 if (small_spans != NULL) {
407 Static::pageheap()->GetSmallSpanStats(small_spans);
408 }
409 if (large_spans != NULL) {
410 Static::pageheap()->GetLargeSpanStats(large_spans);
411 }
412 }
413 }
414
PagesToMiB(uint64_t pages)415 static double PagesToMiB(uint64_t pages) {
416 return (pages << kPageShift) / 1048576.0;
417 }
418
419 // WRITE stats to "out"
DumpStats(TCMalloc_Printer * out,int level)420 static void DumpStats(TCMalloc_Printer* out, int level) {
421 TCMallocStats stats;
422 uint64_t class_count[kClassSizesMax];
423 PageHeap::SmallSpanStats small;
424 PageHeap::LargeSpanStats large;
425 if (level >= 2) {
426 ExtractStats(&stats, class_count, &small, &large);
427 } else {
428 ExtractStats(&stats, NULL, NULL, NULL);
429 }
430
431 static const double MiB = 1048576.0;
432
433 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes
434 + stats.metadata_bytes);
435 const uint64_t physical_memory_used = (virtual_memory_used
436 - stats.pageheap.unmapped_bytes);
437 const uint64_t bytes_in_use_by_app = (physical_memory_used
438 - stats.metadata_bytes
439 - stats.pageheap.free_bytes
440 - stats.central_bytes
441 - stats.transfer_bytes
442 - stats.thread_bytes);
443
444 #ifdef TCMALLOC_SMALL_BUT_SLOW
445 out->printf(
446 "NOTE: SMALL MEMORY MODEL IS IN USE, PERFORMANCE MAY SUFFER.\n");
447 #endif
448 out->printf(
449 "------------------------------------------------\n"
450 "MALLOC: %12" PRIu64 " (%7.1f MiB) Bytes in use by application\n"
451 "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in page heap freelist\n"
452 "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in central cache freelist\n"
453 "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in transfer cache freelist\n"
454 "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in thread cache freelists\n"
455 "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in malloc metadata\n"
456 "MALLOC: ------------\n"
457 "MALLOC: = %12" PRIu64 " (%7.1f MiB) Actual memory used (physical + swap)\n"
458 "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes released to OS (aka unmapped)\n"
459 "MALLOC: ------------\n"
460 "MALLOC: = %12" PRIu64 " (%7.1f MiB) Virtual address space used\n"
461 "MALLOC:\n"
462 "MALLOC: %12" PRIu64 " Spans in use\n"
463 "MALLOC: %12" PRIu64 " Thread heaps in use\n"
464 "MALLOC: %12" PRIu64 " Tcmalloc page size\n"
465 "------------------------------------------------\n"
466 "Call ReleaseFreeMemory() to release freelist memory to the OS"
467 " (via madvise()).\n"
468 "Bytes released to the OS take up virtual address space"
469 " but no physical memory.\n",
470 bytes_in_use_by_app, bytes_in_use_by_app / MiB,
471 stats.pageheap.free_bytes, stats.pageheap.free_bytes / MiB,
472 stats.central_bytes, stats.central_bytes / MiB,
473 stats.transfer_bytes, stats.transfer_bytes / MiB,
474 stats.thread_bytes, stats.thread_bytes / MiB,
475 stats.metadata_bytes, stats.metadata_bytes / MiB,
476 physical_memory_used, physical_memory_used / MiB,
477 stats.pageheap.unmapped_bytes, stats.pageheap.unmapped_bytes / MiB,
478 virtual_memory_used, virtual_memory_used / MiB,
479 uint64_t(Static::span_allocator()->inuse()),
480 uint64_t(ThreadCache::HeapsInUse()),
481 uint64_t(kPageSize));
482
483 if (level >= 2) {
484 out->printf("------------------------------------------------\n");
485 out->printf("Total size of freelists for per-thread caches,\n");
486 out->printf("transfer cache, and central cache, by size class\n");
487 out->printf("------------------------------------------------\n");
488 uint64_t cumulative = 0;
489 for (uint32 cl = 0; cl < Static::num_size_classes(); ++cl) {
490 if (class_count[cl] > 0) {
491 size_t cl_size = Static::sizemap()->ByteSizeForClass(cl);
492 uint64_t class_bytes = class_count[cl] * cl_size;
493 cumulative += class_bytes;
494 out->printf("class %3d [ %8" PRIuS " bytes ] : "
495 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n",
496 cl, cl_size,
497 class_count[cl],
498 class_bytes / MiB,
499 cumulative / MiB);
500 }
501 }
502
503 // append page heap info
504 int nonempty_sizes = 0;
505 for (int s = 0; s < kMaxPages; s++) {
506 if (small.normal_length[s] + small.returned_length[s] > 0) {
507 nonempty_sizes++;
508 }
509 }
510 out->printf("------------------------------------------------\n");
511 out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n",
512 nonempty_sizes, stats.pageheap.free_bytes / MiB,
513 stats.pageheap.unmapped_bytes / MiB);
514 out->printf("------------------------------------------------\n");
515 uint64_t total_normal = 0;
516 uint64_t total_returned = 0;
517 for (int s = 1; s <= kMaxPages; s++) {
518 const int n_length = small.normal_length[s - 1];
519 const int r_length = small.returned_length[s - 1];
520 if (n_length + r_length > 0) {
521 uint64_t n_pages = s * n_length;
522 uint64_t r_pages = s * r_length;
523 total_normal += n_pages;
524 total_returned += r_pages;
525 out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
526 "; unmapped: %6.1f MiB; %6.1f MiB cum\n",
527 s,
528 (n_length + r_length),
529 PagesToMiB(n_pages + r_pages),
530 PagesToMiB(total_normal + total_returned),
531 PagesToMiB(r_pages),
532 PagesToMiB(total_returned));
533 }
534 }
535
536 total_normal += large.normal_pages;
537 total_returned += large.returned_pages;
538 out->printf(">%-5u large * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
539 "; unmapped: %6.1f MiB; %6.1f MiB cum\n",
540 static_cast<unsigned int>(kMaxPages),
541 static_cast<unsigned int>(large.spans),
542 PagesToMiB(large.normal_pages + large.returned_pages),
543 PagesToMiB(total_normal + total_returned),
544 PagesToMiB(large.returned_pages),
545 PagesToMiB(total_returned));
546 }
547 }
548
PrintStats(int level)549 static void PrintStats(int level) {
550 const int kBufferSize = 16 << 10;
551 char* buffer = new char[kBufferSize];
552 TCMalloc_Printer printer(buffer, kBufferSize);
553 DumpStats(&printer, level);
554 write(STDERR_FILENO, buffer, strlen(buffer));
555 delete[] buffer;
556 }
557
DumpHeapGrowthStackTraces()558 static void** DumpHeapGrowthStackTraces() {
559 // Count how much space we need
560 int needed_slots = 0;
561 {
562 SpinLockHolder h(Static::pageheap_lock());
563 for (StackTrace* t = Static::growth_stacks();
564 t != NULL;
565 t = reinterpret_cast<StackTrace*>(
566 t->stack[tcmalloc::kMaxStackDepth-1])) {
567 needed_slots += 3 + t->depth;
568 }
569 needed_slots += 100; // Slop in case list grows
570 needed_slots += needed_slots/8; // An extra 12.5% slop
571 }
572
573 void** result = new void*[needed_slots];
574 if (result == NULL) {
575 Log(kLog, __FILE__, __LINE__,
576 "tcmalloc: allocation failed for stack trace slots",
577 needed_slots * sizeof(*result));
578 return NULL;
579 }
580
581 SpinLockHolder h(Static::pageheap_lock());
582 int used_slots = 0;
583 for (StackTrace* t = Static::growth_stacks();
584 t != NULL;
585 t = reinterpret_cast<StackTrace*>(
586 t->stack[tcmalloc::kMaxStackDepth-1])) {
587 ASSERT(used_slots < needed_slots); // Need to leave room for terminator
588 if (used_slots + 3 + t->depth >= needed_slots) {
589 // No more room
590 break;
591 }
592
593 result[used_slots+0] = reinterpret_cast<void*>(static_cast<uintptr_t>(1));
594 result[used_slots+1] = reinterpret_cast<void*>(t->size);
595 result[used_slots+2] = reinterpret_cast<void*>(t->depth);
596 for (int d = 0; d < t->depth; d++) {
597 result[used_slots+3+d] = t->stack[d];
598 }
599 used_slots += 3 + t->depth;
600 }
601 result[used_slots] = reinterpret_cast<void*>(static_cast<uintptr_t>(0));
602 return result;
603 }
604
IterateOverRanges(void * arg,MallocExtension::RangeFunction func)605 static void IterateOverRanges(void* arg, MallocExtension::RangeFunction func) {
606 PageID page = 1; // Some code may assume that page==0 is never used
607 bool done = false;
608 while (!done) {
609 // Accumulate a small number of ranges in a local buffer
610 static const int kNumRanges = 16;
611 static base::MallocRange ranges[kNumRanges];
612 int n = 0;
613 {
614 SpinLockHolder h(Static::pageheap_lock());
615 while (n < kNumRanges) {
616 if (!Static::pageheap()->GetNextRange(page, &ranges[n])) {
617 done = true;
618 break;
619 } else {
620 uintptr_t limit = ranges[n].address + ranges[n].length;
621 page = (limit + kPageSize - 1) >> kPageShift;
622 n++;
623 }
624 }
625 }
626
627 for (int i = 0; i < n; i++) {
628 (*func)(arg, &ranges[i]);
629 }
630 }
631 }
632
633 // TCMalloc's support for extra malloc interfaces
634 class TCMallocImplementation : public MallocExtension {
635 private:
636 // ReleaseToSystem() might release more than the requested bytes because
637 // the page heap releases at the span granularity, and spans are of wildly
638 // different sizes. This member keeps track of the extra bytes bytes
639 // released so that the app can periodically call ReleaseToSystem() to
640 // release memory at a constant rate.
641 // NOTE: Protected by Static::pageheap_lock().
642 size_t extra_bytes_released_;
643
644 public:
TCMallocImplementation()645 TCMallocImplementation()
646 : extra_bytes_released_(0) {
647 }
648
GetStats(char * buffer,int buffer_length)649 virtual void GetStats(char* buffer, int buffer_length) {
650 ASSERT(buffer_length > 0);
651 TCMalloc_Printer printer(buffer, buffer_length);
652
653 // Print level one stats unless lots of space is available
654 if (buffer_length < 10000) {
655 DumpStats(&printer, 1);
656 } else {
657 DumpStats(&printer, 2);
658 }
659 }
660
661 // We may print an extra, tcmalloc-specific warning message here.
GetHeapSample(MallocExtensionWriter * writer)662 virtual void GetHeapSample(MallocExtensionWriter* writer) {
663 if (Sampler::GetSamplePeriod() == 0) {
664 const char* const kWarningMsg =
665 "%warn\n"
666 "%warn This heap profile does not have any data in it, because\n"
667 "%warn the application was run with heap sampling turned off.\n"
668 "%warn To get useful data from GetHeapSample(), you must\n"
669 "%warn set the environment variable TCMALLOC_SAMPLE_PARAMETER to\n"
670 "%warn a positive sampling period, such as 524288.\n"
671 "%warn\n";
672 writer->append(kWarningMsg, strlen(kWarningMsg));
673 }
674 MallocExtension::GetHeapSample(writer);
675 }
676
ReadStackTraces(int * sample_period)677 virtual void** ReadStackTraces(int* sample_period) {
678 tcmalloc::StackTraceTable table;
679 {
680 SpinLockHolder h(Static::pageheap_lock());
681 Span* sampled = Static::sampled_objects();
682 for (Span* s = sampled->next; s != sampled; s = s->next) {
683 table.AddTrace(*reinterpret_cast<StackTrace*>(s->objects));
684 }
685 }
686 *sample_period = ThreadCache::GetCache()->GetSamplePeriod();
687 return table.ReadStackTracesAndClear(); // grabs and releases pageheap_lock
688 }
689
ReadHeapGrowthStackTraces()690 virtual void** ReadHeapGrowthStackTraces() {
691 return DumpHeapGrowthStackTraces();
692 }
693
GetThreadCacheSize()694 virtual size_t GetThreadCacheSize() {
695 ThreadCache* tc = ThreadCache::GetCacheIfPresent();
696 if (!tc)
697 return 0;
698 return tc->Size();
699 }
700
MarkThreadTemporarilyIdle()701 virtual void MarkThreadTemporarilyIdle() {
702 ThreadCache::BecomeTemporarilyIdle();
703 }
704
Ranges(void * arg,RangeFunction func)705 virtual void Ranges(void* arg, RangeFunction func) {
706 IterateOverRanges(arg, func);
707 }
708
GetNumericProperty(const char * name,size_t * value)709 virtual bool GetNumericProperty(const char* name, size_t* value) {
710 ASSERT(name != NULL);
711
712 if (strcmp(name, "generic.current_allocated_bytes") == 0) {
713 TCMallocStats stats;
714 ExtractStats(&stats, NULL, NULL, NULL);
715 *value = stats.pageheap.system_bytes
716 - stats.thread_bytes
717 - stats.central_bytes
718 - stats.transfer_bytes
719 - stats.pageheap.free_bytes
720 - stats.pageheap.unmapped_bytes;
721 return true;
722 }
723
724 if (strcmp(name, "generic.heap_size") == 0) {
725 TCMallocStats stats;
726 ExtractStats(&stats, NULL, NULL, NULL);
727 *value = stats.pageheap.system_bytes;
728 return true;
729 }
730
731 if (strcmp(name, "generic.total_physical_bytes") == 0) {
732 TCMallocStats stats;
733 ExtractStats(&stats, NULL, NULL, NULL);
734 *value = stats.pageheap.system_bytes + stats.metadata_bytes -
735 stats.pageheap.unmapped_bytes;
736 return true;
737 }
738
739 if (strcmp(name, "tcmalloc.slack_bytes") == 0) {
740 // Kept for backwards compatibility. Now defined externally as:
741 // pageheap_free_bytes + pageheap_unmapped_bytes.
742 SpinLockHolder l(Static::pageheap_lock());
743 PageHeap::Stats stats = Static::pageheap()->stats();
744 *value = stats.free_bytes + stats.unmapped_bytes;
745 return true;
746 }
747
748 if (strcmp(name, "tcmalloc.central_cache_free_bytes") == 0) {
749 TCMallocStats stats;
750 ExtractStats(&stats, NULL, NULL, NULL);
751 *value = stats.central_bytes;
752 return true;
753 }
754
755 if (strcmp(name, "tcmalloc.transfer_cache_free_bytes") == 0) {
756 TCMallocStats stats;
757 ExtractStats(&stats, NULL, NULL, NULL);
758 *value = stats.transfer_bytes;
759 return true;
760 }
761
762 if (strcmp(name, "tcmalloc.thread_cache_free_bytes") == 0) {
763 TCMallocStats stats;
764 ExtractStats(&stats, NULL, NULL, NULL);
765 *value = stats.thread_bytes;
766 return true;
767 }
768
769 if (strcmp(name, "tcmalloc.pageheap_free_bytes") == 0) {
770 SpinLockHolder l(Static::pageheap_lock());
771 *value = Static::pageheap()->stats().free_bytes;
772 return true;
773 }
774
775 if (strcmp(name, "tcmalloc.pageheap_unmapped_bytes") == 0) {
776 SpinLockHolder l(Static::pageheap_lock());
777 *value = Static::pageheap()->stats().unmapped_bytes;
778 return true;
779 }
780
781 if (strcmp(name, "tcmalloc.pageheap_committed_bytes") == 0) {
782 SpinLockHolder l(Static::pageheap_lock());
783 *value = Static::pageheap()->stats().committed_bytes;
784 return true;
785 }
786
787 if (strcmp(name, "tcmalloc.pageheap_scavenge_count") == 0) {
788 SpinLockHolder l(Static::pageheap_lock());
789 *value = Static::pageheap()->stats().scavenge_count;
790 return true;
791 }
792
793 if (strcmp(name, "tcmalloc.pageheap_commit_count") == 0) {
794 SpinLockHolder l(Static::pageheap_lock());
795 *value = Static::pageheap()->stats().commit_count;
796 return true;
797 }
798
799 if (strcmp(name, "tcmalloc.pageheap_total_commit_bytes") == 0) {
800 SpinLockHolder l(Static::pageheap_lock());
801 *value = Static::pageheap()->stats().total_commit_bytes;
802 return true;
803 }
804
805 if (strcmp(name, "tcmalloc.pageheap_decommit_count") == 0) {
806 SpinLockHolder l(Static::pageheap_lock());
807 *value = Static::pageheap()->stats().decommit_count;
808 return true;
809 }
810
811 if (strcmp(name, "tcmalloc.pageheap_total_decommit_bytes") == 0) {
812 SpinLockHolder l(Static::pageheap_lock());
813 *value = Static::pageheap()->stats().total_decommit_bytes;
814 return true;
815 }
816
817 if (strcmp(name, "tcmalloc.pageheap_reserve_count") == 0) {
818 SpinLockHolder l(Static::pageheap_lock());
819 *value = Static::pageheap()->stats().reserve_count;
820 return true;
821 }
822
823 if (strcmp(name, "tcmalloc.pageheap_total_reserve_bytes") == 0) {
824 SpinLockHolder l(Static::pageheap_lock());
825 *value = Static::pageheap()->stats().total_reserve_bytes;
826 return true;
827 }
828
829 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) {
830 SpinLockHolder l(Static::pageheap_lock());
831 *value = ThreadCache::overall_thread_cache_size();
832 return true;
833 }
834
835 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) {
836 TCMallocStats stats;
837 ExtractStats(&stats, NULL, NULL, NULL);
838 *value = stats.thread_bytes;
839 return true;
840 }
841
842 if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) {
843 SpinLockHolder l(Static::pageheap_lock());
844 *value = size_t(Static::pageheap()->GetAggressiveDecommit());
845 return true;
846 }
847
848 if (strcmp(name, "tcmalloc.sampling_period_bytes") == 0) {
849 *value = Sampler::GetSamplePeriod();
850 return true;
851 }
852
853 return false;
854 }
855
SetNumericProperty(const char * name,size_t value)856 virtual bool SetNumericProperty(const char* name, size_t value) {
857 ASSERT(name != NULL);
858
859 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) {
860 SpinLockHolder l(Static::pageheap_lock());
861 ThreadCache::set_overall_thread_cache_size(value);
862 return true;
863 }
864
865 if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) {
866 SpinLockHolder l(Static::pageheap_lock());
867 Static::pageheap()->SetAggressiveDecommit(value != 0);
868 return true;
869 }
870
871 if (strcmp(name, "tcmalloc.sampling_period_bytes") == 0) {
872 Sampler::SetSamplePeriod(value);
873 return true;
874 }
875
876 return false;
877 }
878
MarkThreadIdle()879 virtual void MarkThreadIdle() {
880 ThreadCache::BecomeIdle();
881 }
882
883 virtual void MarkThreadBusy(); // Implemented below
884
GetSystemAllocator()885 virtual SysAllocator* GetSystemAllocator() {
886 SpinLockHolder h(Static::pageheap_lock());
887 return tcmalloc_sys_alloc;
888 }
889
SetSystemAllocator(SysAllocator * alloc)890 virtual void SetSystemAllocator(SysAllocator* alloc) {
891 SpinLockHolder h(Static::pageheap_lock());
892 tcmalloc_sys_alloc = alloc;
893 }
894
ReleaseToSystem(size_t num_bytes)895 virtual void ReleaseToSystem(size_t num_bytes) {
896 SpinLockHolder h(Static::pageheap_lock());
897 if (num_bytes <= extra_bytes_released_) {
898 // We released too much on a prior call, so don't release any
899 // more this time.
900 extra_bytes_released_ = extra_bytes_released_ - num_bytes;
901 return;
902 }
903 num_bytes = num_bytes - extra_bytes_released_;
904 // num_bytes might be less than one page. If we pass zero to
905 // ReleaseAtLeastNPages, it won't do anything, so we release a whole
906 // page now and let extra_bytes_released_ smooth it out over time.
907 Length num_pages = max<Length>(num_bytes >> kPageShift, 1);
908 size_t bytes_released = Static::pageheap()->ReleaseAtLeastNPages(
909 num_pages) << kPageShift;
910 if (bytes_released > num_bytes) {
911 extra_bytes_released_ = bytes_released - num_bytes;
912 } else {
913 // The PageHeap wasn't able to release num_bytes. Don't try to
914 // compensate with a big release next time. Specifically,
915 // ReleaseFreeMemory() calls ReleaseToSystem(LONG_MAX).
916 extra_bytes_released_ = 0;
917 }
918 }
919
SetMemoryReleaseRate(double rate)920 virtual void SetMemoryReleaseRate(double rate) {
921 FLAGS_tcmalloc_release_rate = rate;
922 }
923
GetMemoryReleaseRate()924 virtual double GetMemoryReleaseRate() {
925 return FLAGS_tcmalloc_release_rate;
926 }
927 virtual size_t GetEstimatedAllocatedSize(size_t size);
928
929 // This just calls GetSizeWithCallback, but because that's in an
930 // unnamed namespace, we need to move the definition below it in the
931 // file.
932 virtual size_t GetAllocatedSize(const void* ptr);
933
934 // This duplicates some of the logic in GetSizeWithCallback, but is
935 // faster. This is important on OS X, where this function is called
936 // on every allocation operation.
GetOwnership(const void * ptr)937 virtual Ownership GetOwnership(const void* ptr) {
938 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
939 // The rest of tcmalloc assumes that all allocated pointers use at
940 // most kAddressBits bits. If ptr doesn't, then it definitely
941 // wasn't alloacted by tcmalloc.
942 if ((p >> (kAddressBits - kPageShift)) > 0) {
943 return kNotOwned;
944 }
945 uint32 cl;
946 if (Static::pageheap()->TryGetSizeClass(p, &cl)) {
947 return kOwned;
948 }
949 const Span *span = Static::pageheap()->GetDescriptor(p);
950 return span ? kOwned : kNotOwned;
951 }
952
GetFreeListSizes(vector<MallocExtension::FreeListInfo> * v)953 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {
954 static const char kCentralCacheType[] = "tcmalloc.central";
955 static const char kTransferCacheType[] = "tcmalloc.transfer";
956 static const char kThreadCacheType[] = "tcmalloc.thread";
957 static const char kPageHeapType[] = "tcmalloc.page";
958 static const char kPageHeapUnmappedType[] = "tcmalloc.page_unmapped";
959 static const char kLargeSpanType[] = "tcmalloc.large";
960 static const char kLargeUnmappedSpanType[] = "tcmalloc.large_unmapped";
961
962 v->clear();
963
964 // central class information
965 int64 prev_class_size = 0;
966 for (int cl = 1; cl < Static::num_size_classes(); ++cl) {
967 size_t class_size = Static::sizemap()->ByteSizeForClass(cl);
968 MallocExtension::FreeListInfo i;
969 i.min_object_size = prev_class_size + 1;
970 i.max_object_size = class_size;
971 i.total_bytes_free =
972 Static::central_cache()[cl].length() * class_size;
973 i.type = kCentralCacheType;
974 v->push_back(i);
975
976 // transfer cache
977 i.total_bytes_free =
978 Static::central_cache()[cl].tc_length() * class_size;
979 i.type = kTransferCacheType;
980 v->push_back(i);
981
982 prev_class_size = Static::sizemap()->ByteSizeForClass(cl);
983 }
984
985 // Add stats from per-thread heaps
986 uint64_t class_count[kClassSizesMax];
987 memset(class_count, 0, sizeof(class_count));
988 {
989 SpinLockHolder h(Static::pageheap_lock());
990 uint64_t thread_bytes = 0;
991 ThreadCache::GetThreadStats(&thread_bytes, class_count);
992 }
993
994 prev_class_size = 0;
995 for (int cl = 1; cl < Static::num_size_classes(); ++cl) {
996 MallocExtension::FreeListInfo i;
997 i.min_object_size = prev_class_size + 1;
998 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl);
999 i.total_bytes_free =
1000 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl);
1001 i.type = kThreadCacheType;
1002 v->push_back(i);
1003
1004 prev_class_size = Static::sizemap()->ByteSizeForClass(cl);
1005 }
1006
1007 // append page heap info
1008 PageHeap::SmallSpanStats small;
1009 PageHeap::LargeSpanStats large;
1010 {
1011 SpinLockHolder h(Static::pageheap_lock());
1012 Static::pageheap()->GetSmallSpanStats(&small);
1013 Static::pageheap()->GetLargeSpanStats(&large);
1014 }
1015
1016 // large spans: mapped
1017 MallocExtension::FreeListInfo span_info;
1018 span_info.type = kLargeSpanType;
1019 span_info.max_object_size = (numeric_limits<size_t>::max)();
1020 span_info.min_object_size = kMaxPages << kPageShift;
1021 span_info.total_bytes_free = large.normal_pages << kPageShift;
1022 v->push_back(span_info);
1023
1024 // large spans: unmapped
1025 span_info.type = kLargeUnmappedSpanType;
1026 span_info.total_bytes_free = large.returned_pages << kPageShift;
1027 v->push_back(span_info);
1028
1029 // small spans
1030 for (int s = 1; s <= kMaxPages; s++) {
1031 MallocExtension::FreeListInfo i;
1032 i.max_object_size = (s << kPageShift);
1033 i.min_object_size = ((s - 1) << kPageShift);
1034
1035 i.type = kPageHeapType;
1036 i.total_bytes_free = (s << kPageShift) * small.normal_length[s - 1];
1037 v->push_back(i);
1038
1039 i.type = kPageHeapUnmappedType;
1040 i.total_bytes_free = (s << kPageShift) * small.returned_length[s - 1];
1041 v->push_back(i);
1042 }
1043 }
1044 };
1045
1046 static inline ATTRIBUTE_ALWAYS_INLINE
align_size_up(size_t size,size_t align)1047 size_t align_size_up(size_t size, size_t align) {
1048 ASSERT(align <= kPageSize);
1049 size_t new_size = (size + align - 1) & ~(align - 1);
1050 if (PREDICT_FALSE(new_size == 0)) {
1051 // Note, new_size == 0 catches both integer overflow and size
1052 // being 0.
1053 if (size == 0) {
1054 new_size = align;
1055 } else {
1056 new_size = size;
1057 }
1058 }
1059 return new_size;
1060 }
1061
1062 // Puts in *cl size class that is suitable for allocation of size bytes with
1063 // align alignment. Returns true if such size class exists and false otherwise.
size_class_with_alignment(size_t size,size_t align,uint32_t * cl)1064 static bool size_class_with_alignment(size_t size, size_t align, uint32_t* cl) {
1065 if (PREDICT_FALSE(align > kPageSize)) {
1066 return false;
1067 }
1068 size = align_size_up(size, align);
1069 if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, cl))) {
1070 return false;
1071 }
1072 ASSERT((Static::sizemap()->class_to_size(*cl) & (align - 1)) == 0);
1073 return true;
1074 }
1075
1076 // nallocx slow path. Moved to a separate function because
1077 // ThreadCache::InitModule is not inlined which would cause nallocx to
1078 // become non-leaf function with stack frame and stack spills.
nallocx_slow(size_t size,int flags)1079 static ATTRIBUTE_NOINLINE size_t nallocx_slow(size_t size, int flags) {
1080 if (PREDICT_FALSE(!Static::IsInited())) ThreadCache::InitModule();
1081
1082 size_t align = static_cast<size_t>(1ull << (flags & 0x3f));
1083 uint32 cl;
1084 bool ok = size_class_with_alignment(size, align, &cl);
1085 if (ok) {
1086 return Static::sizemap()->ByteSizeForClass(cl);
1087 } else {
1088 return tcmalloc::pages(size) << kPageShift;
1089 }
1090 }
1091
1092 // The nallocx function allocates no memory, but it performs the same size
1093 // computation as the malloc function, and returns the real size of the
1094 // allocation that would result from the equivalent malloc function call.
1095 // nallocx is a malloc extension originally implemented by jemalloc:
1096 // http://www.unix.com/man-page/freebsd/3/nallocx/
1097 extern "C" PERFTOOLS_DLL_DECL
tc_nallocx(size_t size,int flags)1098 size_t tc_nallocx(size_t size, int flags) {
1099 if (PREDICT_FALSE(flags != 0)) {
1100 return nallocx_slow(size, flags);
1101 }
1102 uint32 cl;
1103 // size class 0 is only possible if malloc is not yet initialized
1104 if (Static::sizemap()->GetSizeClass(size, &cl) && cl != 0) {
1105 return Static::sizemap()->ByteSizeForClass(cl);
1106 } else {
1107 return nallocx_slow(size, 0);
1108 }
1109 }
1110
1111 extern "C" PERFTOOLS_DLL_DECL
1112 size_t nallocx(size_t size, int flags)
1113 #ifdef TC_ALIAS
1114 TC_ALIAS(tc_nallocx);
1115 #else
1116 {
1117 return nallocx_slow(size, flags);
1118 }
1119 #endif
1120
1121
GetEstimatedAllocatedSize(size_t size)1122 size_t TCMallocImplementation::GetEstimatedAllocatedSize(size_t size) {
1123 return tc_nallocx(size, 0);
1124 }
1125
1126 // The constructor allocates an object to ensure that initialization
1127 // runs before main(), and therefore we do not have a chance to become
1128 // multi-threaded before initialization. We also create the TSD key
1129 // here. Presumably by the time this constructor runs, glibc is in
1130 // good enough shape to handle pthread_key_create().
1131 //
1132 // The constructor also takes the opportunity to tell STL to use
1133 // tcmalloc. We want to do this early, before construct time, so
1134 // all user STL allocations go through tcmalloc (which works really
1135 // well for STL).
1136 //
1137 // The destructor prints stats when the program exits.
1138 static int tcmallocguard_refcount = 0; // no lock needed: runs before main()
TCMallocGuard()1139 TCMallocGuard::TCMallocGuard() {
1140 if (tcmallocguard_refcount++ == 0) {
1141 ReplaceSystemAlloc(); // defined in libc_override_*.h
1142 tc_free(tc_malloc(1));
1143 ThreadCache::InitTSD();
1144 tc_free(tc_malloc(1));
1145 // Either we, or debugallocation.cc, or valgrind will control memory
1146 // management. We register our extension if we're the winner.
1147 #ifdef TCMALLOC_USING_DEBUGALLOCATION
1148 // Let debugallocation register its extension.
1149 #else
1150 if (RunningOnValgrind()) {
1151 // Let Valgrind uses its own malloc (so don't register our extension).
1152 } else {
1153 MallocExtension::Register(new TCMallocImplementation);
1154 }
1155 #endif
1156 }
1157 }
1158
~TCMallocGuard()1159 TCMallocGuard::~TCMallocGuard() {
1160 if (--tcmallocguard_refcount == 0) {
1161 const char* env = NULL;
1162 if (!RunningOnValgrind()) {
1163 // Valgrind uses it's own malloc so we cannot do MALLOCSTATS
1164 env = getenv("MALLOCSTATS");
1165 }
1166 if (env != NULL) {
1167 int level = atoi(env);
1168 if (level < 1) level = 1;
1169 PrintStats(level);
1170 }
1171 }
1172 }
1173 #ifndef WIN32_OVERRIDE_ALLOCATORS
1174 static TCMallocGuard module_enter_exit_hook;
1175 #endif
1176
1177 //-------------------------------------------------------------------
1178 // Helpers for the exported routines below
1179 //-------------------------------------------------------------------
1180
CheckCachedSizeClass(void * ptr)1181 static inline bool CheckCachedSizeClass(void *ptr) {
1182 PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
1183 uint32 cached_value;
1184 if (!Static::pageheap()->TryGetSizeClass(p, &cached_value)) {
1185 return true;
1186 }
1187 return cached_value == Static::pageheap()->GetDescriptor(p)->sizeclass;
1188 }
1189
CheckedMallocResult(void * result)1190 static inline ATTRIBUTE_ALWAYS_INLINE void* CheckedMallocResult(void* result) {
1191 ASSERT(result == NULL || CheckCachedSizeClass(result));
1192 return result;
1193 }
1194
SpanToMallocResult(Span * span)1195 static inline ATTRIBUTE_ALWAYS_INLINE void* SpanToMallocResult(Span *span) {
1196 Static::pageheap()->InvalidateCachedSizeClass(span->start);
1197 return CheckedMallocResult(
1198 reinterpret_cast<void*>(span->start << kPageShift));
1199 }
1200
DoSampledAllocation(size_t size)1201 static void* DoSampledAllocation(size_t size) {
1202 #ifndef NO_TCMALLOC_SAMPLES
1203 // Grab the stack trace outside the heap lock
1204 StackTrace tmp;
1205 tmp.depth = GetStackTrace(tmp.stack, tcmalloc::kMaxStackDepth, 1);
1206 tmp.size = size;
1207
1208 SpinLockHolder h(Static::pageheap_lock());
1209 // Allocate span
1210 Span *span = Static::pageheap()->New(tcmalloc::pages(size == 0 ? 1 : size));
1211 if (PREDICT_FALSE(span == NULL)) {
1212 return NULL;
1213 }
1214
1215 // Allocate stack trace
1216 StackTrace *stack = Static::stacktrace_allocator()->New();
1217 if (PREDICT_FALSE(stack == NULL)) {
1218 // Sampling failed because of lack of memory
1219 return span;
1220 }
1221 *stack = tmp;
1222 span->sample = 1;
1223 span->objects = stack;
1224 tcmalloc::DLL_Prepend(Static::sampled_objects(), span);
1225
1226 return SpanToMallocResult(span);
1227 #else
1228 abort();
1229 #endif
1230 }
1231
1232 namespace {
1233
1234 typedef void* (*malloc_fn)(void *arg);
1235
1236 SpinLock set_new_handler_lock(SpinLock::LINKER_INITIALIZED);
1237
handle_oom(malloc_fn retry_fn,void * retry_arg,bool from_operator,bool nothrow)1238 void* handle_oom(malloc_fn retry_fn,
1239 void* retry_arg,
1240 bool from_operator,
1241 bool nothrow) {
1242 // we hit out of memory condition, usually if it happens we've
1243 // called sbrk or mmap and failed, and thus errno is set. But there
1244 // is support for setting up custom system allocator or setting up
1245 // page heap size limit, in which cases errno may remain
1246 // untouched.
1247 //
1248 // So we set errno here. C++ operator new doesn't require ENOMEM to
1249 // be set, but doesn't forbid it too (and often C++ oom does happen
1250 // with ENOMEM set).
1251 errno = ENOMEM;
1252 if (!from_operator && !tc_new_mode) {
1253 // we're out of memory in C library function (malloc etc) and no
1254 // "new mode" forced on us. Just return NULL
1255 return NULL;
1256 }
1257 // we're OOM in operator new or "new mode" is set. We might have to
1258 // call new_handle and maybe retry allocation.
1259
1260 for (;;) {
1261 // Get the current new handler. NB: this function is not
1262 // thread-safe. We make a feeble stab at making it so here, but
1263 // this lock only protects against tcmalloc interfering with
1264 // itself, not with other libraries calling set_new_handler.
1265 std::new_handler nh;
1266 {
1267 SpinLockHolder h(&set_new_handler_lock);
1268 nh = std::set_new_handler(0);
1269 (void) std::set_new_handler(nh);
1270 }
1271 #if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
1272 if (!nh) {
1273 return NULL;
1274 }
1275 // Since exceptions are disabled, we don't really know if new_handler
1276 // failed. Assume it will abort if it fails.
1277 (*nh)();
1278 #else
1279 // If no new_handler is established, the allocation failed.
1280 if (!nh) {
1281 if (nothrow) {
1282 return NULL;
1283 }
1284 throw std::bad_alloc();
1285 }
1286 // Otherwise, try the new_handler. If it returns, retry the
1287 // allocation. If it throws std::bad_alloc, fail the allocation.
1288 // if it throws something else, don't interfere.
1289 try {
1290 (*nh)();
1291 } catch (const std::bad_alloc&) {
1292 if (!nothrow) throw;
1293 return NULL;
1294 }
1295 #endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
1296
1297 // we get here if new_handler returns successfully. So we retry
1298 // allocation.
1299 void* rv = retry_fn(retry_arg);
1300 if (rv != NULL) {
1301 return rv;
1302 }
1303
1304 // if allocation failed again we go to next loop iteration
1305 }
1306 }
1307
1308 // Copy of FLAGS_tcmalloc_large_alloc_report_threshold with
1309 // automatic increases factored in.
1310 static int64_t large_alloc_threshold =
1311 (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold
1312 ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold);
1313
ReportLargeAlloc(Length num_pages,void * result)1314 static void ReportLargeAlloc(Length num_pages, void* result) {
1315 StackTrace stack;
1316 stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1);
1317
1318 static const int N = 1000;
1319 char buffer[N];
1320 TCMalloc_Printer printer(buffer, N);
1321 printer.printf("tcmalloc: large alloc %" PRIu64 " bytes == %p @ ",
1322 static_cast<uint64>(num_pages) << kPageShift,
1323 result);
1324 for (int i = 0; i < stack.depth; i++) {
1325 printer.printf(" %p", stack.stack[i]);
1326 }
1327 printer.printf("\n");
1328 write(STDERR_FILENO, buffer, strlen(buffer));
1329 }
1330
1331 // Must be called with the page lock held.
should_report_large(Length num_pages)1332 inline bool should_report_large(Length num_pages) {
1333 const int64 threshold = large_alloc_threshold;
1334 if (threshold > 0 && num_pages >= (threshold >> kPageShift)) {
1335 // Increase the threshold by 1/8 every time we generate a report.
1336 // We cap the threshold at 8GiB to avoid overflow problems.
1337 large_alloc_threshold = (threshold + threshold/8 < 8ll<<30
1338 ? threshold + threshold/8 : 8ll<<30);
1339 return true;
1340 }
1341 return false;
1342 }
1343
1344 // Helper for do_malloc().
do_malloc_pages(ThreadCache * heap,size_t size)1345 static void* do_malloc_pages(ThreadCache* heap, size_t size) {
1346 void* result;
1347 bool report_large;
1348
1349 Length num_pages = tcmalloc::pages(size);
1350
1351 // Chromium profiling. Measurements in March 2013 suggest this
1352 // imposes a small enough runtime cost that there's no reason to
1353 // try to optimize it.
1354 heap->AddToByteAllocatedTotal(size);
1355
1356 // NOTE: we're passing original size here as opposed to rounded-up
1357 // size as we do in do_malloc_small. The difference is small here
1358 // (at most 4k out of at least 256k). And not rounding up saves us
1359 // from possibility of overflow, which rounding up could produce.
1360 //
1361 // See https://github.com/gperftools/gperftools/issues/723
1362 if (heap->SampleAllocation(size)) {
1363 result = DoSampledAllocation(size);
1364
1365 SpinLockHolder h(Static::pageheap_lock());
1366 report_large = should_report_large(num_pages);
1367 } else {
1368 SpinLockHolder h(Static::pageheap_lock());
1369 Span* span = Static::pageheap()->New(num_pages);
1370 result = (PREDICT_FALSE(span == NULL) ? NULL : SpanToMallocResult(span));
1371 report_large = should_report_large(num_pages);
1372 }
1373
1374 if (report_large) {
1375 ReportLargeAlloc(num_pages, result);
1376 }
1377 return result;
1378 }
1379
nop_oom_handler(size_t size)1380 static void *nop_oom_handler(size_t size) {
1381 return NULL;
1382 }
1383
do_malloc(size_t size)1384 ATTRIBUTE_ALWAYS_INLINE inline void* do_malloc(size_t size) {
1385 #if defined(TCMALLOC_DISABLE_HUGE_ALLOCATIONS)
1386 if (!IsAllocSizePermitted(size)) {
1387 errno = ENOMEM;
1388 return NULL;
1389 }
1390 #endif
1391
1392 if (PREDICT_FALSE(ThreadCache::IsUseEmergencyMalloc())) {
1393 return tcmalloc::EmergencyMalloc(size);
1394 }
1395
1396 // note: it will force initialization of malloc if necessary
1397 ThreadCache* cache = ThreadCache::GetCache();
1398 uint32 cl;
1399
1400 ASSERT(Static::IsInited());
1401 ASSERT(cache != NULL);
1402
1403 if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, &cl))) {
1404 return do_malloc_pages(cache, size);
1405 }
1406
1407 // TODO(jar): If this has any detectable performance impact, it can be
1408 // optimized by only tallying sizes if the profiler was activated to recall
1409 // these tallies. I don't think this is performance critical, but we really
1410 // should measure it.
1411 cache->AddToByteAllocatedTotal(size); // Chromium profiling.
1412
1413 size_t allocated_size = Static::sizemap()->class_to_size(cl);
1414 if (PREDICT_FALSE(cache->SampleAllocation(allocated_size))) {
1415 return DoSampledAllocation(size);
1416 }
1417
1418 // The common case, and also the simplest. This just pops the
1419 // size-appropriate freelist, after replenishing it if it's empty.
1420 return CheckedMallocResult(
1421 cache->Allocate(allocated_size, cl, nop_oom_handler));
1422 }
1423
retry_malloc(void * size)1424 static void *retry_malloc(void* size) {
1425 return do_malloc(reinterpret_cast<size_t>(size));
1426 }
1427
do_malloc_or_cpp_alloc(size_t size)1428 ATTRIBUTE_ALWAYS_INLINE inline void* do_malloc_or_cpp_alloc(size_t size) {
1429 void *rv = do_malloc(size);
1430 if (PREDICT_TRUE(rv != NULL)) {
1431 return rv;
1432 }
1433 return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
1434 false, true);
1435 }
1436
do_calloc(size_t n,size_t elem_size)1437 ATTRIBUTE_ALWAYS_INLINE inline void* do_calloc(size_t n, size_t elem_size) {
1438 // Overflow check
1439 const size_t size = n * elem_size;
1440 if (elem_size != 0 && size / elem_size != n) return NULL;
1441
1442 void* result = do_malloc_or_cpp_alloc(size);
1443 if (result != NULL) {
1444 memset(result, 0, size);
1445 }
1446 return result;
1447 }
1448
1449 // If ptr is NULL, do nothing. Otherwise invoke the given function.
free_null_or_invalid(void * ptr,void (* invalid_free_fn)(void *))1450 inline void free_null_or_invalid(void* ptr, void (*invalid_free_fn)(void*)) {
1451 if (ptr != NULL) {
1452 (*invalid_free_fn)(ptr);
1453 }
1454 }
1455
do_free_pages(Span * span,void * ptr)1456 static ATTRIBUTE_NOINLINE void do_free_pages(Span* span, void* ptr) {
1457 // Check to see if the object is in use.
1458 CHECK_CONDITION_PRINT(span->location == Span::IN_USE,
1459 "Object was not in-use");
1460 CHECK_CONDITION_PRINT(
1461 span->start << kPageShift == reinterpret_cast<uintptr_t>(ptr),
1462 "Pointer is not pointing to the start of a span");
1463
1464 SpinLockHolder h(Static::pageheap_lock());
1465 if (span->sample) {
1466 StackTrace* st = reinterpret_cast<StackTrace*>(span->objects);
1467 tcmalloc::DLL_Remove(span);
1468 Static::stacktrace_allocator()->Delete(st);
1469 span->objects = NULL;
1470 }
1471 Static::pageheap()->Delete(span);
1472 }
1473
1474 // Helper for the object deletion (free, delete, etc.). Inputs:
1475 // ptr is object to be freed
1476 // invalid_free_fn is a function that gets invoked on certain "bad frees"
1477 //
1478 // We can usually detect the case where ptr is not pointing to a page that
1479 // tcmalloc is using, and in those cases we invoke invalid_free_fn.
1480 ATTRIBUTE_ALWAYS_INLINE inline
do_free_with_callback(void * ptr,void (* invalid_free_fn)(void *),bool use_hint,size_t size_hint)1481 void do_free_with_callback(void* ptr,
1482 void (*invalid_free_fn)(void*),
1483 bool use_hint, size_t size_hint) {
1484 ThreadCache* heap = ThreadCache::GetCacheIfPresent();
1485
1486 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
1487 uint32 cl;
1488
1489 #ifndef NO_TCMALLOC_SAMPLES
1490 // we only pass size hint when ptr is not page aligned. Which
1491 // implies that it must be very small object.
1492 ASSERT(!use_hint || size_hint < kPageSize);
1493 #endif
1494
1495 if (!use_hint || PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size_hint, &cl))) {
1496 // if we're in sized delete, but size is too large, no need to
1497 // probe size cache
1498 bool cache_hit = !use_hint && Static::pageheap()->TryGetSizeClass(p, &cl);
1499 if (PREDICT_FALSE(!cache_hit)) {
1500 Span* span = Static::pageheap()->GetDescriptor(p);
1501 if (PREDICT_FALSE(!span)) {
1502 // span can be NULL because the pointer passed in is NULL or invalid
1503 // (not something returned by malloc or friends), or because the
1504 // pointer was allocated with some other allocator besides
1505 // tcmalloc. The latter can happen if tcmalloc is linked in via
1506 // a dynamic library, but is not listed last on the link line.
1507 // In that case, libraries after it on the link line will
1508 // allocate with libc malloc, but free with tcmalloc's free.
1509 free_null_or_invalid(ptr, invalid_free_fn);
1510 return;
1511 }
1512 cl = span->sizeclass;
1513 if (PREDICT_FALSE(cl == 0)) {
1514 ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0);
1515 ASSERT(span != NULL && span->start == p);
1516 do_free_pages(span, ptr);
1517 return;
1518 }
1519 if (!use_hint) {
1520 Static::pageheap()->SetCachedSizeClass(p, cl);
1521 }
1522 }
1523 }
1524
1525 if (PREDICT_TRUE(heap != NULL)) {
1526 ASSERT(Static::IsInited());
1527 // If we've hit initialized thread cache, so we're done.
1528 heap->Deallocate(ptr, cl);
1529 return;
1530 }
1531
1532 if (PREDICT_FALSE(!Static::IsInited())) {
1533 // if free was called very early we've could have missed the case
1534 // of invalid or nullptr free. I.e. because probing size classes
1535 // cache could return bogus result (cl = 0 as of this
1536 // writing). But since there is no way we could be dealing with
1537 // ptr we've allocated, since successfull malloc implies IsInited,
1538 // we can just call "invalid free" handling code.
1539 free_null_or_invalid(ptr, invalid_free_fn);
1540 return;
1541 }
1542
1543 // Otherwise, delete directly into central cache
1544 tcmalloc::FL_Init(ptr);
1545 Static::central_cache()[cl].InsertRange(ptr, ptr, 1);
1546 }
1547
1548 // The default "do_free" that uses the default callback.
do_free(void * ptr)1549 ATTRIBUTE_ALWAYS_INLINE inline void do_free(void* ptr) {
1550 return do_free_with_callback(ptr, &InvalidFree, false, 0);
1551 }
1552
1553 // NOTE: some logic here is duplicated in GetOwnership (above), for
1554 // speed. If you change this function, look at that one too.
GetSizeWithCallback(const void * ptr,size_t (* invalid_getsize_fn)(const void *))1555 inline size_t GetSizeWithCallback(const void* ptr,
1556 size_t (*invalid_getsize_fn)(const void*)) {
1557 if (ptr == NULL)
1558 return 0;
1559 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
1560 uint32 cl;
1561 if (Static::pageheap()->TryGetSizeClass(p, &cl)) {
1562 return Static::sizemap()->ByteSizeForClass(cl);
1563 }
1564
1565 const Span *span = Static::pageheap()->GetDescriptor(p);
1566 if (PREDICT_FALSE(span == NULL)) { // means we do not own this memory
1567 return (*invalid_getsize_fn)(ptr);
1568 }
1569
1570 if (span->sizeclass != 0) {
1571 return Static::sizemap()->ByteSizeForClass(span->sizeclass);
1572 }
1573
1574 if (span->sample) {
1575 size_t orig_size = reinterpret_cast<StackTrace*>(span->objects)->size;
1576 return tc_nallocx(orig_size, 0);
1577 }
1578
1579 return span->length << kPageShift;
1580 }
1581
1582 // This lets you call back to a given function pointer if ptr is invalid.
1583 // It is used primarily by windows code which wants a specialized callback.
do_realloc_with_callback(void * old_ptr,size_t new_size,void (* invalid_free_fn)(void *),size_t (* invalid_get_size_fn)(const void *))1584 ATTRIBUTE_ALWAYS_INLINE inline void* do_realloc_with_callback(
1585 void* old_ptr, size_t new_size,
1586 void (*invalid_free_fn)(void*),
1587 size_t (*invalid_get_size_fn)(const void*)) {
1588 // Get the size of the old entry
1589 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn);
1590
1591 // Reallocate if the new size is larger than the old size,
1592 // or if the new size is significantly smaller than the old size.
1593 // We do hysteresis to avoid resizing ping-pongs:
1594 // . If we need to grow, grow to max(new_size, old_size * 1.X)
1595 // . Don't shrink unless new_size < old_size * 0.Y
1596 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5.
1597 const size_t min_growth = min(old_size / 4,
1598 (std::numeric_limits<size_t>::max)() - old_size); // Avoid overflow.
1599 const size_t lower_bound_to_grow = old_size + min_growth;
1600 const size_t upper_bound_to_shrink = old_size / 2ul;
1601 if ((new_size > old_size) || (new_size < upper_bound_to_shrink)) {
1602 // Need to reallocate.
1603 void* new_ptr = NULL;
1604
1605 if (new_size > old_size && new_size < lower_bound_to_grow) {
1606 new_ptr = do_malloc_or_cpp_alloc(lower_bound_to_grow);
1607 }
1608 if (new_ptr == NULL) {
1609 // Either new_size is not a tiny increment, or last do_malloc failed.
1610 new_ptr = do_malloc_or_cpp_alloc(new_size);
1611 }
1612 if (PREDICT_FALSE(new_ptr == NULL)) {
1613 return NULL;
1614 }
1615 MallocHook::InvokeNewHook(new_ptr, new_size);
1616 memcpy(new_ptr, old_ptr, ((old_size < new_size) ? old_size : new_size));
1617 MallocHook::InvokeDeleteHook(old_ptr);
1618 // We could use a variant of do_free() that leverages the fact
1619 // that we already know the sizeclass of old_ptr. The benefit
1620 // would be small, so don't bother.
1621 do_free_with_callback(old_ptr, invalid_free_fn, false, 0);
1622 return new_ptr;
1623 } else {
1624 // We still need to call hooks to report the updated size:
1625 MallocHook::InvokeDeleteHook(old_ptr);
1626 MallocHook::InvokeNewHook(old_ptr, new_size);
1627 return old_ptr;
1628 }
1629 }
1630
do_realloc(void * old_ptr,size_t new_size)1631 ATTRIBUTE_ALWAYS_INLINE inline void* do_realloc(void* old_ptr, size_t new_size) {
1632 return do_realloc_with_callback(old_ptr, new_size,
1633 &InvalidFree, &InvalidGetSizeForRealloc);
1634 }
1635
1636 static ATTRIBUTE_ALWAYS_INLINE inline
do_memalign_pages(size_t align,size_t size)1637 void* do_memalign_pages(size_t align, size_t size) {
1638 ASSERT((align & (align - 1)) == 0);
1639 ASSERT(align > kPageSize);
1640 if (size + align < size) return NULL; // Overflow
1641
1642 if (PREDICT_FALSE(Static::pageheap() == NULL)) ThreadCache::InitModule();
1643
1644 // Allocate at least one byte to avoid boundary conditions below
1645 if (size == 0) size = 1;
1646
1647 // We will allocate directly from the page heap
1648 SpinLockHolder h(Static::pageheap_lock());
1649
1650 // Allocate extra pages and carve off an aligned portion
1651 const Length alloc = tcmalloc::pages(size + align);
1652 Span* span = Static::pageheap()->New(alloc);
1653 if (PREDICT_FALSE(span == NULL)) return NULL;
1654
1655 // Skip starting portion so that we end up aligned
1656 Length skip = 0;
1657 while ((((span->start+skip) << kPageShift) & (align - 1)) != 0) {
1658 skip++;
1659 }
1660 ASSERT(skip < alloc);
1661 if (skip > 0) {
1662 Span* rest = Static::pageheap()->Split(span, skip);
1663 Static::pageheap()->Delete(span);
1664 span = rest;
1665 }
1666
1667 // Skip trailing portion that we do not need to return
1668 const Length needed = tcmalloc::pages(size);
1669 ASSERT(span->length >= needed);
1670 if (span->length > needed) {
1671 Span* trailer = Static::pageheap()->Split(span, needed);
1672 Static::pageheap()->Delete(trailer);
1673 }
1674 return SpanToMallocResult(span);
1675 }
1676
1677 // Helpers for use by exported routines below:
1678
do_malloc_stats()1679 inline void do_malloc_stats() {
1680 PrintStats(1);
1681 }
1682
do_mallopt(int cmd,int value)1683 inline int do_mallopt(int cmd, int value) {
1684 if (cmd == TC_MALLOPT_IS_OVERRIDDEN_BY_TCMALLOC)
1685 return TC_MALLOPT_IS_OVERRIDDEN_BY_TCMALLOC;
1686
1687 // 1 is the success return value according to man mallopt(). However (see the
1688 // BUGS section in the manpage), most implementations return always 1.
1689 // This code is just complying with that (buggy) expectation.
1690 return 1;
1691 }
1692
1693 #ifdef HAVE_STRUCT_MALLINFO
do_mallinfo()1694 inline struct mallinfo do_mallinfo() {
1695 TCMallocStats stats;
1696 ExtractStats(&stats, NULL, NULL, NULL);
1697
1698 // Just some of the fields are filled in.
1699 struct mallinfo info;
1700 memset(&info, 0, sizeof(info));
1701
1702 // Unfortunately, the struct contains "int" field, so some of the
1703 // size values will be truncated.
1704 info.arena = static_cast<int>(stats.pageheap.system_bytes);
1705 info.fsmblks = static_cast<int>(stats.thread_bytes
1706 + stats.central_bytes
1707 + stats.transfer_bytes);
1708 info.fordblks = static_cast<int>(stats.pageheap.free_bytes +
1709 stats.pageheap.unmapped_bytes);
1710 info.uordblks = static_cast<int>(stats.pageheap.system_bytes
1711 - stats.thread_bytes
1712 - stats.central_bytes
1713 - stats.transfer_bytes
1714 - stats.pageheap.free_bytes
1715 - stats.pageheap.unmapped_bytes);
1716
1717 return info;
1718 }
1719 #endif // HAVE_STRUCT_MALLINFO
1720
1721 } // end unnamed namespace
1722
1723 // As promised, the definition of this function, declared above.
GetAllocatedSize(const void * ptr)1724 size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) {
1725 if (ptr == NULL)
1726 return 0;
1727 ASSERT(TCMallocImplementation::GetOwnership(ptr)
1728 != TCMallocImplementation::kNotOwned);
1729 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize);
1730 }
1731
MarkThreadBusy()1732 void TCMallocImplementation::MarkThreadBusy() {
1733 // Allocate to force the creation of a thread cache, but avoid
1734 // invoking any hooks.
1735 do_free(do_malloc(0));
1736 }
1737
1738 //-------------------------------------------------------------------
1739 // Exported routines
1740 //-------------------------------------------------------------------
1741
tc_version(int * major,int * minor,const char ** patch)1742 extern "C" PERFTOOLS_DLL_DECL const char* tc_version(
1743 int* major, int* minor, const char** patch) PERFTOOLS_NOTHROW {
1744 if (major) *major = TC_VERSION_MAJOR;
1745 if (minor) *minor = TC_VERSION_MINOR;
1746 if (patch) *patch = TC_VERSION_PATCH;
1747 return TC_VERSION_STRING;
1748 }
1749
1750 // This function behaves similarly to MSVC's _set_new_mode.
1751 // If flag is 0 (default), calls to malloc will behave normally.
1752 // If flag is 1, calls to malloc will behave like calls to new,
1753 // and the std_new_handler will be invoked on failure.
1754 // Returns the previous mode.
tc_set_new_mode(int flag)1755 extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) PERFTOOLS_NOTHROW {
1756 int old_mode = tc_new_mode;
1757 tc_new_mode = flag;
1758 return old_mode;
1759 }
1760
tc_query_new_mode()1761 extern "C" PERFTOOLS_DLL_DECL int tc_query_new_mode() PERFTOOLS_NOTHROW {
1762 return tc_new_mode;
1763 }
1764
1765 #ifndef TCMALLOC_USING_DEBUGALLOCATION // debugallocation.cc defines its own
1766
1767 // CAVEAT: The code structure below ensures that MallocHook methods are always
1768 // called from the stack frame of the invoked allocation function.
1769 // heap-checker.cc depends on this to start a stack trace from
1770 // the call to the (de)allocation function.
1771
1772 namespace tcmalloc {
1773
1774
ATTRIBUTE_SECTION(google_malloc)1775 static ATTRIBUTE_SECTION(google_malloc)
1776 void invoke_hooks_and_free(void *ptr) {
1777 MallocHook::InvokeDeleteHook(ptr);
1778 do_free(ptr);
1779 }
1780
ATTRIBUTE_SECTION(google_malloc)1781 ATTRIBUTE_SECTION(google_malloc)
1782 void* cpp_throw_oom(size_t size) {
1783 return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
1784 true, false);
1785 }
1786
ATTRIBUTE_SECTION(google_malloc)1787 ATTRIBUTE_SECTION(google_malloc)
1788 void* cpp_nothrow_oom(size_t size) {
1789 return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
1790 true, true);
1791 }
1792
ATTRIBUTE_SECTION(google_malloc)1793 ATTRIBUTE_SECTION(google_malloc)
1794 void* malloc_oom(size_t size) {
1795 return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
1796 false, true);
1797 }
1798
1799 // tcmalloc::allocate_full_XXX is called by fast-path malloc when some
1800 // complex handling is needed (such as fetching object from central
1801 // freelist or malloc sampling). It contains all 'operator new' logic,
1802 // as opposed to malloc_fast_path which only deals with important
1803 // subset of cases.
1804 //
1805 // Note that this is under tcmalloc namespace so that pprof
1806 // can automatically filter it out of growthz/heapz profiles.
1807 //
1808 // We have slightly fancy setup because we need to call hooks from
1809 // function in 'google_malloc' section and we cannot place template
1810 // into this section. Thus 3 separate functions 'built' by macros.
1811 //
1812 // Also note that we're carefully orchestrating for
1813 // MallocHook::GetCallerStackTrace to work even if compiler isn't
1814 // optimizing tail calls (e.g. -O0 is given). We still require
1815 // ATTRIBUTE_ALWAYS_INLINE to work for that case, but it was seen to
1816 // work for -O0 -fno-inline across both GCC and clang. I.e. in this
1817 // case we'll get stack frame for tc_new, followed by stack frame for
1818 // allocate_full_cpp_throw_oom, followed by hooks machinery and user
1819 // code's stack frames. So GetCallerStackTrace will find 2
1820 // subsequent stack frames in google_malloc section and correctly
1821 // 'cut' stack trace just before tc_new.
1822 template <void* OOMHandler(size_t)>
1823 ATTRIBUTE_ALWAYS_INLINE inline
do_allocate_full(size_t size)1824 static void* do_allocate_full(size_t size) {
1825 void* p = do_malloc(size);
1826 if (PREDICT_FALSE(p == NULL)) {
1827 p = OOMHandler(size);
1828 }
1829 MallocHook::InvokeNewHook(p, size);
1830 return CheckedMallocResult(p);
1831 }
1832
1833 #define AF(oom) \
1834 ATTRIBUTE_SECTION(google_malloc) \
1835 void* allocate_full_##oom(size_t size) { \
1836 return do_allocate_full<oom>(size); \
1837 }
1838
1839 AF(cpp_throw_oom)
AF(cpp_nothrow_oom)1840 AF(cpp_nothrow_oom)
1841 AF(malloc_oom)
1842
1843 #undef AF
1844
1845 template <void* OOMHandler(size_t)>
1846 static ATTRIBUTE_ALWAYS_INLINE inline void* dispatch_allocate_full(size_t size) {
1847 if (OOMHandler == cpp_throw_oom) {
1848 return allocate_full_cpp_throw_oom(size);
1849 }
1850 if (OOMHandler == cpp_nothrow_oom) {
1851 return allocate_full_cpp_nothrow_oom(size);
1852 }
1853 ASSERT(OOMHandler == malloc_oom);
1854 return allocate_full_malloc_oom(size);
1855 }
1856
1857 struct retry_memalign_data {
1858 size_t align;
1859 size_t size;
1860 };
1861
retry_do_memalign(void * arg)1862 static void *retry_do_memalign(void *arg) {
1863 retry_memalign_data *data = static_cast<retry_memalign_data *>(arg);
1864 return do_memalign_pages(data->align, data->size);
1865 }
1866
ATTRIBUTE_SECTION(google_malloc)1867 static ATTRIBUTE_SECTION(google_malloc)
1868 void* memalign_pages(size_t align, size_t size,
1869 bool from_operator, bool nothrow) {
1870 void *rv = do_memalign_pages(align, size);
1871 if (PREDICT_FALSE(rv == NULL)) {
1872 retry_memalign_data data;
1873 data.align = align;
1874 data.size = size;
1875 rv = handle_oom(retry_do_memalign, &data,
1876 from_operator, nothrow);
1877 }
1878 MallocHook::InvokeNewHook(rv, size);
1879 return CheckedMallocResult(rv);
1880 }
1881
1882 } // namespace tcmalloc
1883
1884 // This is quick, fast-path-only implementation of malloc/new. It is
1885 // designed to only have support for fast-path. It checks if more
1886 // complex handling is needed (such as a pageheap allocation or
1887 // sampling) and only performs allocation if none of those uncommon
1888 // conditions hold. When we have one of those odd cases it simply
1889 // tail-calls to one of tcmalloc::allocate_full_XXX defined above.
1890 //
1891 // Such approach was found to be quite effective. Generated code for
1892 // tc_{new,malloc} either succeeds quickly or tail-calls to
1893 // allocate_full. Terseness of the source and lack of
1894 // non-tail calls enables compiler to produce better code. Also
1895 // produced code is short enough to enable effort-less human
1896 // comprehension. Which itself led to elimination of various checks
1897 // that were not necessary for fast-path.
1898 template <void* OOMHandler(size_t)>
1899 ATTRIBUTE_ALWAYS_INLINE inline
malloc_fast_path(size_t size)1900 static void * malloc_fast_path(size_t size) {
1901 if (PREDICT_FALSE(!base::internal::new_hooks_.empty())) {
1902 return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
1903 }
1904
1905 ThreadCache *cache = ThreadCache::GetFastPathCache();
1906
1907 if (PREDICT_FALSE(cache == NULL)) {
1908 return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
1909 }
1910
1911 uint32 cl;
1912 if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, &cl))) {
1913 return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
1914 }
1915
1916 size_t allocated_size = Static::sizemap()->ByteSizeForClass(cl);
1917
1918 if (PREDICT_FALSE(!cache->TryRecordAllocationFast(allocated_size))) {
1919 return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
1920 }
1921
1922 return CheckedMallocResult(cache->Allocate(allocated_size, cl, OOMHandler));
1923 }
1924
1925 template <void* OOMHandler(size_t)>
1926 ATTRIBUTE_ALWAYS_INLINE inline
memalign_fast_path(size_t align,size_t size)1927 static void* memalign_fast_path(size_t align, size_t size) {
1928 if (PREDICT_FALSE(align > kPageSize)) {
1929 if (OOMHandler == tcmalloc::cpp_throw_oom) {
1930 return tcmalloc::memalign_pages(align, size, true, false);
1931 } else if (OOMHandler == tcmalloc::cpp_nothrow_oom) {
1932 return tcmalloc::memalign_pages(align, size, true, true);
1933 } else {
1934 ASSERT(OOMHandler == tcmalloc::malloc_oom);
1935 return tcmalloc::memalign_pages(align, size, false, true);
1936 }
1937 }
1938
1939 // Everything with alignment <= kPageSize we can easily delegate to
1940 // regular malloc
1941
1942 return malloc_fast_path<OOMHandler>(align_size_up(size, align));
1943 }
1944
1945 extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
tc_malloc(size_t size)1946 void* tc_malloc(size_t size) PERFTOOLS_NOTHROW {
1947 return malloc_fast_path<tcmalloc::malloc_oom>(size);
1948 }
1949
1950 static ATTRIBUTE_ALWAYS_INLINE inline
free_fast_path(void * ptr)1951 void free_fast_path(void *ptr) {
1952 if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
1953 tcmalloc::invoke_hooks_and_free(ptr);
1954 return;
1955 }
1956 do_free(ptr);
1957 }
1958
1959 extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
tc_free(void * ptr)1960 void tc_free(void* ptr) PERFTOOLS_NOTHROW {
1961 free_fast_path(ptr);
1962 }
1963
1964 extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
tc_free_sized(void * ptr,size_t size)1965 void tc_free_sized(void *ptr, size_t size) PERFTOOLS_NOTHROW {
1966 if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
1967 tcmalloc::invoke_hooks_and_free(ptr);
1968 return;
1969 }
1970 #ifndef NO_TCMALLOC_SAMPLES
1971 // if ptr is kPageSize-aligned, then it could be sampled allocation,
1972 // thus we don't trust hint and just do plain free. It also handles
1973 // nullptr for us.
1974 if (PREDICT_FALSE((reinterpret_cast<uintptr_t>(ptr) & (kPageSize-1)) == 0)) {
1975 tc_free(ptr);
1976 return;
1977 }
1978 #else
1979 if (!ptr) {
1980 return;
1981 }
1982 #endif
1983 do_free_with_callback(ptr, &InvalidFree, true, size);
1984 }
1985
1986 #ifdef TC_ALIAS
1987
1988 extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) PERFTOOLS_NOTHROW
1989 TC_ALIAS(tc_free_sized);
1990 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) PERFTOOLS_NOTHROW
1991 TC_ALIAS(tc_free_sized);
1992
1993 #else
1994
tc_delete_sized(void * p,size_t size)1995 extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) PERFTOOLS_NOTHROW {
1996 tc_free_sized(p, size);
1997 }
tc_deletearray_sized(void * p,size_t size)1998 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) PERFTOOLS_NOTHROW {
1999 tc_free_sized(p, size);
2000 }
2001
2002 #endif
2003
tc_calloc(size_t n,size_t elem_size)2004 extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n,
2005 size_t elem_size) PERFTOOLS_NOTHROW {
2006 if (ThreadCache::IsUseEmergencyMalloc()) {
2007 return tcmalloc::EmergencyCalloc(n, elem_size);
2008 }
2009 void* result = do_calloc(n, elem_size);
2010 MallocHook::InvokeNewHook(result, n * elem_size);
2011 return result;
2012 }
2013
2014 extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_NOTHROW
2015 #ifdef TC_ALIAS
2016 TC_ALIAS(tc_free);
2017 #else
2018 {
2019 free_fast_path(ptr);
2020 }
2021 #endif
2022
tc_realloc(void * old_ptr,size_t new_size)2023 extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* old_ptr,
2024 size_t new_size) PERFTOOLS_NOTHROW {
2025 if (old_ptr == NULL) {
2026 void* result = do_malloc_or_cpp_alloc(new_size);
2027 MallocHook::InvokeNewHook(result, new_size);
2028 return result;
2029 }
2030 if (new_size == 0) {
2031 MallocHook::InvokeDeleteHook(old_ptr);
2032 do_free(old_ptr);
2033 return NULL;
2034 }
2035 if (PREDICT_FALSE(tcmalloc::IsEmergencyPtr(old_ptr))) {
2036 return tcmalloc::EmergencyRealloc(old_ptr, new_size);
2037 }
2038 return do_realloc(old_ptr, new_size);
2039 }
2040
2041 extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
tc_new(size_t size)2042 void* tc_new(size_t size) {
2043 return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
2044 }
2045
2046 extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
tc_new_nothrow(size_t size,const std::nothrow_t &)2047 void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW {
2048 return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
2049 }
2050
2051 extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_NOTHROW
2052 #ifdef TC_ALIAS
2053 TC_ALIAS(tc_free);
2054 #else
2055 {
2056 free_fast_path(p);
2057 }
2058 #endif
2059
2060 // Standard C++ library implementations define and use this
2061 // (via ::operator delete(ptr, nothrow)).
2062 // But it's really the same as normal delete, so we just do the same thing.
2063 extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW
2064 #ifdef TC_ALIAS
2065 TC_ALIAS(tc_free);
2066 #else
2067 {
2068 if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
2069 tcmalloc::invoke_hooks_and_free(p);
2070 return;
2071 }
2072 do_free(p);
2073 }
2074 #endif
2075
2076 extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size)
2077 #ifdef TC_ALIAS
2078 TC_ALIAS(tc_new);
2079 #else
2080 {
2081 return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
2082 }
2083 #endif
2084
2085 extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&)
2086 PERFTOOLS_NOTHROW
2087 #ifdef TC_ALIAS
2088 TC_ALIAS(tc_new_nothrow);
2089 #else
2090 {
2091 return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
2092 }
2093 #endif
2094
2095 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_NOTHROW
2096 #ifdef TC_ALIAS
2097 TC_ALIAS(tc_free);
2098 #else
2099 {
2100 free_fast_path(p);
2101 }
2102 #endif
2103
2104 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW
2105 #ifdef TC_ALIAS
2106 TC_ALIAS(tc_delete_nothrow);
2107 #else
2108 {
2109 free_fast_path(p);
2110 }
2111 #endif
2112
2113 extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
tc_memalign(size_t align,size_t size)2114 void* tc_memalign(size_t align, size_t size) PERFTOOLS_NOTHROW {
2115 return memalign_fast_path<tcmalloc::malloc_oom>(align, size);
2116 }
2117
tc_posix_memalign(void ** result_ptr,size_t align,size_t size)2118 extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(
2119 void** result_ptr, size_t align, size_t size) PERFTOOLS_NOTHROW {
2120 if (((align % sizeof(void*)) != 0) ||
2121 ((align & (align - 1)) != 0) ||
2122 (align == 0)) {
2123 return EINVAL;
2124 }
2125
2126 void* result = tc_memalign(align, size);
2127 if (PREDICT_FALSE(result == NULL)) {
2128 return ENOMEM;
2129 } else {
2130 *result_ptr = result;
2131 return 0;
2132 }
2133 }
2134
2135 #if defined(ENABLE_ALIGNED_NEW_DELETE)
2136
tc_new_aligned(size_t size,std::align_val_t align)2137 extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned(size_t size, std::align_val_t align) {
2138 return memalign_fast_path<tcmalloc::cpp_throw_oom>(static_cast<size_t>(align), size);
2139 }
2140
tc_new_aligned_nothrow(size_t size,std::align_val_t align,const std::nothrow_t &)2141 extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t&) PERFTOOLS_NOTHROW {
2142 return memalign_fast_path<tcmalloc::cpp_nothrow_oom>(static_cast<size_t>(align), size);
2143 }
2144
2145 extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW
2146 #ifdef TC_ALIAS
2147 TC_ALIAS(tc_delete);
2148 #else
2149 {
2150 free_fast_path(p);
2151 }
2152 #endif
2153
2154 // There is no easy way to obtain the actual size used by do_memalign to allocate aligned storage, so for now
2155 // just ignore the size. It might get useful in the future.
2156 extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW
2157 #ifdef TC_ALIAS
2158 TC_ALIAS(tc_delete);
2159 #else
2160 {
2161 free_fast_path(p);
2162 }
2163 #endif
2164
2165 extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW
2166 #ifdef TC_ALIAS
2167 TC_ALIAS(tc_delete);
2168 #else
2169 {
2170 free_fast_path(p);
2171 }
2172 #endif
2173
2174 extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned(size_t size, std::align_val_t align)
2175 #ifdef TC_ALIAS
2176 TC_ALIAS(tc_new_aligned);
2177 #else
2178 {
2179 return memalign_fast_path<tcmalloc::cpp_throw_oom>(static_cast<size_t>(align), size);
2180 }
2181 #endif
2182
2183 extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t& nt) PERFTOOLS_NOTHROW
2184 #ifdef TC_ALIAS
2185 TC_ALIAS(tc_new_aligned_nothrow);
2186 #else
2187 {
2188 return memalign_fast_path<tcmalloc::cpp_nothrow_oom>(static_cast<size_t>(align), size);
2189 }
2190 #endif
2191
2192 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW
2193 #ifdef TC_ALIAS
2194 TC_ALIAS(tc_delete_aligned);
2195 #else
2196 {
2197 free_fast_path(p);
2198 }
2199 #endif
2200
2201 // There is no easy way to obtain the actual size used by do_memalign to allocate aligned storage, so for now
2202 // just ignore the size. It might get useful in the future.
2203 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW
2204 #ifdef TC_ALIAS
2205 TC_ALIAS(tc_delete_sized_aligned);
2206 #else
2207 {
2208 free_fast_path(p);
2209 }
2210 #endif
2211
2212 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW
2213 #ifdef TC_ALIAS
2214 TC_ALIAS(tc_delete_aligned_nothrow);
2215 #else
2216 {
2217 free_fast_path(p);
2218 }
2219 #endif
2220
2221 #endif // defined(ENABLE_ALIGNED_NEW_DELETE)
2222
2223 static size_t pagesize = 0;
2224
tc_valloc(size_t size)2225 extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) PERFTOOLS_NOTHROW {
2226 // Allocate page-aligned object of length >= size bytes
2227 if (pagesize == 0) pagesize = getpagesize();
2228 return tc_memalign(pagesize, size);
2229 }
2230
tc_pvalloc(size_t size)2231 extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) PERFTOOLS_NOTHROW {
2232 // Round up size to a multiple of pagesize
2233 if (pagesize == 0) pagesize = getpagesize();
2234 if (size == 0) { // pvalloc(0) should allocate one page, according to
2235 size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html
2236 }
2237 size = (size + pagesize - 1) & ~(pagesize - 1);
2238 return tc_memalign(pagesize, size);
2239 }
2240
tc_malloc_stats(void)2241 extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_NOTHROW {
2242 do_malloc_stats();
2243 }
2244
tc_mallopt(int cmd,int value)2245 extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW {
2246 return do_mallopt(cmd, value);
2247 }
2248
2249 #ifdef HAVE_STRUCT_MALLINFO
tc_mallinfo(void)2250 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW {
2251 return do_mallinfo();
2252 }
2253 #endif
2254
tc_malloc_size(void * ptr)2255 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_NOTHROW {
2256 return MallocExtension::instance()->GetAllocatedSize(ptr);
2257 }
2258
tc_malloc_skip_new_handler(size_t size)2259 extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_NOTHROW {
2260 void* result = do_malloc(size);
2261 MallocHook::InvokeNewHook(result, size);
2262 return result;
2263 }
2264
2265 #endif // TCMALLOC_USING_DEBUGALLOCATION
2266