1 // -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
2 // Copyright (c) 2005, Google Inc.
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // ---
32 // Author: Sanjay Ghemawat
33
34 #include <config.h>
35 #include <errno.h> // for EAGAIN, errno
36 #include <fcntl.h> // for open, O_RDWR
37 #include <stddef.h> // for size_t, NULL, ptrdiff_t
38 #if defined HAVE_STDINT_H
39 #include <stdint.h> // for uintptr_t, intptr_t
40 #elif defined HAVE_INTTYPES_H
41 #include <inttypes.h>
42 #else
43 #include <sys/types.h>
44 #endif
45 #ifdef HAVE_MMAP
46 #include <sys/mman.h> // for munmap, mmap, MADV_DONTNEED, etc
47 #endif
48 #ifdef HAVE_UNISTD_H
49 #include <unistd.h> // for sbrk, getpagesize, off_t
50 #endif
51 #include <new> // for operator new
52 #include <gperftools/malloc_extension.h>
53 #include "base/basictypes.h"
54 #include "base/commandlineflags.h"
55 #include "base/spinlock.h" // for SpinLockHolder, SpinLock, etc
56 #include "common.h"
57 #include "internal_logging.h"
58
59 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old
60 // form of the name instead.
61 #ifndef MAP_ANONYMOUS
62 # define MAP_ANONYMOUS MAP_ANON
63 #endif
64
65 // Linux added support for MADV_FREE in 4.5 but we aren't ready to use it
66 // yet. Among other things, using compile-time detection leads to poor
67 // results when compiling on a system with MADV_FREE and running on a
68 // system without it. See https://github.com/gperftools/gperftools/issues/780.
69 #if defined(__linux__) && defined(MADV_FREE) && !defined(TCMALLOC_USE_MADV_FREE)
70 # undef MADV_FREE
71 #endif
72
73 // MADV_FREE is specifically designed for use by malloc(), but only
74 // FreeBSD supports it; in linux we fall back to the somewhat inferior
75 // MADV_DONTNEED.
76 #if !defined(MADV_FREE) && defined(MADV_DONTNEED)
77 # define MADV_FREE MADV_DONTNEED
78 #endif
79
80 // Solaris has a bug where it doesn't declare madvise() for C++.
81 // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0
82 #if defined(__sun) && defined(__SVR4)
83 # include <sys/types.h> // for caddr_t
84 extern "C" { extern int madvise(caddr_t, size_t, int); }
85 #endif
86
87 // Set kDebugMode mode so that we can have use C++ conditionals
88 // instead of preprocessor conditionals.
89 #ifdef NDEBUG
90 static const bool kDebugMode = false;
91 #else
92 static const bool kDebugMode = true;
93 #endif
94
95 // TODO(sanjay): Move the code below into the tcmalloc namespace
96 using tcmalloc::kLog;
97 using tcmalloc::Log;
98
99 // Check that no bit is set at position ADDRESS_BITS or higher.
CheckAddressBits(uintptr_t ptr)100 static bool CheckAddressBits(uintptr_t ptr) {
101 bool always_ok = (kAddressBits == 8 * sizeof(void*));
102 // this is a bit insane but otherwise we get compiler warning about
103 // shifting right by word size even if this code is dead :(
104 int shift_bits = always_ok ? 0 : kAddressBits;
105 return always_ok || ((ptr >> shift_bits) == 0);
106 }
107
108 COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*),
109 address_bits_larger_than_pointer_size);
110
111 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED);
112
113 #if defined(HAVE_MMAP) || defined(MADV_FREE)
114 // Page size is initialized on demand (only needed for mmap-based allocators)
115 static size_t pagesize = 0;
116 #endif
117
118 // The current system allocator
119 SysAllocator* tcmalloc_sys_alloc = NULL;
120
121 // Number of bytes taken from system.
122 size_t TCMalloc_SystemTaken = 0;
123
124 // Configuration parameters.
125 DEFINE_int32(malloc_devmem_start,
126 EnvToInt("TCMALLOC_DEVMEM_START", 0),
127 "Physical memory starting location in MB for /dev/mem allocation."
128 " Setting this to 0 disables /dev/mem allocation");
129 DEFINE_int32(malloc_devmem_limit,
130 EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0),
131 "Physical memory limit location in MB for /dev/mem allocation."
132 " Setting this to 0 means no limit.");
133 DEFINE_bool(malloc_skip_sbrk,
134 EnvToBool("TCMALLOC_SKIP_SBRK", false),
135 "Whether sbrk can be used to obtain memory.");
136 DEFINE_bool(malloc_skip_mmap,
137 EnvToBool("TCMALLOC_SKIP_MMAP", false),
138 "Whether mmap can be used to obtain memory.");
139 DEFINE_bool(malloc_disable_memory_release,
140 EnvToBool("TCMALLOC_DISABLE_MEMORY_RELEASE", false),
141 "Whether MADV_FREE/MADV_DONTNEED should be used"
142 " to return unused memory to the system.");
143
144 // static allocators
145 class SbrkSysAllocator : public SysAllocator {
146 public:
SbrkSysAllocator()147 SbrkSysAllocator() : SysAllocator() {
148 }
149 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
150 };
151 static union {
152 char buf[sizeof(SbrkSysAllocator)];
153 void *ptr;
154 } sbrk_space;
155
156 class MmapSysAllocator : public SysAllocator {
157 public:
MmapSysAllocator()158 MmapSysAllocator() : SysAllocator() {
159 }
160 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
161 };
162 static union {
163 char buf[sizeof(MmapSysAllocator)];
164 void *ptr;
165 } mmap_space;
166
167 class DevMemSysAllocator : public SysAllocator {
168 public:
DevMemSysAllocator()169 DevMemSysAllocator() : SysAllocator() {
170 }
171 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
172 };
173
174 class DefaultSysAllocator : public SysAllocator {
175 public:
DefaultSysAllocator()176 DefaultSysAllocator() : SysAllocator() {
177 for (int i = 0; i < kMaxAllocators; i++) {
178 failed_[i] = true;
179 allocs_[i] = NULL;
180 names_[i] = NULL;
181 }
182 }
SetChildAllocator(SysAllocator * alloc,unsigned int index,const char * name)183 void SetChildAllocator(SysAllocator* alloc, unsigned int index,
184 const char* name) {
185 if (index < kMaxAllocators && alloc != NULL) {
186 allocs_[index] = alloc;
187 failed_[index] = false;
188 names_[index] = name;
189 }
190 }
191 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
192
193 private:
194 static const int kMaxAllocators = 2;
195 bool failed_[kMaxAllocators];
196 SysAllocator* allocs_[kMaxAllocators];
197 const char* names_[kMaxAllocators];
198 };
199 static union {
200 char buf[sizeof(DefaultSysAllocator)];
201 void *ptr;
202 } default_space;
203 static const char sbrk_name[] = "SbrkSysAllocator";
204 static const char mmap_name[] = "MmapSysAllocator";
205
206
Alloc(size_t size,size_t * actual_size,size_t alignment)207 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size,
208 size_t alignment) {
209 #if !defined(HAVE_SBRK) || defined(__UCLIBC__)
210 return NULL;
211 #else
212 // Check if we should use sbrk allocation.
213 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized
214 // state) and eventually gets initialized to the specified value. Note
215 // that this code runs for a while before the flags are initialized.
216 // That means that even if this flag is set to true, some (initial)
217 // memory will be allocated with sbrk before the flag takes effect.
218 if (FLAGS_malloc_skip_sbrk) {
219 return NULL;
220 }
221
222 // sbrk will release memory if passed a negative number, so we do
223 // a strict check here
224 if (static_cast<ptrdiff_t>(size + alignment) < 0) return NULL;
225
226 // This doesn't overflow because TCMalloc_SystemAlloc has already
227 // tested for overflow at the alignment boundary.
228 size = ((size + alignment - 1) / alignment) * alignment;
229
230 // "actual_size" indicates that the bytes from the returned pointer
231 // p up to and including (p + actual_size - 1) have been allocated.
232 if (actual_size) {
233 *actual_size = size;
234 }
235
236 // Check that we we're not asking for so much more memory that we'd
237 // wrap around the end of the virtual address space. (This seems
238 // like something sbrk() should check for us, and indeed opensolaris
239 // does, but glibc does not:
240 // http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/port/sys/sbrk.c?a=true
241 // http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/libc/misc/sbrk.c?rev=1.1.2.1&content-type=text/plain&cvsroot=glibc
242 // Without this check, sbrk may succeed when it ought to fail.)
243 if (reinterpret_cast<intptr_t>(sbrk(0)) + size < size) {
244 return NULL;
245 }
246
247 void* result = sbrk(size);
248 if (result == reinterpret_cast<void*>(-1)) {
249 return NULL;
250 }
251
252 // Is it aligned?
253 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
254 if ((ptr & (alignment-1)) == 0) return result;
255
256 // Try to get more memory for alignment
257 size_t extra = alignment - (ptr & (alignment-1));
258 void* r2 = sbrk(extra);
259 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
260 // Contiguous with previous result
261 return reinterpret_cast<void*>(ptr + extra);
262 }
263
264 // Give up and ask for "size + alignment - 1" bytes so
265 // that we can find an aligned region within it.
266 result = sbrk(size + alignment - 1);
267 if (result == reinterpret_cast<void*>(-1)) {
268 return NULL;
269 }
270 ptr = reinterpret_cast<uintptr_t>(result);
271 if ((ptr & (alignment-1)) != 0) {
272 ptr += alignment - (ptr & (alignment-1));
273 }
274 return reinterpret_cast<void*>(ptr);
275 #endif // HAVE_SBRK
276 }
277
Alloc(size_t size,size_t * actual_size,size_t alignment)278 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size,
279 size_t alignment) {
280 #ifndef HAVE_MMAP
281 return NULL;
282 #else
283 // Check if we should use mmap allocation.
284 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized
285 // state) and eventually gets initialized to the specified value. Note
286 // that this code runs for a while before the flags are initialized.
287 // Chances are we never get here before the flags are initialized since
288 // sbrk is used until the heap is exhausted (before mmap is used).
289 if (FLAGS_malloc_skip_mmap) {
290 return NULL;
291 }
292
293 // Enforce page alignment
294 if (pagesize == 0) pagesize = getpagesize();
295 if (alignment < pagesize) alignment = pagesize;
296 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
297 if (aligned_size < size) {
298 return NULL;
299 }
300 size = aligned_size;
301
302 // "actual_size" indicates that the bytes from the returned pointer
303 // p up to and including (p + actual_size - 1) have been allocated.
304 if (actual_size) {
305 *actual_size = size;
306 }
307
308 // Ask for extra memory if alignment > pagesize
309 size_t extra = 0;
310 if (alignment > pagesize) {
311 extra = alignment - pagesize;
312 }
313
314 // Note: size + extra does not overflow since:
315 // size + alignment < (1<<NBITS).
316 // and extra <= alignment
317 // therefore size + extra < (1<<NBITS)
318 void* result = mmap(NULL, size + extra,
319 PROT_READ|PROT_WRITE,
320 MAP_PRIVATE|MAP_ANONYMOUS,
321 -1, 0);
322 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
323 return NULL;
324 }
325
326 // Adjust the return memory so it is aligned
327 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
328 size_t adjust = 0;
329 if ((ptr & (alignment - 1)) != 0) {
330 adjust = alignment - (ptr & (alignment - 1));
331 }
332
333 // Return the unused memory to the system
334 if (adjust > 0) {
335 munmap(reinterpret_cast<void*>(ptr), adjust);
336 }
337 if (adjust < extra) {
338 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
339 }
340
341 ptr += adjust;
342 return reinterpret_cast<void*>(ptr);
343 #endif // HAVE_MMAP
344 }
345
Alloc(size_t size,size_t * actual_size,size_t alignment)346 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size,
347 size_t alignment) {
348 #ifndef HAVE_MMAP
349 return NULL;
350 #else
351 static bool initialized = false;
352 static off_t physmem_base; // next physical memory address to allocate
353 static off_t physmem_limit; // maximum physical address allowed
354 static int physmem_fd; // file descriptor for /dev/mem
355
356 // Check if we should use /dev/mem allocation. Note that it may take
357 // a while to get this flag initialized, so meanwhile we fall back to
358 // the next allocator. (It looks like 7MB gets allocated before
359 // this flag gets initialized -khr.)
360 if (FLAGS_malloc_devmem_start == 0) {
361 // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to
362 // try us again next time.
363 return NULL;
364 }
365
366 if (!initialized) {
367 physmem_fd = open("/dev/mem", O_RDWR);
368 if (physmem_fd < 0) {
369 return NULL;
370 }
371 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
372 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
373 initialized = true;
374 }
375
376 // Enforce page alignment
377 if (pagesize == 0) pagesize = getpagesize();
378 if (alignment < pagesize) alignment = pagesize;
379 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
380 if (aligned_size < size) {
381 return NULL;
382 }
383 size = aligned_size;
384
385 // "actual_size" indicates that the bytes from the returned pointer
386 // p up to and including (p + actual_size - 1) have been allocated.
387 if (actual_size) {
388 *actual_size = size;
389 }
390
391 // Ask for extra memory if alignment > pagesize
392 size_t extra = 0;
393 if (alignment > pagesize) {
394 extra = alignment - pagesize;
395 }
396
397 // check to see if we have any memory left
398 if (physmem_limit != 0 &&
399 ((size + extra) > (physmem_limit - physmem_base))) {
400 return NULL;
401 }
402
403 // Note: size + extra does not overflow since:
404 // size + alignment < (1<<NBITS).
405 // and extra <= alignment
406 // therefore size + extra < (1<<NBITS)
407 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
408 MAP_SHARED, physmem_fd, physmem_base);
409 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
410 return NULL;
411 }
412 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
413
414 // Adjust the return memory so it is aligned
415 size_t adjust = 0;
416 if ((ptr & (alignment - 1)) != 0) {
417 adjust = alignment - (ptr & (alignment - 1));
418 }
419
420 // Return the unused virtual memory to the system
421 if (adjust > 0) {
422 munmap(reinterpret_cast<void*>(ptr), adjust);
423 }
424 if (adjust < extra) {
425 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
426 }
427
428 ptr += adjust;
429 physmem_base += adjust + size;
430
431 return reinterpret_cast<void*>(ptr);
432 #endif // HAVE_MMAP
433 }
434
Alloc(size_t size,size_t * actual_size,size_t alignment)435 void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size,
436 size_t alignment) {
437 for (int i = 0; i < kMaxAllocators; i++) {
438 if (!failed_[i] && allocs_[i] != NULL) {
439 void* result = allocs_[i]->Alloc(size, actual_size, alignment);
440 if (result != NULL) {
441 return result;
442 }
443 failed_[i] = true;
444 }
445 }
446 // After both failed, reset "failed_" to false so that a single failed
447 // allocation won't make the allocator never work again.
448 for (int i = 0; i < kMaxAllocators; i++) {
449 failed_[i] = false;
450 }
451 return NULL;
452 }
453
454 ATTRIBUTE_WEAK ATTRIBUTE_NOINLINE
tc_get_sysalloc_override(SysAllocator * def)455 SysAllocator *tc_get_sysalloc_override(SysAllocator *def)
456 {
457 return def;
458 }
459
460 static bool system_alloc_inited = false;
InitSystemAllocators(void)461 void InitSystemAllocators(void) {
462 MmapSysAllocator *mmap = new (mmap_space.buf) MmapSysAllocator();
463 SbrkSysAllocator *sbrk = new (sbrk_space.buf) SbrkSysAllocator();
464
465 // In 64-bit debug mode, place the mmap allocator first since it
466 // allocates pointers that do not fit in 32 bits and therefore gives
467 // us better testing of code's 64-bit correctness. It also leads to
468 // less false negatives in heap-checking code. (Numbers are less
469 // likely to look like pointers and therefore the conservative gc in
470 // the heap-checker is less likely to misinterpret a number as a
471 // pointer).
472 DefaultSysAllocator *sdef = new (default_space.buf) DefaultSysAllocator();
473 if (kDebugMode && sizeof(void*) > 4) {
474 sdef->SetChildAllocator(mmap, 0, mmap_name);
475 sdef->SetChildAllocator(sbrk, 1, sbrk_name);
476 } else {
477 sdef->SetChildAllocator(sbrk, 0, sbrk_name);
478 sdef->SetChildAllocator(mmap, 1, mmap_name);
479 }
480
481 tcmalloc_sys_alloc = tc_get_sysalloc_override(sdef);
482 }
483
TCMalloc_SystemAlloc(size_t size,size_t * actual_size,size_t alignment)484 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
485 size_t alignment) {
486 // Discard requests that overflow
487 if (size + alignment < size) return NULL;
488
489 SpinLockHolder lock_holder(&spinlock);
490
491 if (!system_alloc_inited) {
492 InitSystemAllocators();
493 system_alloc_inited = true;
494 }
495
496 // Enforce minimum alignment
497 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
498
499 size_t actual_size_storage;
500 if (actual_size == NULL) {
501 actual_size = &actual_size_storage;
502 }
503
504 void* result = tcmalloc_sys_alloc->Alloc(size, actual_size, alignment);
505 if (result != NULL) {
506 CHECK_CONDITION(
507 CheckAddressBits(reinterpret_cast<uintptr_t>(result) + *actual_size - 1));
508 TCMalloc_SystemTaken += *actual_size;
509 }
510 return result;
511 }
512
TCMalloc_SystemRelease(void * start,size_t length)513 bool TCMalloc_SystemRelease(void* start, size_t length) {
514 #ifdef MADV_FREE
515 if (FLAGS_malloc_devmem_start) {
516 // It's not safe to use MADV_FREE/MADV_DONTNEED if we've been
517 // mapping /dev/mem for heap memory.
518 return false;
519 }
520 if (FLAGS_malloc_disable_memory_release) return false;
521 if (pagesize == 0) pagesize = getpagesize();
522 const size_t pagemask = pagesize - 1;
523
524 size_t new_start = reinterpret_cast<size_t>(start);
525 size_t end = new_start + length;
526 size_t new_end = end;
527
528 // Round up the starting address and round down the ending address
529 // to be page aligned:
530 new_start = (new_start + pagesize - 1) & ~pagemask;
531 new_end = new_end & ~pagemask;
532
533 ASSERT((new_start & pagemask) == 0);
534 ASSERT((new_end & pagemask) == 0);
535 ASSERT(new_start >= reinterpret_cast<size_t>(start));
536 ASSERT(new_end <= end);
537
538 if (new_end > new_start) {
539 int result;
540 do {
541 result = madvise(reinterpret_cast<char*>(new_start),
542 new_end - new_start, MADV_FREE);
543 } while (result == -1 && errno == EAGAIN);
544
545 return result != -1;
546 }
547 #endif
548 return false;
549 }
550
TCMalloc_SystemCommit(void * start,size_t length)551 void TCMalloc_SystemCommit(void* start, size_t length) {
552 // Nothing to do here. TCMalloc_SystemRelease does not alter pages
553 // such that they need to be re-committed before they can be used by the
554 // application.
555 }
556