1 /*
2 * system-alloc.cc
3 * Avida
4 *
5 * Added by David on 10/14/09.
6 * Copyright 2009-2011 Michigan State University. All rights reserved.
7 *
8 *
9 * This file is part of Avida.
10 *
11 * Avida is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
13 *
14 * Avida is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public License along with Avida.
18 * If not, see <http://www.gnu.org/licenses/>.
19 *
20 */
21
22 // Copyright (c) 2005, Google Inc.
23 // All rights reserved.
24 //
25 // Redistribution and use in source and binary forms, with or without
26 // modification, are permitted provided that the following conditions are
27 // met:
28 //
29 // * Redistributions of source code must retain the above copyright
30 // notice, this list of conditions and the following disclaimer.
31 // * Redistributions in binary form must reproduce the above
32 // copyright notice, this list of conditions and the following disclaimer
33 // in the documentation and/or other materials provided with the
34 // distribution.
35 // * Neither the name of Google Inc. nor the names of its
36 // contributors may be used to endorse or promote products derived from
37 // this software without specific prior written permission.
38 //
39 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
40 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
41 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
42 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
43 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
45 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
46 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
47 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
48 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
49 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
50
51 // ---
52 // Author: Sanjay Ghemawat
53
54 #include "tcmalloc-platform.h"
55 #if defined HAVE_STDINT_H
56 #include <stdint.h>
57 #elif defined HAVE_INTTYPES_H
58 #include <inttypes.h>
59 #else
60 #include <sys/types.h>
61 #endif
62 #ifdef HAVE_UNISTD_H
63 #include <unistd.h>
64 #endif
65 #include <fcntl.h> // for open()
66 #ifdef HAVE_MMAP
67 #include <sys/mman.h>
68 #endif
69 #include <errno.h>
70 #include "system-alloc.h"
71 #include "internal_logging.h"
72 #include "commandlineflags.h"
73 #include "spinlock.h"
74 #include <stddef.h>
75 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old
76 // form of the name instead.
77 #ifndef MAP_ANONYMOUS
78 # define MAP_ANONYMOUS MAP_ANON
79 #endif
80
81 // Solaris has a bug where it doesn't declare madvise() for C++.
82 // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0
83 #if defined(__sun) && defined(__SVR4)
84 # include <sys/types.h> // for caddr_t
85 extern "C" { extern int madvise(caddr_t, size_t, int); }
86 #endif
87
88 // Set kDebugMode mode so that we can have use C++ conditionals
89 // instead of preprocessor conditionals.
90 #ifdef NDEBUG
91 static const bool kDebugMode = false;
92 #else
93 static const bool kDebugMode = true;
94 #endif
95
96 // Structure for discovering alignment
97 union MemoryAligner {
98 void* p;
99 double d;
100 size_t s;
101 };
102
103 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED);
104
105 #if defined(HAVE_MMAP) || defined(MADV_DONTNEED)
106 // Page size is initialized on demand (only needed for mmap-based allocators)
107 static size_t pagesize = 0;
108 #endif
109
110 // Configuration parameters.
111
112 DEFINE_int32(malloc_devmem_start,
113 EnvToInt("TCMALLOC_DEVMEM_START", 0),
114 "Physical memory starting location in MB for /dev/mem allocation."
115 " Setting this to 0 disables /dev/mem allocation");
116 DEFINE_int32(malloc_devmem_limit,
117 EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0),
118 "Physical memory limit location in MB for /dev/mem allocation."
119 " Setting this to 0 means no limit.");
120 DEFINE_bool(malloc_skip_sbrk,
121 EnvToBool("TCMALLOC_SKIP_SBRK", false),
122 "Whether sbrk can be used to obtain memory.");
123 DEFINE_bool(malloc_skip_mmap,
124 EnvToBool("TCMALLOC_SKIP_MMAP", false),
125 "Whether mmap can be used to obtain memory.");
126
127 // static allocators
128 class SbrkSysAllocator : public SysAllocator {
129 public:
SbrkSysAllocator()130 SbrkSysAllocator() : SysAllocator() {
131 }
132 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
133 void DumpStats(TCMalloc_Printer* printer);
134 };
135 static char sbrk_space[sizeof(SbrkSysAllocator)];
136
137 class MmapSysAllocator : public SysAllocator {
138 public:
MmapSysAllocator()139 MmapSysAllocator() : SysAllocator() {
140 }
141 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
142 void DumpStats(TCMalloc_Printer* printer);
143 };
144 static char mmap_space[sizeof(MmapSysAllocator)];
145
146 class DevMemSysAllocator : public SysAllocator {
147 public:
DevMemSysAllocator()148 DevMemSysAllocator() : SysAllocator() {
149 }
150 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
151 void DumpStats(TCMalloc_Printer* printer);
152 };
153 static char devmem_space[sizeof(DevMemSysAllocator)];
154
155 static const int kStaticAllocators = 3;
156 // kMaxDynamicAllocators + kStaticAllocators;
157 static const int kMaxAllocators = 5;
158 static SysAllocator *allocators[kMaxAllocators];
159
RegisterSystemAllocator(SysAllocator * a,int priority)160 bool RegisterSystemAllocator(SysAllocator *a, int priority) {
161 SpinLockHolder lock_holder(&spinlock);
162
163 // No two allocators should have a priority conflict, since the order
164 // is determined at compile time.
165 CHECK_CONDITION(allocators[priority] == NULL);
166 allocators[priority] = a;
167 return true;
168 }
169
170
Alloc(size_t size,size_t * actual_size,size_t alignment)171 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size,
172 size_t alignment) {
173 #ifndef HAVE_SBRK
174 failed_ = true;
175 return NULL;
176 #else
177 // Check if we should use sbrk allocation.
178 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized
179 // state) and eventually gets initialized to the specified value. Note
180 // that this code runs for a while before the flags are initialized.
181 // That means that even if this flag is set to true, some (initial)
182 // memory will be allocated with sbrk before the flag takes effect.
183 if (FLAGS_malloc_skip_sbrk) {
184 return NULL;
185 }
186
187 // sbrk will release memory if passed a negative number, so we do
188 // a strict check here
189 if ( static_cast<ptrdiff_t>(size + alignment) < 0 ) return NULL;
190
191 // could theoretically return the "extra" bytes here, but this
192 // is simple and correct.
193 if (actual_size) {
194 *actual_size = size;
195 }
196
197 // This doesn't overflow because TCMalloc_SystemAlloc has already
198 // tested for overflow at the alignment boundary.
199 size = ((size + alignment - 1) / alignment) * alignment;
200
201 // Check that we we're not asking for so much more memory that we'd
202 // wrap around the end of the virtual address space. (This seems
203 // like something sbrk() should check for us, and indeed opensolaris
204 // does, but glibc does not:
205 // http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/port/sys/sbrk.c?a=true
206 // http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/libc/misc/sbrk.c?rev=1.1.2.1&content-type=text/plain&cvsroot=glibc
207 // Without this check, sbrk may succeed when it ought to fail.)
208 if (reinterpret_cast<intptr_t>(sbrk(0)) + size < size) {
209 failed_ = true;
210 return NULL;
211 }
212
213 void* result = sbrk(size);
214 if (result == reinterpret_cast<void*>(-1)) {
215 failed_ = true;
216 return NULL;
217 }
218
219 // Is it aligned?
220 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
221 if ((ptr & (alignment-1)) == 0) return result;
222
223 // Try to get more memory for alignment
224 size_t extra = alignment - (ptr & (alignment-1));
225 void* r2 = sbrk(extra);
226 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
227 // Contiguous with previous result
228 return reinterpret_cast<void*>(ptr + extra);
229 }
230
231 // Give up and ask for "size + alignment - 1" bytes so
232 // that we can find an aligned region within it.
233 result = sbrk(size + alignment - 1);
234 if (result == reinterpret_cast<void*>(-1)) {
235 failed_ = true;
236 return NULL;
237 }
238 ptr = reinterpret_cast<uintptr_t>(result);
239 if ((ptr & (alignment-1)) != 0) {
240 ptr += alignment - (ptr & (alignment-1));
241 }
242 return reinterpret_cast<void*>(ptr);
243 #endif // HAVE_SBRK
244 }
245
DumpStats(TCMalloc_Printer * printer)246 void SbrkSysAllocator::DumpStats(TCMalloc_Printer* printer) {
247 printer->printf("SbrkSysAllocator: failed_=%d\n", failed_);
248 }
249
Alloc(size_t size,size_t * actual_size,size_t alignment)250 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size,
251 size_t alignment) {
252 #ifndef HAVE_MMAP
253 failed_ = true;
254 return NULL;
255 #else
256 // Check if we should use mmap allocation.
257 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized
258 // state) and eventually gets initialized to the specified value. Note
259 // that this code runs for a while before the flags are initialized.
260 // Chances are we never get here before the flags are initialized since
261 // sbrk is used until the heap is exhausted (before mmap is used).
262 if (FLAGS_malloc_skip_mmap) {
263 return NULL;
264 }
265
266 // could theoretically return the "extra" bytes here, but this
267 // is simple and correct.
268 if (actual_size) {
269 *actual_size = size;
270 }
271
272 // Enforce page alignment
273 if (pagesize == 0) pagesize = getpagesize();
274 if (alignment < pagesize) alignment = pagesize;
275 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
276 if (aligned_size < size) {
277 return NULL;
278 }
279 size = aligned_size;
280
281 // Ask for extra memory if alignment > pagesize
282 size_t extra = 0;
283 if (alignment > pagesize) {
284 extra = alignment - pagesize;
285 }
286
287 // Note: size + extra does not overflow since:
288 // size + alignment < (1<<NBITS).
289 // and extra <= alignment
290 // therefore size + extra < (1<<NBITS)
291 void* result = mmap(NULL, size + extra,
292 PROT_READ|PROT_WRITE,
293 MAP_PRIVATE|MAP_ANONYMOUS,
294 -1, 0);
295 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
296 failed_ = true;
297 return NULL;
298 }
299
300 // Adjust the return memory so it is aligned
301 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
302 size_t adjust = 0;
303 if ((ptr & (alignment - 1)) != 0) {
304 adjust = alignment - (ptr & (alignment - 1));
305 }
306
307 // Return the unused memory to the system
308 if (adjust > 0) {
309 munmap(reinterpret_cast<void*>(ptr), adjust);
310 }
311 if (adjust < extra) {
312 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
313 }
314
315 ptr += adjust;
316 return reinterpret_cast<void*>(ptr);
317 #endif // HAVE_MMAP
318 }
319
DumpStats(TCMalloc_Printer * printer)320 void MmapSysAllocator::DumpStats(TCMalloc_Printer* printer) {
321 printer->printf("MmapSysAllocator: failed_=%d\n", failed_);
322 }
323
Alloc(size_t size,size_t * actual_size,size_t alignment)324 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size,
325 size_t alignment) {
326 #ifndef HAVE_MMAP
327 failed_ = true;
328 return NULL;
329 #else
330 static bool initialized = false;
331 static off_t physmem_base; // next physical memory address to allocate
332 static off_t physmem_limit; // maximum physical address allowed
333 static int physmem_fd; // file descriptor for /dev/mem
334
335 // Check if we should use /dev/mem allocation. Note that it may take
336 // a while to get this flag initialized, so meanwhile we fall back to
337 // the next allocator. (It looks like 7MB gets allocated before
338 // this flag gets initialized -khr.)
339 if (FLAGS_malloc_devmem_start == 0) {
340 // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to
341 // try us again next time.
342 return NULL;
343 }
344
345 if (!initialized) {
346 physmem_fd = open("/dev/mem", O_RDWR);
347 if (physmem_fd < 0) {
348 failed_ = true;
349 return NULL;
350 }
351 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
352 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
353 initialized = true;
354 }
355
356 // could theoretically return the "extra" bytes here, but this
357 // is simple and correct.
358 if (actual_size) {
359 *actual_size = size;
360 }
361
362 // Enforce page alignment
363 if (pagesize == 0) pagesize = getpagesize();
364 if (alignment < pagesize) alignment = pagesize;
365 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
366 if (aligned_size < size) {
367 return NULL;
368 }
369 size = aligned_size;
370
371 // Ask for extra memory if alignment > pagesize
372 size_t extra = 0;
373 if (alignment > pagesize) {
374 extra = alignment - pagesize;
375 }
376
377 // check to see if we have any memory left
378 if (physmem_limit != 0 &&
379 ((size + extra) > static_cast<unsigned int>(physmem_limit - physmem_base))) {
380 failed_ = true;
381 return NULL;
382 }
383
384 // Note: size + extra does not overflow since:
385 // size + alignment < (1<<NBITS).
386 // and extra <= alignment
387 // therefore size + extra < (1<<NBITS)
388 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
389 MAP_SHARED, physmem_fd, physmem_base);
390 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
391 failed_ = true;
392 return NULL;
393 }
394 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
395
396 // Adjust the return memory so it is aligned
397 size_t adjust = 0;
398 if ((ptr & (alignment - 1)) != 0) {
399 adjust = alignment - (ptr & (alignment - 1));
400 }
401
402 // Return the unused virtual memory to the system
403 if (adjust > 0) {
404 munmap(reinterpret_cast<void*>(ptr), adjust);
405 }
406 if (adjust < extra) {
407 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
408 }
409
410 ptr += adjust;
411 physmem_base += adjust + size;
412
413 return reinterpret_cast<void*>(ptr);
414 #endif // HAVE_MMAP
415 }
416
DumpStats(TCMalloc_Printer * printer)417 void DevMemSysAllocator::DumpStats(TCMalloc_Printer* printer) {
418 printer->printf("DevMemSysAllocator: failed_=%d\n", failed_);
419 }
420
421 static bool system_alloc_inited = false;
InitSystemAllocators(void)422 void InitSystemAllocators(void) {
423 // This determines the order in which system allocators are called
424 int i = kMaxDynamicAllocators;
425 allocators[i++] = new (devmem_space) DevMemSysAllocator();
426
427 // In 64-bit debug mode, place the mmap allocator first since it
428 // allocates pointers that do not fit in 32 bits and therefore gives
429 // us better testing of code's 64-bit correctness. It also leads to
430 // less false negatives in heap-checking code. (Numbers are less
431 // likely to look like pointers and therefore the conservative gc in
432 // the heap-checker is less likely to misinterpret a number as a
433 // pointer).
434 if (kDebugMode && sizeof(void*) > 4) {
435 allocators[i++] = new (mmap_space) MmapSysAllocator();
436 allocators[i++] = new (sbrk_space) SbrkSysAllocator();
437 } else {
438 allocators[i++] = new (sbrk_space) SbrkSysAllocator();
439 allocators[i++] = new (mmap_space) MmapSysAllocator();
440 }
441 }
442
TCMalloc_SystemAlloc(size_t size,size_t * actual_size,size_t alignment)443 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
444 size_t alignment) {
445 // Discard requests that overflow
446 if (size + alignment < size) return NULL;
447
448 SpinLockHolder lock_holder(&spinlock);
449
450 if (!system_alloc_inited) {
451 InitSystemAllocators();
452 system_alloc_inited = true;
453 }
454
455 // Enforce minimum alignment
456 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
457
458 // Try twice, once avoiding allocators that failed before, and once
459 // more trying all allocators even if they failed before.
460 for (int i = 0; i < 2; i++) {
461 for (int j = 0; j < kMaxAllocators; j++) {
462 SysAllocator *a = allocators[j];
463 if (a == NULL) continue;
464 if (a->usable_ && !a->failed_) {
465 void* result = a->Alloc(size, actual_size, alignment);
466 if (result != NULL) return result;
467 }
468 }
469
470 // nothing worked - reset failed_ flags and try again
471 for (int j = 0; j < kMaxAllocators; j++) {
472 SysAllocator *a = allocators[j];
473 if (a == NULL) continue;
474 a->failed_ = false;
475 }
476 }
477 return NULL;
478 }
479
TCMalloc_SystemRelease(void * start,size_t length)480 void TCMalloc_SystemRelease(void* start, size_t length) {
481 #ifdef MADV_DONTNEED
482 if (FLAGS_malloc_devmem_start) {
483 // It's not safe to use MADV_DONTNEED if we've been mapping
484 // /dev/mem for heap memory
485 return;
486 }
487 if (pagesize == 0) pagesize = getpagesize();
488 const size_t pagemask = pagesize - 1;
489
490 size_t new_start = reinterpret_cast<size_t>(start);
491 size_t end = new_start + length;
492 size_t new_end = end;
493
494 // Round up the starting address and round down the ending address
495 // to be page aligned:
496 new_start = (new_start + pagesize - 1) & ~pagemask;
497 new_end = new_end & ~pagemask;
498
499 ASSERT((new_start & pagemask) == 0);
500 ASSERT((new_end & pagemask) == 0);
501 ASSERT(new_start >= reinterpret_cast<size_t>(start));
502 ASSERT(new_end <= end);
503
504 if (new_end > new_start) {
505 // Note -- ignoring most return codes, because if this fails it
506 // doesn't matter...
507 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
508 MADV_DONTNEED) == -1 &&
509 errno == EAGAIN) {
510 // NOP
511 }
512 }
513 #endif
514 }
515
DumpSystemAllocatorStats(TCMalloc_Printer * printer)516 void DumpSystemAllocatorStats(TCMalloc_Printer* printer) {
517 for (int j = 0; j < kMaxAllocators; j++) {
518 SysAllocator *a = allocators[j];
519 if (a == NULL) continue;
520 if (a->usable_) {
521 a->DumpStats(printer);
522 }
523 }
524 }
525