1 // -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
2 // Copyright (c) 2005, Google Inc.
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // ---
32 // Unittest for the TCMalloc implementation.
33 //
34 // * The test consists of a set of threads.
35 // * Each thread maintains a set of allocated objects, with
36 // a bound on the total amount of data in the set.
37 // * Each allocated object's contents are generated by
38 // hashing the object pointer, and a generation count
39 // in the object. This allows us to easily check for
40 // data corruption.
41 // * At any given step, the thread can do any of the following:
42 // a. Allocate an object
43 // b. Increment an object's generation count and update
44 // its contents.
45 // c. Pass the object to another thread
46 // d. Free an object
47 // Also, at the end of every step, object(s) are freed to maintain
48 // the memory upper-bound.
49 //
50 // If this test is compiled with -DDEBUGALLOCATION, then we don't
51 // run some tests that test the inner workings of tcmalloc and
52 // break on debugallocation: that certain allocations are aligned
53 // in a certain way (even though no standard requires it), and that
54 // realloc() tries to minimize copying (which debug allocators don't
55 // care about).
56
57 #include "config_for_unittests.h"
58 // Complicated ordering requirements. tcmalloc.h defines (indirectly)
59 // _POSIX_C_SOURCE, which it needs so stdlib.h defines posix_memalign.
60 // unistd.h, on the other hand, requires _POSIX_C_SOURCE to be unset,
61 // at least on FreeBSD, in order to define sbrk. The solution
62 // is to #include unistd.h first. This is safe because unistd.h
63 // doesn't sub-include stdlib.h, so we'll still get posix_memalign
64 // when we #include stdlib.h. Blah.
65 #ifdef HAVE_UNISTD_H
66 #include <unistd.h> // for testing sbrk hooks
67 #endif
68 #include "tcmalloc.h" // must come early, to pick up posix_memalign
69 #include <stdlib.h>
70 #include <string.h>
71 #include <stdio.h>
72 #ifdef HAVE_STDINT_H
73 #include <stdint.h> // for intptr_t
74 #endif
75 #include <sys/types.h> // for size_t
76 #ifdef HAVE_FCNTL_H
77 #include <fcntl.h> // for open; used with mmap-hook test
78 #endif
79 #ifdef HAVE_MMAP
80 #include <sys/mman.h> // for testing mmap hooks
81 #endif
82 #ifdef HAVE_MALLOC_H
83 #include <malloc.h> // defines pvalloc/etc on cygwin
84 #endif
85 #include <assert.h>
86 #include <vector>
87 #include <algorithm>
88 #include <string>
89 #include <new>
90 #include "base/logging.h"
91 #include "base/simple_mutex.h"
92 #include "gperftools/malloc_hook.h"
93 #include "gperftools/malloc_extension.h"
94 #include "gperftools/nallocx.h"
95 #include "gperftools/tcmalloc.h"
96 #include "thread_cache.h"
97 #include "system-alloc.h"
98 #include "tests/testutil.h"
99
100 // Windows doesn't define pvalloc and a few other obsolete unix
101 // functions; nor does it define posix_memalign (which is not obsolete).
102 #if defined(_WIN32)
103 # define cfree free // don't bother to try to test these obsolete fns
104 # define valloc malloc
105 # define pvalloc malloc
106 // I'd like to map posix_memalign to _aligned_malloc, but _aligned_malloc
107 // must be paired with _aligned_free (not normal free), which is too
108 // invasive a change to how we allocate memory here. So just bail
109 static bool kOSSupportsMemalign = false;
Memalign(size_t align,size_t size)110 static inline void* Memalign(size_t align, size_t size) {
111 //LOG(FATAL) << "memalign not supported on windows";
112 exit(1);
113 return NULL;
114 }
PosixMemalign(void ** ptr,size_t align,size_t size)115 static inline int PosixMemalign(void** ptr, size_t align, size_t size) {
116 //LOG(FATAL) << "posix_memalign not supported on windows";
117 exit(1);
118 return -1;
119 }
120
121 // OS X defines posix_memalign in some OS versions but not others;
122 // it's confusing enough to check that it's easiest to just not to test.
123 #elif defined(__APPLE__)
124 static bool kOSSupportsMemalign = false;
Memalign(size_t align,size_t size)125 static inline void* Memalign(size_t align, size_t size) {
126 //LOG(FATAL) << "memalign not supported on OS X";
127 exit(1);
128 return NULL;
129 }
PosixMemalign(void ** ptr,size_t align,size_t size)130 static inline int PosixMemalign(void** ptr, size_t align, size_t size) {
131 //LOG(FATAL) << "posix_memalign not supported on OS X";
132 exit(1);
133 return -1;
134 }
135
136 #else
137 static bool kOSSupportsMemalign = true;
Memalign(size_t align,size_t size)138 static inline void* Memalign(size_t align, size_t size) {
139 return memalign(align, size);
140 }
PosixMemalign(void ** ptr,size_t align,size_t size)141 static inline int PosixMemalign(void** ptr, size_t align, size_t size) {
142 return posix_memalign(ptr, align, size);
143 }
144
145 #endif
146
147 #if defined(ENABLE_ALIGNED_NEW_DELETE)
148
149 #define OVERALIGNMENT 64
150
151 struct overaligned_type
152 {
153 #if defined(__GNUC__)
154 __attribute__((__aligned__(OVERALIGNMENT)))
155 #elif defined(_MSC_VER)
156 __declspec(align(OVERALIGNMENT))
157 #else
158 alignas(OVERALIGNMENT)
159 #endif
160 unsigned char data[OVERALIGNMENT * 2]; // make the object size different from
161 // alignment to make sure the correct
162 // values are passed to the new/delete
163 // implementation functions
164 };
165
166 #endif // defined(ENABLE_ALIGNED_NEW_DELETE)
167
168 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old
169 // form of the name instead.
170 #ifndef MAP_ANONYMOUS
171 # define MAP_ANONYMOUS MAP_ANON
172 #endif
173
174 #define LOGSTREAM stdout
175
176 using std::vector;
177 using std::string;
178
179 DECLARE_double(tcmalloc_release_rate);
180 DECLARE_int32(max_free_queue_size); // in debugallocation.cc
181 DECLARE_int64(tcmalloc_sample_parameter);
182
183 struct OOMAbleSysAlloc : public SysAllocator {
184 SysAllocator *child;
185 int simulate_oom;
186
AllocOOMAbleSysAlloc187 void* Alloc(size_t size, size_t* actual_size, size_t alignment) {
188 if (simulate_oom) {
189 return NULL;
190 }
191 return child->Alloc(size, actual_size, alignment);
192 }
193 };
194
195 static union {
196 char buf[sizeof(OOMAbleSysAlloc)];
197 void *ptr;
198 } test_sys_alloc_space;
199
get_test_sys_alloc()200 static OOMAbleSysAlloc* get_test_sys_alloc() {
201 return reinterpret_cast<OOMAbleSysAlloc*>(&test_sys_alloc_space);
202 }
203
setup_oomable_sys_alloc()204 void setup_oomable_sys_alloc() {
205 SysAllocator *def = MallocExtension::instance()->GetSystemAllocator();
206
207 OOMAbleSysAlloc *alloc = get_test_sys_alloc();
208 new (alloc) OOMAbleSysAlloc;
209 alloc->child = def;
210
211 MallocExtension::instance()->SetSystemAllocator(alloc);
212 }
213
214 namespace testing {
215
216 static const int FLAGS_numtests = 50000;
217 static const int FLAGS_log_every_n_tests = 50000; // log exactly once
218
219 // Testing parameters
220 static const int FLAGS_lgmaxsize = 16; // lg() of the max size object to alloc
221 static const int FLAGS_numthreads = 10; // Number of threads
222 static const int FLAGS_threadmb = 4; // Max memory size allocated by thread
223 static const int FLAGS_lg_max_memalign = 18; // lg of max alignment for memalign
224
225 static const double FLAGS_memalign_min_fraction = 0; // min expected%
226 static const double FLAGS_memalign_max_fraction = 0.4; // max expected%
227 static const double FLAGS_memalign_max_alignment_ratio = 6; // alignment/size
228
229 // Weights of different operations
230 static const int FLAGS_allocweight = 50; // Weight for picking allocation
231 static const int FLAGS_freeweight = 50; // Weight for picking free
232 static const int FLAGS_updateweight = 10; // Weight for picking update
233 static const int FLAGS_passweight = 1; // Weight for passing object
234
235 static const int kSizeBits = 8 * sizeof(size_t);
236 static const size_t kMaxSize = ~static_cast<size_t>(0);
237 static const size_t kMaxSignedSize = ((size_t(1) << (kSizeBits-1)) - 1);
238
239 static const size_t kNotTooBig = 100000;
240 // We want an allocation that is definitely more than main memory. OS
241 // X has special logic to discard very big allocs before even passing
242 // the request along to the user-defined memory allocator; we're not
243 // interested in testing their logic, so we have to make sure we're
244 // not *too* big.
245 static const size_t kTooBig = kMaxSize - 100000;
246
247 static int news_handled = 0;
248
249 // Global array of threads
250 class TesterThread;
251 static TesterThread** threads;
252
253 // To help with generating random numbers
254 class TestHarness {
255 private:
256 // Information kept per type
257 struct Type {
258 string name;
259 int type;
260 int weight;
261 };
262
263 public:
TestHarness(int seed)264 TestHarness(int seed)
265 : types_(new vector<Type>), total_weight_(0), num_tests_(0) {
266 srandom(seed);
267 }
~TestHarness()268 ~TestHarness() {
269 delete types_;
270 }
271
272 // Add operation type with specified weight. When starting a new
273 // iteration, an operation type is picked with probability
274 // proportional to its weight.
275 //
276 // "type" must be non-negative.
277 // "weight" must be non-negative.
278 void AddType(int type, int weight, const char* name);
279
280 // Call this to get the type of operation for the next iteration.
281 // It returns a random operation type from the set of registered
282 // operations. Returns -1 if tests should finish.
283 int PickType();
284
285 // If n == 0, returns the next pseudo-random number in the range [0 .. 0]
286 // If n != 0, returns the next pseudo-random number in the range [0 .. n)
Uniform(int n)287 int Uniform(int n) {
288 if (n == 0) {
289 return random() * 0;
290 } else {
291 return random() % n;
292 }
293 }
294 // Pick "base" uniformly from range [0,max_log] and then return
295 // "base" random bits. The effect is to pick a number in the range
296 // [0,2^max_log-1] with bias towards smaller numbers.
Skewed(int max_log)297 int Skewed(int max_log) {
298 const int base = random() % (max_log+1);
299 return random() % (1 << base);
300 }
301
302 private:
303 vector<Type>* types_; // Registered types
304 int total_weight_; // Total weight of all types
305 int num_tests_; // Num tests run so far
306 };
307
AddType(int type,int weight,const char * name)308 void TestHarness::AddType(int type, int weight, const char* name) {
309 Type t;
310 t.name = name;
311 t.type = type;
312 t.weight = weight;
313 types_->push_back(t);
314 total_weight_ += weight;
315 }
316
PickType()317 int TestHarness::PickType() {
318 if (num_tests_ >= FLAGS_numtests) return -1;
319 num_tests_++;
320
321 assert(total_weight_ > 0);
322 // This is a little skewed if total_weight_ doesn't divide 2^31, but it's close
323 int v = Uniform(total_weight_);
324 int i;
325 for (i = 0; i < types_->size(); i++) {
326 v -= (*types_)[i].weight;
327 if (v < 0) {
328 break;
329 }
330 }
331
332 assert(i < types_->size());
333 if ((num_tests_ % FLAGS_log_every_n_tests) == 0) {
334 fprintf(LOGSTREAM, " Test %d out of %d: %s\n",
335 num_tests_, FLAGS_numtests, (*types_)[i].name.c_str());
336 }
337 return (*types_)[i].type;
338 }
339
340 class AllocatorState : public TestHarness {
341 public:
AllocatorState(int seed)342 explicit AllocatorState(int seed) : TestHarness(seed), memalign_fraction_(0) {
343 if (kOSSupportsMemalign) {
344 CHECK_GE(FLAGS_memalign_max_fraction, 0);
345 CHECK_LE(FLAGS_memalign_max_fraction, 1);
346 CHECK_GE(FLAGS_memalign_min_fraction, 0);
347 CHECK_LE(FLAGS_memalign_min_fraction, 1);
348 double delta = FLAGS_memalign_max_fraction - FLAGS_memalign_min_fraction;
349 CHECK_GE(delta, 0);
350 memalign_fraction_ = (Uniform(10000)/10000.0 * delta +
351 FLAGS_memalign_min_fraction);
352 //fprintf(LOGSTREAM, "memalign fraction: %f\n", memalign_fraction_);
353 }
354 }
~AllocatorState()355 virtual ~AllocatorState() {}
356
357 // Allocate memory. Randomly choose between malloc() or posix_memalign().
alloc(size_t size)358 void* alloc(size_t size) {
359 if (Uniform(100) < memalign_fraction_ * 100) {
360 // Try a few times to find a reasonable alignment, or fall back on malloc.
361 for (int i = 0; i < 5; i++) {
362 size_t alignment = 1 << Uniform(FLAGS_lg_max_memalign);
363 if (alignment >= sizeof(intptr_t) &&
364 (size < sizeof(intptr_t) ||
365 alignment < FLAGS_memalign_max_alignment_ratio * size)) {
366 void *result = reinterpret_cast<void*>(static_cast<intptr_t>(0x1234));
367 int err = PosixMemalign(&result, alignment, size);
368 if (err != 0) {
369 CHECK_EQ(err, ENOMEM);
370 }
371 return err == 0 ? result : NULL;
372 }
373 }
374 }
375 return malloc(size);
376 }
377
378 private:
379 double memalign_fraction_;
380 };
381
382
383 // Info kept per thread
384 class TesterThread {
385 private:
386 // Info kept per allocated object
387 struct Object {
388 char* ptr; // Allocated pointer
389 int size; // Allocated size
390 int generation; // Generation counter of object contents
391 };
392
393 Mutex lock_; // For passing in another thread's obj
394 int id_; // My thread id
395 AllocatorState rnd_; // For generating random numbers
396 vector<Object> heap_; // This thread's heap
397 vector<Object> passed_; // Pending objects passed from others
398 size_t heap_size_; // Current heap size
399 int locks_ok_; // Number of OK TryLock() ops
400 int locks_failed_; // Number of failed TryLock() ops
401
402 // Type of operations
403 enum Type { ALLOC, FREE, UPDATE, PASS };
404
405 // ACM minimal standard random number generator. (re-entrant.)
406 class ACMRandom {
407 int32 seed_;
408 public:
ACMRandom(int32 seed)409 explicit ACMRandom(int32 seed) { seed_ = seed; }
Next()410 int32 Next() {
411 const int32 M = 2147483647L; // 2^31-1
412 const int32 A = 16807;
413 // In effect, we are computing seed_ = (seed_ * A) % M, where M = 2^31-1
414 uint32 lo = A * (int32)(seed_ & 0xFFFF);
415 uint32 hi = A * (int32)((uint32)seed_ >> 16);
416 lo += (hi & 0x7FFF) << 16;
417 if (lo > M) {
418 lo &= M;
419 ++lo;
420 }
421 lo += hi >> 15;
422 if (lo > M) {
423 lo &= M;
424 ++lo;
425 }
426 return (seed_ = (int32) lo);
427 }
428 };
429
430 public:
TesterThread(int id)431 TesterThread(int id)
432 : id_(id),
433 rnd_(id+1),
434 heap_size_(0),
435 locks_ok_(0),
436 locks_failed_(0) {
437 }
438
~TesterThread()439 virtual ~TesterThread() {
440 if (FLAGS_verbose)
441 fprintf(LOGSTREAM, "Thread %2d: locks %6d ok; %6d trylocks failed\n",
442 id_, locks_ok_, locks_failed_);
443 if (locks_ok_ + locks_failed_ >= 1000) {
444 CHECK_LE(locks_failed_, locks_ok_ / 2);
445 }
446 }
447
Run()448 virtual void Run() {
449 rnd_.AddType(ALLOC, FLAGS_allocweight, "allocate");
450 rnd_.AddType(FREE, FLAGS_freeweight, "free");
451 rnd_.AddType(UPDATE, FLAGS_updateweight, "update");
452 rnd_.AddType(PASS, FLAGS_passweight, "pass");
453
454 while (true) {
455 AcquirePassedObjects();
456
457 switch (rnd_.PickType()) {
458 case ALLOC: AllocateObject(); break;
459 case FREE: FreeObject(); break;
460 case UPDATE: UpdateObject(); break;
461 case PASS: PassObject(); break;
462 case -1: goto done;
463 default: assert(NULL == "Unknown type");
464 }
465
466 ShrinkHeap();
467 }
468
469 done:
470 DeleteHeap();
471 }
472
473 // Allocate a new object
AllocateObject()474 void AllocateObject() {
475 Object object;
476 object.size = rnd_.Skewed(FLAGS_lgmaxsize);
477 object.ptr = static_cast<char*>(rnd_.alloc(object.size));
478 CHECK(object.ptr);
479 object.generation = 0;
480 FillContents(&object);
481 heap_.push_back(object);
482 heap_size_ += object.size;
483 }
484
485 // Mutate a random object
UpdateObject()486 void UpdateObject() {
487 if (heap_.empty()) return;
488 const int index = rnd_.Uniform(heap_.size());
489 CheckContents(heap_[index]);
490 heap_[index].generation++;
491 FillContents(&heap_[index]);
492 }
493
494 // Free a random object
FreeObject()495 void FreeObject() {
496 if (heap_.empty()) return;
497 const int index = rnd_.Uniform(heap_.size());
498 Object object = heap_[index];
499 CheckContents(object);
500 free(object.ptr);
501 heap_size_ -= object.size;
502 heap_[index] = heap_[heap_.size()-1];
503 heap_.pop_back();
504 }
505
506 // Delete all objects in the heap
DeleteHeap()507 void DeleteHeap() {
508 while (!heap_.empty()) {
509 FreeObject();
510 }
511 }
512
513 // Free objects until our heap is small enough
ShrinkHeap()514 void ShrinkHeap() {
515 while (heap_size_ > FLAGS_threadmb << 20) {
516 assert(!heap_.empty());
517 FreeObject();
518 }
519 }
520
521 // Pass a random object to another thread
PassObject()522 void PassObject() {
523 // Pick object to pass
524 if (heap_.empty()) return;
525 const int index = rnd_.Uniform(heap_.size());
526 Object object = heap_[index];
527 CheckContents(object);
528
529 // Pick thread to pass
530 const int tid = rnd_.Uniform(FLAGS_numthreads);
531 TesterThread* thread = threads[tid];
532
533 if (thread->lock_.TryLock()) {
534 // Pass the object
535 locks_ok_++;
536 thread->passed_.push_back(object);
537 thread->lock_.Unlock();
538 heap_size_ -= object.size;
539 heap_[index] = heap_[heap_.size()-1];
540 heap_.pop_back();
541 } else {
542 locks_failed_++;
543 }
544 }
545
546 // Grab any objects passed to this thread by another thread
AcquirePassedObjects()547 void AcquirePassedObjects() {
548 // We do not create unnecessary contention by always using
549 // TryLock(). Plus we unlock immediately after swapping passed
550 // objects into a local vector.
551 vector<Object> copy;
552 { // Locking scope
553 if (!lock_.TryLock()) {
554 locks_failed_++;
555 return;
556 }
557 locks_ok_++;
558 swap(copy, passed_);
559 lock_.Unlock();
560 }
561
562 for (int i = 0; i < copy.size(); ++i) {
563 const Object& object = copy[i];
564 CheckContents(object);
565 heap_.push_back(object);
566 heap_size_ += object.size;
567 }
568 }
569
570 // Fill object contents according to ptr/generation
FillContents(Object * object)571 void FillContents(Object* object) {
572 ACMRandom r(reinterpret_cast<intptr_t>(object->ptr) & 0x7fffffff);
573 for (int i = 0; i < object->generation; ++i) {
574 r.Next();
575 }
576 const char c = static_cast<char>(r.Next());
577 memset(object->ptr, c, object->size);
578 }
579
580 // Check object contents
CheckContents(const Object & object)581 void CheckContents(const Object& object) {
582 ACMRandom r(reinterpret_cast<intptr_t>(object.ptr) & 0x7fffffff);
583 for (int i = 0; i < object.generation; ++i) {
584 r.Next();
585 }
586
587 // For large objects, we just check a prefix/suffix
588 const char expected = static_cast<char>(r.Next());
589 const int limit1 = object.size < 32 ? object.size : 32;
590 const int start2 = limit1 > object.size - 32 ? limit1 : object.size - 32;
591 for (int i = 0; i < limit1; ++i) {
592 CHECK_EQ(object.ptr[i], expected);
593 }
594 for (int i = start2; i < object.size; ++i) {
595 CHECK_EQ(object.ptr[i], expected);
596 }
597 }
598 };
599
RunThread(int thread_id)600 static void RunThread(int thread_id) {
601 threads[thread_id]->Run();
602 }
603
TryHugeAllocation(size_t s,AllocatorState * rnd)604 static void TryHugeAllocation(size_t s, AllocatorState* rnd) {
605 void* p = rnd->alloc(s);
606 CHECK(p == NULL); // huge allocation s should fail!
607 }
608
TestHugeAllocations(AllocatorState * rnd)609 static void TestHugeAllocations(AllocatorState* rnd) {
610 // Check that asking for stuff tiny bit smaller than largest possible
611 // size returns NULL.
612 for (size_t i = 0; i < 70000; i += rnd->Uniform(20)) {
613 TryHugeAllocation(kMaxSize - i, rnd);
614 }
615 // Asking for memory sizes near signed/unsigned boundary (kMaxSignedSize)
616 // might work or not, depending on the amount of virtual memory.
617 #ifndef DEBUGALLOCATION // debug allocation takes forever for huge allocs
618 for (size_t i = 0; i < 100; i++) {
619 void* p = NULL;
620 p = rnd->alloc(kMaxSignedSize + i);
621 if (p) free(p); // if: free(NULL) is not necessarily defined
622 p = rnd->alloc(kMaxSignedSize - i);
623 if (p) free(p);
624 }
625 #endif
626
627 // Check that ReleaseFreeMemory has no visible effect (aka, does not
628 // crash the test):
629 MallocExtension* inst = MallocExtension::instance();
630 CHECK(inst);
631 inst->ReleaseFreeMemory();
632 }
633
TestCalloc(size_t n,size_t s,bool ok)634 static void TestCalloc(size_t n, size_t s, bool ok) {
635 char* p = reinterpret_cast<char*>(calloc(n, s));
636 if (FLAGS_verbose)
637 fprintf(LOGSTREAM, "calloc(%" PRIxS ", %" PRIxS "): %p\n", n, s, p);
638 if (!ok) {
639 CHECK(p == NULL); // calloc(n, s) should not succeed
640 } else {
641 CHECK(p != NULL); // calloc(n, s) should succeed
642 for (int i = 0; i < n*s; i++) {
643 CHECK(p[i] == '\0');
644 }
645 free(p);
646 }
647 }
648
649 // This makes sure that reallocing a small number of bytes in either
650 // direction doesn't cause us to allocate new memory.
TestRealloc()651 static void TestRealloc() {
652 #ifndef DEBUGALLOCATION // debug alloc doesn't try to minimize reallocs
653 // When sampling, we always allocate in units of page-size, which
654 // makes reallocs of small sizes do extra work (thus, failing these
655 // checks). Since sampling is random, we turn off sampling to make
656 // sure that doesn't happen to us here.
657 const int64 old_sample_parameter = FLAGS_tcmalloc_sample_parameter;
658 FLAGS_tcmalloc_sample_parameter = 0; // turn off sampling
659
660 int start_sizes[] = { 100, 1000, 10000, 100000 };
661 int deltas[] = { 1, -2, 4, -8, 16, -32, 64, -128 };
662
663 for (int s = 0; s < sizeof(start_sizes)/sizeof(*start_sizes); ++s) {
664 void* p = malloc(start_sizes[s]);
665 CHECK(p);
666 // The larger the start-size, the larger the non-reallocing delta.
667 for (int d = 0; d < (s+1) * 2; ++d) {
668 void* new_p = realloc(p, start_sizes[s] + deltas[d]);
669 CHECK(p == new_p); // realloc should not allocate new memory
670 }
671 // Test again, but this time reallocing smaller first.
672 for (int d = 0; d < s*2; ++d) {
673 void* new_p = realloc(p, start_sizes[s] - deltas[d]);
674 CHECK(p == new_p); // realloc should not allocate new memory
675 }
676 free(p);
677 }
678 FLAGS_tcmalloc_sample_parameter = old_sample_parameter;
679 #endif
680 }
681
TestNewHandler()682 static void TestNewHandler() {
683 ++news_handled;
684 throw std::bad_alloc();
685 }
686
TestOneNew(void * (* func)(size_t))687 static void TestOneNew(void* (*func)(size_t)) {
688 // success test
689 try {
690 void* ptr = (*func)(kNotTooBig);
691 if (0 == ptr) {
692 fprintf(LOGSTREAM, "allocation should not have failed.\n");
693 abort();
694 }
695 } catch (...) {
696 fprintf(LOGSTREAM, "allocation threw unexpected exception.\n");
697 abort();
698 }
699
700 // failure test
701 // we should always receive a bad_alloc exception
702 try {
703 (*func)(kTooBig);
704 fprintf(LOGSTREAM, "allocation should have failed.\n");
705 abort();
706 } catch (const std::bad_alloc&) {
707 // correct
708 } catch (...) {
709 fprintf(LOGSTREAM, "allocation threw unexpected exception.\n");
710 abort();
711 }
712 }
713
TestNew(void * (* func)(size_t))714 static void TestNew(void* (*func)(size_t)) {
715 news_handled = 0;
716
717 // test without new_handler:
718 std::new_handler saved_handler = std::set_new_handler(0);
719 TestOneNew(func);
720
721 // test with new_handler:
722 std::set_new_handler(TestNewHandler);
723 TestOneNew(func);
724 if (news_handled != 1) {
725 fprintf(LOGSTREAM, "new_handler was not called.\n");
726 abort();
727 }
728 std::set_new_handler(saved_handler);
729 }
730
TestOneNothrowNew(void * (* func)(size_t,const std::nothrow_t &))731 static void TestOneNothrowNew(void* (*func)(size_t, const std::nothrow_t&)) {
732 // success test
733 try {
734 void* ptr = (*func)(kNotTooBig, std::nothrow);
735 if (0 == ptr) {
736 fprintf(LOGSTREAM, "allocation should not have failed.\n");
737 abort();
738 }
739 } catch (...) {
740 fprintf(LOGSTREAM, "allocation threw unexpected exception.\n");
741 abort();
742 }
743
744 // failure test
745 // we should always receive a bad_alloc exception
746 try {
747 if ((*func)(kTooBig, std::nothrow) != 0) {
748 fprintf(LOGSTREAM, "allocation should have failed.\n");
749 abort();
750 }
751 } catch (...) {
752 fprintf(LOGSTREAM, "nothrow allocation threw unexpected exception.\n");
753 abort();
754 }
755 }
756
TestNothrowNew(void * (* func)(size_t,const std::nothrow_t &))757 static void TestNothrowNew(void* (*func)(size_t, const std::nothrow_t&)) {
758 news_handled = 0;
759
760 // test without new_handler:
761 std::new_handler saved_handler = std::set_new_handler(0);
762 TestOneNothrowNew(func);
763
764 // test with new_handler:
765 std::set_new_handler(TestNewHandler);
766 TestOneNothrowNew(func);
767 if (news_handled != 1) {
768 fprintf(LOGSTREAM, "nothrow new_handler was not called.\n");
769 abort();
770 }
771 std::set_new_handler(saved_handler);
772 }
773
774
775 // These are used as callbacks by the sanity-check. Set* and Reset*
776 // register the hook that counts how many times the associated memory
777 // function is called. After each such call, call Verify* to verify
778 // that we used the tcmalloc version of the call, and not the libc.
779 // Note the ... in the hook signature: we don't care what arguments
780 // the hook takes.
781 #define MAKE_HOOK_CALLBACK(hook_type, ...) \
782 static volatile int g_##hook_type##_calls = 0; \
783 static void IncrementCallsTo##hook_type(__VA_ARGS__) { \
784 g_##hook_type##_calls++; \
785 } \
786 static void Verify##hook_type##WasCalled() { \
787 CHECK_GT(g_##hook_type##_calls, 0); \
788 g_##hook_type##_calls = 0; /* reset for next call */ \
789 } \
790 static void Set##hook_type() { \
791 CHECK(MallocHook::Add##hook_type( \
792 (MallocHook::hook_type)&IncrementCallsTo##hook_type)); \
793 } \
794 static void Reset##hook_type() { \
795 CHECK(MallocHook::Remove##hook_type( \
796 (MallocHook::hook_type)&IncrementCallsTo##hook_type)); \
797 }
798
799 // We do one for each hook typedef in malloc_hook.h
800 MAKE_HOOK_CALLBACK(NewHook, const void*, size_t);
801 MAKE_HOOK_CALLBACK(DeleteHook, const void*);
802 MAKE_HOOK_CALLBACK(MmapHook, const void*, const void*, size_t, int, int, int,
803 off_t);
804 MAKE_HOOK_CALLBACK(MremapHook, const void*, const void*, size_t, size_t, int,
805 const void*);
806 MAKE_HOOK_CALLBACK(MunmapHook, const void *, size_t);
807 MAKE_HOOK_CALLBACK(SbrkHook, const void *, ptrdiff_t);
808
TestAlignmentForSize(int size)809 static void TestAlignmentForSize(int size) {
810 fprintf(LOGSTREAM, "Testing alignment of malloc(%d)\n", size);
811 static const int kNum = 100;
812 void* ptrs[kNum];
813 for (int i = 0; i < kNum; i++) {
814 ptrs[i] = malloc(size);
815 uintptr_t p = reinterpret_cast<uintptr_t>(ptrs[i]);
816 CHECK((p % sizeof(void*)) == 0);
817 CHECK((p % sizeof(double)) == 0);
818
819 // Must have 16-byte (or 8-byte in case of -DTCMALLOC_ALIGN_8BYTES)
820 // alignment for large enough objects
821 if (size >= kMinAlign) {
822 CHECK((p % kMinAlign) == 0);
823 }
824 }
825 for (int i = 0; i < kNum; i++) {
826 free(ptrs[i]);
827 }
828 }
829
TestMallocAlignment()830 static void TestMallocAlignment() {
831 for (int lg = 0; lg < 16; lg++) {
832 TestAlignmentForSize((1<<lg) - 1);
833 TestAlignmentForSize((1<<lg) + 0);
834 TestAlignmentForSize((1<<lg) + 1);
835 }
836 }
837
TestHugeThreadCache()838 static void TestHugeThreadCache() {
839 fprintf(LOGSTREAM, "==== Testing huge thread cache\n");
840 // More than 2^16 to cause integer overflow of 16 bit counters.
841 static const int kNum = 70000;
842 char** array = new char*[kNum];
843 for (int i = 0; i < kNum; ++i) {
844 array[i] = new char[10];
845 }
846 for (int i = 0; i < kNum; ++i) {
847 delete[] array[i];
848 }
849 delete[] array;
850 }
851
852 namespace {
853
854 struct RangeCallbackState {
855 uintptr_t ptr;
856 base::MallocRange::Type expected_type;
857 size_t min_size;
858 bool matched;
859 };
860
RangeCallback(void * arg,const base::MallocRange * r)861 static void RangeCallback(void* arg, const base::MallocRange* r) {
862 RangeCallbackState* state = reinterpret_cast<RangeCallbackState*>(arg);
863 if (state->ptr >= r->address &&
864 state->ptr < r->address + r->length) {
865 if (state->expected_type == base::MallocRange::FREE) {
866 // We are expecting r->type == FREE, but ReleaseMemory
867 // may have already moved us to UNMAPPED state instead (this happens in
868 // approximately 0.1% of executions). Accept either state.
869 CHECK(r->type == base::MallocRange::FREE ||
870 r->type == base::MallocRange::UNMAPPED);
871 } else {
872 CHECK_EQ(r->type, state->expected_type);
873 }
874 CHECK_GE(r->length, state->min_size);
875 state->matched = true;
876 }
877 }
878
879 // Check that at least one of the callbacks from Ranges() contains
880 // the specified address with the specified type, and has size
881 // >= min_size.
CheckRangeCallback(void * ptr,base::MallocRange::Type type,size_t min_size)882 static void CheckRangeCallback(void* ptr, base::MallocRange::Type type,
883 size_t min_size) {
884 RangeCallbackState state;
885 state.ptr = reinterpret_cast<uintptr_t>(ptr);
886 state.expected_type = type;
887 state.min_size = min_size;
888 state.matched = false;
889 MallocExtension::instance()->Ranges(&state, RangeCallback);
890 CHECK(state.matched);
891 }
892
893 }
894
895 static bool HaveSystemRelease =
896 TCMalloc_SystemRelease(TCMalloc_SystemAlloc(kPageSize, NULL, 0), kPageSize);
897
TestRanges()898 static void TestRanges() {
899 static const int MB = 1048576;
900 void* a = malloc(MB);
901 void* b = malloc(MB);
902 base::MallocRange::Type releasedType =
903 HaveSystemRelease ? base::MallocRange::UNMAPPED : base::MallocRange::FREE;
904
905 CheckRangeCallback(a, base::MallocRange::INUSE, MB);
906 CheckRangeCallback(b, base::MallocRange::INUSE, MB);
907 free(a);
908 CheckRangeCallback(a, base::MallocRange::FREE, MB);
909 CheckRangeCallback(b, base::MallocRange::INUSE, MB);
910 MallocExtension::instance()->ReleaseFreeMemory();
911 CheckRangeCallback(a, releasedType, MB);
912 CheckRangeCallback(b, base::MallocRange::INUSE, MB);
913 free(b);
914 CheckRangeCallback(a, releasedType, MB);
915 CheckRangeCallback(b, base::MallocRange::FREE, MB);
916 }
917
918 #ifndef DEBUGALLOCATION
GetUnmappedBytes()919 static size_t GetUnmappedBytes() {
920 size_t bytes;
921 CHECK(MallocExtension::instance()->GetNumericProperty(
922 "tcmalloc.pageheap_unmapped_bytes", &bytes));
923 return bytes;
924 }
925 #endif
926
927 class AggressiveDecommitChanger {
928 size_t old_value_;
929 public:
AggressiveDecommitChanger(size_t new_value)930 AggressiveDecommitChanger(size_t new_value) {
931 MallocExtension *inst = MallocExtension::instance();
932 bool rv = inst->GetNumericProperty("tcmalloc.aggressive_memory_decommit", &old_value_);
933 CHECK_CONDITION(rv);
934 rv = inst->SetNumericProperty("tcmalloc.aggressive_memory_decommit", new_value);
935 CHECK_CONDITION(rv);
936 }
~AggressiveDecommitChanger()937 ~AggressiveDecommitChanger() {
938 MallocExtension *inst = MallocExtension::instance();
939 bool rv = inst->SetNumericProperty("tcmalloc.aggressive_memory_decommit", old_value_);
940 CHECK_CONDITION(rv);
941 }
942 };
943
TestReleaseToSystem()944 static void TestReleaseToSystem() {
945 // Debug allocation mode adds overhead to each allocation which
946 // messes up all the equality tests here. I just disable the
947 // teset in this mode. TODO(csilvers): get it to work for debugalloc?
948 #ifndef DEBUGALLOCATION
949
950 if(!HaveSystemRelease) return;
951
952 const double old_tcmalloc_release_rate = FLAGS_tcmalloc_release_rate;
953 FLAGS_tcmalloc_release_rate = 0;
954
955 AggressiveDecommitChanger disabler(0);
956
957 static const int MB = 1048576;
958 void* a = malloc(MB);
959 void* b = malloc(MB);
960 MallocExtension::instance()->ReleaseFreeMemory();
961 size_t starting_bytes = GetUnmappedBytes();
962
963 // Calling ReleaseFreeMemory() a second time shouldn't do anything.
964 MallocExtension::instance()->ReleaseFreeMemory();
965 EXPECT_EQ(starting_bytes, GetUnmappedBytes());
966
967 // ReleaseToSystem shouldn't do anything either.
968 MallocExtension::instance()->ReleaseToSystem(MB);
969 EXPECT_EQ(starting_bytes, GetUnmappedBytes());
970
971 free(a);
972
973 // The span to release should be 1MB.
974 MallocExtension::instance()->ReleaseToSystem(MB/2);
975 EXPECT_EQ(starting_bytes + MB, GetUnmappedBytes());
976
977 // Should do nothing since the previous call released too much.
978 MallocExtension::instance()->ReleaseToSystem(MB/4);
979 EXPECT_EQ(starting_bytes + MB, GetUnmappedBytes());
980
981 free(b);
982
983 // Use up the extra MB/4 bytes from 'a' and also release 'b'.
984 MallocExtension::instance()->ReleaseToSystem(MB/2);
985 EXPECT_EQ(starting_bytes + 2*MB, GetUnmappedBytes());
986
987 // Should do nothing since the previous call released too much.
988 MallocExtension::instance()->ReleaseToSystem(MB/2);
989 EXPECT_EQ(starting_bytes + 2*MB, GetUnmappedBytes());
990
991 // Nothing else to release.
992 MallocExtension::instance()->ReleaseFreeMemory();
993 EXPECT_EQ(starting_bytes + 2*MB, GetUnmappedBytes());
994
995 a = malloc(MB);
996 free(a);
997 EXPECT_EQ(starting_bytes + MB, GetUnmappedBytes());
998
999 // Releasing less than a page should still trigger a release.
1000 MallocExtension::instance()->ReleaseToSystem(1);
1001 EXPECT_EQ(starting_bytes + 2*MB, GetUnmappedBytes());
1002
1003 FLAGS_tcmalloc_release_rate = old_tcmalloc_release_rate;
1004 #endif // #ifndef DEBUGALLOCATION
1005 }
1006
TestAggressiveDecommit()1007 static void TestAggressiveDecommit() {
1008 // Debug allocation mode adds overhead to each allocation which
1009 // messes up all the equality tests here. I just disable the
1010 // teset in this mode.
1011 #ifndef DEBUGALLOCATION
1012
1013 if(!HaveSystemRelease) return;
1014
1015 fprintf(LOGSTREAM, "Testing aggressive de-commit\n");
1016
1017 AggressiveDecommitChanger enabler(1);
1018
1019 static const int MB = 1048576;
1020 void* a = malloc(MB);
1021 void* b = malloc(MB);
1022
1023 size_t starting_bytes = GetUnmappedBytes();
1024
1025 // ReleaseToSystem shouldn't do anything either.
1026 MallocExtension::instance()->ReleaseToSystem(MB);
1027 EXPECT_EQ(starting_bytes, GetUnmappedBytes());
1028
1029 free(a);
1030
1031 // The span to release should be 1MB.
1032 EXPECT_EQ(starting_bytes + MB, GetUnmappedBytes());
1033
1034 free(b);
1035
1036 EXPECT_EQ(starting_bytes + 2*MB, GetUnmappedBytes());
1037
1038 // Nothing else to release.
1039 MallocExtension::instance()->ReleaseFreeMemory();
1040 EXPECT_EQ(starting_bytes + 2*MB, GetUnmappedBytes());
1041
1042 a = malloc(MB);
1043 free(a);
1044
1045 EXPECT_EQ(starting_bytes + 2*MB, GetUnmappedBytes());
1046
1047 fprintf(LOGSTREAM, "Done testing aggressive de-commit\n");
1048
1049 #endif // #ifndef DEBUGALLOCATION
1050 }
1051
1052 // On MSVC10, in release mode, the optimizer convinces itself
1053 // g_no_memory is never changed (I guess it doesn't realize OnNoMemory
1054 // might be called). Work around this by setting the var volatile.
1055 volatile bool g_no_memory = false;
1056 std::new_handler g_old_handler = NULL;
OnNoMemory()1057 static void OnNoMemory() {
1058 g_no_memory = true;
1059 std::set_new_handler(g_old_handler);
1060 }
1061
TestSetNewMode()1062 static void TestSetNewMode() {
1063 int old_mode = tc_set_new_mode(1);
1064
1065 g_old_handler = std::set_new_handler(&OnNoMemory);
1066 g_no_memory = false;
1067 void* ret = malloc(kTooBig);
1068 EXPECT_EQ(NULL, ret);
1069 EXPECT_TRUE(g_no_memory);
1070
1071 g_old_handler = std::set_new_handler(&OnNoMemory);
1072 g_no_memory = false;
1073 ret = calloc(1, kTooBig);
1074 EXPECT_EQ(NULL, ret);
1075 EXPECT_TRUE(g_no_memory);
1076
1077 g_old_handler = std::set_new_handler(&OnNoMemory);
1078 g_no_memory = false;
1079 ret = realloc(NULL, kTooBig);
1080 EXPECT_EQ(NULL, ret);
1081 EXPECT_TRUE(g_no_memory);
1082
1083 if (kOSSupportsMemalign) {
1084 // Not really important, but must be small enough such that
1085 // kAlignment + kTooBig does not overflow.
1086 const int kAlignment = 1 << 5;
1087
1088 g_old_handler = std::set_new_handler(&OnNoMemory);
1089 g_no_memory = false;
1090 ret = Memalign(kAlignment, kTooBig);
1091 EXPECT_EQ(NULL, ret);
1092 EXPECT_TRUE(g_no_memory);
1093
1094 g_old_handler = std::set_new_handler(&OnNoMemory);
1095 g_no_memory = false;
1096 EXPECT_EQ(ENOMEM,
1097 PosixMemalign(&ret, kAlignment, kTooBig));
1098 EXPECT_EQ(NULL, ret);
1099 EXPECT_TRUE(g_no_memory);
1100 }
1101
1102 tc_set_new_mode(old_mode);
1103 }
1104
TestErrno(void)1105 static void TestErrno(void) {
1106 void* ret;
1107 if (kOSSupportsMemalign) {
1108 errno = 0;
1109 ret = Memalign(128, kTooBig);
1110 EXPECT_EQ(NULL, ret);
1111 EXPECT_EQ(ENOMEM, errno);
1112 }
1113
1114 errno = 0;
1115 ret = malloc(kTooBig);
1116 EXPECT_EQ(NULL, ret);
1117 EXPECT_EQ(ENOMEM, errno);
1118
1119 errno = 0;
1120 ret = tc_malloc_skip_new_handler(kTooBig);
1121 EXPECT_EQ(NULL, ret);
1122 EXPECT_EQ(ENOMEM, errno);
1123 }
1124
1125
1126 #ifndef DEBUGALLOCATION
1127 // Ensure that nallocx works before main.
1128 struct GlobalNallocx {
GlobalNallocxtesting::GlobalNallocx1129 GlobalNallocx() { CHECK_GT(nallocx(99, 0), 99); }
1130 } global_nallocx;
1131
1132 #if defined(__GNUC__)
1133
1134 static void check_global_nallocx() __attribute__((constructor));
check_global_nallocx()1135 static void check_global_nallocx() { CHECK_GT(nallocx(99, 0), 99); }
1136
1137 #endif // __GNUC__
1138
TestNAllocX()1139 static void TestNAllocX() {
1140 for (size_t size = 0; size <= (1 << 20); size += 7) {
1141 size_t rounded = nallocx(size, 0);
1142 ASSERT_GE(rounded, size);
1143 void* ptr = malloc(size);
1144 ASSERT_EQ(rounded, MallocExtension::instance()->GetAllocatedSize(ptr));
1145 free(ptr);
1146 }
1147 }
1148
TestNAllocXAlignment()1149 static void TestNAllocXAlignment() {
1150 for (size_t size = 0; size <= (1 << 20); size += 7) {
1151 for (size_t align = 0; align < 10; align++) {
1152 size_t rounded = nallocx(size, MALLOCX_LG_ALIGN(align));
1153 ASSERT_GE(rounded, size);
1154 ASSERT_EQ(rounded % (1 << align), 0);
1155 void* ptr = tc_memalign(1 << align, size);
1156 ASSERT_EQ(rounded, MallocExtension::instance()->GetAllocatedSize(ptr));
1157 free(ptr);
1158 }
1159 }
1160 }
1161
1162 static int saw_new_handler_runs;
1163 static void* volatile oom_test_last_ptr;
1164
test_new_handler()1165 static void test_new_handler() {
1166 get_test_sys_alloc()->simulate_oom = false;
1167 void *ptr = oom_test_last_ptr;
1168 oom_test_last_ptr = NULL;
1169 ::operator delete[](ptr);
1170 saw_new_handler_runs++;
1171 }
1172
TestNewOOMHandling()1173 static ATTRIBUTE_NOINLINE void TestNewOOMHandling() {
1174 // debug allocator does internal allocations and crashes when such
1175 // internal allocation fails. So don't test it.
1176 setup_oomable_sys_alloc();
1177
1178 std::new_handler old = std::set_new_handler(test_new_handler);
1179 get_test_sys_alloc()->simulate_oom = true;
1180
1181 ASSERT_EQ(saw_new_handler_runs, 0);
1182
1183 for (int i = 0; i < 10240; i++) {
1184 oom_test_last_ptr = new char [512];
1185 ASSERT_NE(oom_test_last_ptr, NULL);
1186 if (saw_new_handler_runs) {
1187 break;
1188 }
1189 }
1190
1191 ASSERT_GE(saw_new_handler_runs, 1);
1192
1193 get_test_sys_alloc()->simulate_oom = false;
1194 std::set_new_handler(old);
1195 }
1196 #endif // !DEBUGALLOCATION
1197
RunAllTests(int argc,char ** argv)1198 static int RunAllTests(int argc, char** argv) {
1199 // Optional argv[1] is the seed
1200 AllocatorState rnd(argc > 1 ? atoi(argv[1]) : 100);
1201
1202 SetTestResourceLimit();
1203
1204 #ifndef DEBUGALLOCATION
1205 TestNewOOMHandling();
1206 #endif
1207
1208 // TODO(odo): This test has been disabled because it is only by luck that it
1209 // does not result in fragmentation. When tcmalloc makes an allocation which
1210 // spans previously unused leaves of the pagemap it will allocate and fill in
1211 // the leaves to cover the new allocation. The leaves happen to be 256MiB in
1212 // the 64-bit build, and with the sbrk allocator these allocations just
1213 // happen to fit in one leaf by luck. With other allocators (mmap,
1214 // memfs_malloc when used with small pages) the allocations generally span
1215 // two leaves and this results in a very bad fragmentation pattern with this
1216 // code. The same failure can be forced with the sbrk allocator just by
1217 // allocating something on the order of 128MiB prior to starting this test so
1218 // that the test allocations straddle a 256MiB boundary.
1219
1220 // TODO(csilvers): port MemoryUsage() over so the test can use that
1221 #if 0
1222 # include <unistd.h> // for getpid()
1223 // Allocate and deallocate blocks of increasing sizes to check if the alloc
1224 // metadata fragments the memory. (Do not put other allocations/deallocations
1225 // before this test, it may break).
1226 {
1227 size_t memory_usage = MemoryUsage(getpid());
1228 fprintf(LOGSTREAM, "Testing fragmentation\n");
1229 for ( int i = 200; i < 240; ++i ) {
1230 int size = i << 20;
1231 void *test1 = rnd.alloc(size);
1232 CHECK(test1);
1233 for ( int j = 0; j < size; j += (1 << 12) ) {
1234 static_cast<char*>(test1)[j] = 1;
1235 }
1236 free(test1);
1237 }
1238 // There may still be a bit of fragmentation at the beginning, until we
1239 // reach kPageMapBigAllocationThreshold bytes so we check for
1240 // 200 + 240 + margin.
1241 CHECK_LT(MemoryUsage(getpid()), memory_usage + (450 << 20) );
1242 }
1243 #endif
1244
1245 // Check that empty allocation works
1246 fprintf(LOGSTREAM, "Testing empty allocation\n");
1247 {
1248 void* p1 = rnd.alloc(0);
1249 CHECK(p1 != NULL);
1250 void* p2 = rnd.alloc(0);
1251 CHECK(p2 != NULL);
1252 CHECK(p1 != p2);
1253 free(p1);
1254 free(p2);
1255 }
1256
1257 // This code stresses some of the memory allocation via STL.
1258 // It may call operator delete(void*, nothrow_t).
1259 fprintf(LOGSTREAM, "Testing STL use\n");
1260 {
1261 std::vector<int> v;
1262 v.push_back(1);
1263 v.push_back(2);
1264 v.push_back(3);
1265 v.push_back(0);
1266 std::stable_sort(v.begin(), v.end());
1267 }
1268
1269 // Test each of the memory-allocation functions once, just as a sanity-check
1270 fprintf(LOGSTREAM, "Sanity-testing all the memory allocation functions\n");
1271 {
1272 // We use new-hook and delete-hook to verify we actually called the
1273 // tcmalloc version of these routines, and not the libc version.
1274 SetNewHook(); // defined as part of MAKE_HOOK_CALLBACK, above
1275 SetDeleteHook(); // ditto
1276
1277 void* p1 = malloc(10);
1278 CHECK(p1 != NULL); // force use of this variable
1279 VerifyNewHookWasCalled();
1280 // Also test the non-standard tc_malloc_size
1281 size_t actual_p1_size = tc_malloc_size(p1);
1282 CHECK_GE(actual_p1_size, 10);
1283 CHECK_LT(actual_p1_size, 100000); // a reasonable upper-bound, I think
1284 free(p1);
1285 VerifyDeleteHookWasCalled();
1286
1287 p1 = tc_malloc_skip_new_handler(10);
1288 CHECK(p1 != NULL);
1289 VerifyNewHookWasCalled();
1290 free(p1);
1291 VerifyDeleteHookWasCalled();
1292
1293 p1 = calloc(10, 2);
1294 CHECK(p1 != NULL);
1295 VerifyNewHookWasCalled();
1296 // We make sure we realloc to a big size, since some systems (OS
1297 // X) will notice if the realloced size continues to fit into the
1298 // malloc-block and make this a noop if so.
1299 p1 = realloc(p1, 30000);
1300 CHECK(p1 != NULL);
1301 VerifyNewHookWasCalled();
1302 VerifyDeleteHookWasCalled();
1303 cfree(p1); // synonym for free
1304 VerifyDeleteHookWasCalled();
1305
1306 if (kOSSupportsMemalign) {
1307 CHECK_EQ(PosixMemalign(&p1, sizeof(p1), 40), 0);
1308 CHECK(p1 != NULL);
1309 VerifyNewHookWasCalled();
1310 free(p1);
1311 VerifyDeleteHookWasCalled();
1312
1313 p1 = Memalign(sizeof(p1) * 2, 50);
1314 CHECK(p1 != NULL);
1315 VerifyNewHookWasCalled();
1316 free(p1);
1317 VerifyDeleteHookWasCalled();
1318 }
1319
1320 // Windows has _aligned_malloc. Let's test that that's captured too.
1321 #if (defined(_MSC_VER) || defined(__MINGW32__)) && !defined(PERFTOOLS_NO_ALIGNED_MALLOC)
1322 p1 = _aligned_malloc(sizeof(p1) * 2, 64);
1323 CHECK(p1 != NULL);
1324 VerifyNewHookWasCalled();
1325 _aligned_free(p1);
1326 VerifyDeleteHookWasCalled();
1327 #endif
1328
1329 p1 = valloc(60);
1330 CHECK(p1 != NULL);
1331 VerifyNewHookWasCalled();
1332 free(p1);
1333 VerifyDeleteHookWasCalled();
1334
1335 p1 = pvalloc(70);
1336 CHECK(p1 != NULL);
1337 VerifyNewHookWasCalled();
1338 free(p1);
1339 VerifyDeleteHookWasCalled();
1340
1341 char* p2 = new char;
1342 CHECK(p2 != NULL);
1343 VerifyNewHookWasCalled();
1344 delete p2;
1345 VerifyDeleteHookWasCalled();
1346
1347 p2 = new char[100];
1348 CHECK(p2 != NULL);
1349 VerifyNewHookWasCalled();
1350 delete[] p2;
1351 VerifyDeleteHookWasCalled();
1352
1353 p2 = new(std::nothrow) char;
1354 CHECK(p2 != NULL);
1355 VerifyNewHookWasCalled();
1356 delete p2;
1357 VerifyDeleteHookWasCalled();
1358
1359 p2 = new(std::nothrow) char[100];
1360 CHECK(p2 != NULL);
1361 VerifyNewHookWasCalled();
1362 delete[] p2;
1363 VerifyDeleteHookWasCalled();
1364
1365 // Another way of calling operator new
1366 p2 = static_cast<char*>(::operator new(100));
1367 CHECK(p2 != NULL);
1368 VerifyNewHookWasCalled();
1369 ::operator delete(p2);
1370 VerifyDeleteHookWasCalled();
1371
1372 // Try to call nothrow's delete too. Compilers use this.
1373 p2 = static_cast<char*>(::operator new(100, std::nothrow));
1374 CHECK(p2 != NULL);
1375 VerifyNewHookWasCalled();
1376 ::operator delete(p2, std::nothrow);
1377 VerifyDeleteHookWasCalled();
1378
1379 #ifdef ENABLE_SIZED_DELETE
1380 p2 = new char;
1381 CHECK(p2 != NULL);
1382 VerifyNewHookWasCalled();
1383 ::operator delete(p2, sizeof(char));
1384 VerifyDeleteHookWasCalled();
1385
1386 p2 = new char[100];
1387 CHECK(p2 != NULL);
1388 VerifyNewHookWasCalled();
1389 ::operator delete[](p2, sizeof(char) * 100);
1390 VerifyDeleteHookWasCalled();
1391 #endif
1392
1393 #if defined(ENABLE_ALIGNED_NEW_DELETE)
1394
1395 overaligned_type* poveraligned = new overaligned_type;
1396 CHECK(poveraligned != NULL);
1397 CHECK((((size_t)poveraligned) % OVERALIGNMENT) == 0u);
1398 VerifyNewHookWasCalled();
1399 delete poveraligned;
1400 VerifyDeleteHookWasCalled();
1401
1402 poveraligned = new overaligned_type[10];
1403 CHECK(poveraligned != NULL);
1404 CHECK((((size_t)poveraligned) % OVERALIGNMENT) == 0u);
1405 VerifyNewHookWasCalled();
1406 delete[] poveraligned;
1407 VerifyDeleteHookWasCalled();
1408
1409 poveraligned = new(std::nothrow) overaligned_type;
1410 CHECK(poveraligned != NULL);
1411 CHECK((((size_t)poveraligned) % OVERALIGNMENT) == 0u);
1412 VerifyNewHookWasCalled();
1413 delete poveraligned;
1414 VerifyDeleteHookWasCalled();
1415
1416 poveraligned = new(std::nothrow) overaligned_type[10];
1417 CHECK(poveraligned != NULL);
1418 CHECK((((size_t)poveraligned) % OVERALIGNMENT) == 0u);
1419 VerifyNewHookWasCalled();
1420 delete[] poveraligned;
1421 VerifyDeleteHookWasCalled();
1422
1423 // Another way of calling operator new
1424 p2 = static_cast<char*>(::operator new(100, std::align_val_t(OVERALIGNMENT)));
1425 CHECK(p2 != NULL);
1426 CHECK((((size_t)p2) % OVERALIGNMENT) == 0u);
1427 VerifyNewHookWasCalled();
1428 ::operator delete(p2, std::align_val_t(OVERALIGNMENT));
1429 VerifyDeleteHookWasCalled();
1430
1431 p2 = static_cast<char*>(::operator new(100, std::align_val_t(OVERALIGNMENT), std::nothrow));
1432 CHECK(p2 != NULL);
1433 CHECK((((size_t)p2) % OVERALIGNMENT) == 0u);
1434 VerifyNewHookWasCalled();
1435 ::operator delete(p2, std::align_val_t(OVERALIGNMENT), std::nothrow);
1436 VerifyDeleteHookWasCalled();
1437
1438 #ifdef ENABLE_SIZED_DELETE
1439 poveraligned = new overaligned_type;
1440 CHECK(poveraligned != NULL);
1441 CHECK((((size_t)poveraligned) % OVERALIGNMENT) == 0u);
1442 VerifyNewHookWasCalled();
1443 ::operator delete(poveraligned, sizeof(overaligned_type), std::align_val_t(OVERALIGNMENT));
1444 VerifyDeleteHookWasCalled();
1445
1446 poveraligned = new overaligned_type[10];
1447 CHECK(poveraligned != NULL);
1448 CHECK((((size_t)poveraligned) % OVERALIGNMENT) == 0u);
1449 VerifyNewHookWasCalled();
1450 ::operator delete[](poveraligned, sizeof(overaligned_type) * 10, std::align_val_t(OVERALIGNMENT));
1451 VerifyDeleteHookWasCalled();
1452 #endif
1453
1454 #endif // defined(ENABLE_ALIGNED_NEW_DELETE)
1455
1456 // Try strdup(), which the system allocates but we must free. If
1457 // all goes well, libc will use our malloc!
1458 p2 = strdup("in memory of James Golick");
1459 CHECK(p2 != NULL);
1460 VerifyNewHookWasCalled();
1461 free(p2);
1462 VerifyDeleteHookWasCalled();
1463
1464
1465 // Test mmap too: both anonymous mmap and mmap of a file
1466 // Note that for right now we only override mmap on linux
1467 // systems, so those are the only ones for which we check.
1468 SetMmapHook();
1469 SetMremapHook();
1470 SetMunmapHook();
1471 #if defined(HAVE_MMAP) && defined(__linux) && \
1472 (defined(__i386__) || defined(__x86_64__))
1473 int size = 8192*2;
1474 p1 = mmap(NULL, size, PROT_WRITE|PROT_READ, MAP_ANONYMOUS|MAP_PRIVATE,
1475 -1, 0);
1476 CHECK(p1 != NULL);
1477 VerifyMmapHookWasCalled();
1478 p1 = mremap(p1, size, size/2, 0);
1479 CHECK(p1 != NULL);
1480 VerifyMremapHookWasCalled();
1481 size /= 2;
1482 munmap(p1, size);
1483 VerifyMunmapHookWasCalled();
1484
1485 int fd = open("/dev/zero", O_RDONLY);
1486 CHECK_GE(fd, 0); // make sure the open succeeded
1487 p1 = mmap(NULL, 8192, PROT_READ, MAP_SHARED, fd, 0);
1488 CHECK(p1 != NULL);
1489 VerifyMmapHookWasCalled();
1490 munmap(p1, 8192);
1491 VerifyMunmapHookWasCalled();
1492 close(fd);
1493 #else // this is just to quiet the compiler: make sure all fns are called
1494 IncrementCallsToMmapHook(NULL, NULL, 0, 0, 0, 0, 0);
1495 IncrementCallsToMunmapHook(NULL, 0);
1496 IncrementCallsToMremapHook(NULL, NULL, 0, 0, 0, NULL);
1497 VerifyMmapHookWasCalled();
1498 VerifyMremapHookWasCalled();
1499 VerifyMunmapHookWasCalled();
1500 #endif
1501
1502 // Test sbrk
1503 SetSbrkHook();
1504 #if defined(HAVE_SBRK) && defined(__linux) && \
1505 (defined(__i386__) || defined(__x86_64__))
1506 p1 = sbrk(8192);
1507 CHECK(p1 != NULL);
1508 VerifySbrkHookWasCalled();
1509 p1 = sbrk(-8192);
1510 CHECK(p1 != NULL);
1511 VerifySbrkHookWasCalled();
1512 // However, sbrk hook should *not* be called with sbrk(0)
1513 p1 = sbrk(0);
1514 CHECK(p1 != NULL);
1515 CHECK_EQ(g_SbrkHook_calls, 0);
1516 #else // this is just to quiet the compiler: make sure all fns are called
1517 IncrementCallsToSbrkHook(NULL, 0);
1518 VerifySbrkHookWasCalled();
1519 #endif
1520
1521 // Reset the hooks to what they used to be. These are all
1522 // defined as part of MAKE_HOOK_CALLBACK, above.
1523 ResetNewHook();
1524 ResetDeleteHook();
1525 ResetMmapHook();
1526 ResetMremapHook();
1527 ResetMunmapHook();
1528 ResetSbrkHook();
1529 }
1530
1531 // Check that "lots" of memory can be allocated
1532 fprintf(LOGSTREAM, "Testing large allocation\n");
1533 {
1534 const int mb_to_allocate = 100;
1535 void* p = rnd.alloc(mb_to_allocate << 20);
1536 CHECK(p != NULL); // could not allocate
1537 free(p);
1538 }
1539
1540 TestMallocAlignment();
1541
1542 // Check calloc() with various arguments
1543 fprintf(LOGSTREAM, "Testing calloc\n");
1544 TestCalloc(0, 0, true);
1545 TestCalloc(0, 1, true);
1546 TestCalloc(1, 1, true);
1547 TestCalloc(1<<10, 0, true);
1548 TestCalloc(1<<20, 0, true);
1549 TestCalloc(0, 1<<10, true);
1550 TestCalloc(0, 1<<20, true);
1551 TestCalloc(1<<20, 2, true);
1552 TestCalloc(2, 1<<20, true);
1553 TestCalloc(1000, 1000, true);
1554
1555 TestCalloc(kMaxSize, 2, false);
1556 TestCalloc(2, kMaxSize, false);
1557 TestCalloc(kMaxSize, kMaxSize, false);
1558
1559 TestCalloc(kMaxSignedSize, 3, false);
1560 TestCalloc(3, kMaxSignedSize, false);
1561 TestCalloc(kMaxSignedSize, kMaxSignedSize, false);
1562
1563 // Test that realloc doesn't always reallocate and copy memory.
1564 fprintf(LOGSTREAM, "Testing realloc\n");
1565 TestRealloc();
1566
1567 fprintf(LOGSTREAM, "Testing operator new(nothrow).\n");
1568 TestNothrowNew(&::operator new);
1569 fprintf(LOGSTREAM, "Testing operator new[](nothrow).\n");
1570 TestNothrowNew(&::operator new[]);
1571 fprintf(LOGSTREAM, "Testing operator new.\n");
1572 TestNew(&::operator new);
1573 fprintf(LOGSTREAM, "Testing operator new[].\n");
1574 TestNew(&::operator new[]);
1575
1576 // Create threads
1577 fprintf(LOGSTREAM, "Testing threaded allocation/deallocation (%d threads)\n",
1578 FLAGS_numthreads);
1579 threads = new TesterThread*[FLAGS_numthreads];
1580 for (int i = 0; i < FLAGS_numthreads; ++i) {
1581 threads[i] = new TesterThread(i);
1582 }
1583
1584 // This runs all the tests at the same time, with a 1M stack size each
1585 RunManyThreadsWithId(RunThread, FLAGS_numthreads, 1<<20);
1586
1587 for (int i = 0; i < FLAGS_numthreads; ++i) delete threads[i]; // Cleanup
1588
1589 // Do the memory intensive tests after threads are done, since exhausting
1590 // the available address space can make pthread_create to fail.
1591
1592 // Check that huge allocations fail with NULL instead of crashing
1593 fprintf(LOGSTREAM, "Testing huge allocations\n");
1594 TestHugeAllocations(&rnd);
1595
1596 // Check that large allocations fail with NULL instead of crashing
1597 #ifndef DEBUGALLOCATION // debug allocation takes forever for huge allocs
1598 fprintf(LOGSTREAM, "Testing out of memory\n");
1599 for (int s = 0; ; s += (10<<20)) {
1600 void* large_object = rnd.alloc(s);
1601 if (large_object == NULL) break;
1602 free(large_object);
1603 }
1604 #endif
1605
1606 TestHugeThreadCache();
1607 TestRanges();
1608 TestReleaseToSystem();
1609 TestAggressiveDecommit();
1610 TestSetNewMode();
1611 TestErrno();
1612
1613 // GetAllocatedSize under DEBUGALLOCATION returns the size that we asked for.
1614 #ifndef DEBUGALLOCATION
1615 TestNAllocX();
1616 TestNAllocXAlignment();
1617 #endif
1618
1619 return 0;
1620 }
1621
1622 }
1623
1624 using testing::RunAllTests;
1625
main(int argc,char ** argv)1626 int main(int argc, char** argv) {
1627 #ifdef DEBUGALLOCATION // debug allocation takes forever for huge allocs
1628 FLAGS_max_free_queue_size = 0; // return freed blocks to tcmalloc immediately
1629 #endif
1630
1631 RunAllTests(argc, argv);
1632
1633 // Test tc_version()
1634 fprintf(LOGSTREAM, "Testing tc_version()\n");
1635 int major;
1636 int minor;
1637 const char* patch;
1638 char mmp[64];
1639 const char* human_version = tc_version(&major, &minor, &patch);
1640 snprintf(mmp, sizeof(mmp), "%d.%d%s", major, minor, patch);
1641 CHECK(!strcmp(PACKAGE_STRING, human_version));
1642 CHECK(!strcmp(PACKAGE_VERSION, mmp));
1643
1644 fprintf(LOGSTREAM, "PASS\n");
1645 }
1646