1 /* ----------------------------------------------------------------------------
2 Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
3 This is free software; you can redistribute it and/or modify it under the
4 terms of the MIT license. A copy of the license can be found in the file
5 "LICENSE" at the root of this distribution.
6 -----------------------------------------------------------------------------*/
7
8 #include "mimalloc.h"
9 #include "mimalloc-internal.h"
10
11 #include <string.h> // memset
12
13 // ------------------------------------------------------
14 // Aligned Allocation
15 // ------------------------------------------------------
16
mi_heap_malloc_zero_aligned_at(mi_heap_t * const heap,const size_t size,const size_t alignment,const size_t offset,const bool zero)17 static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept {
18 // note: we don't require `size > offset`, we just guarantee that
19 // the address at offset is aligned regardless of the allocated size.
20 mi_assert(alignment > 0);
21 if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
22 if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
23 const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
24
25 // try if there is a small block available with just the right alignment
26 const size_t padsize = size + MI_PADDING_SIZE;
27 if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) {
28 mi_page_t* page = _mi_heap_get_free_small_page(heap,padsize);
29 const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0;
30 if (mi_likely(page->free != NULL && is_aligned))
31 {
32 #if MI_STAT>1
33 mi_heap_stat_increase( heap, malloc, size);
34 #endif
35 void* p = _mi_page_malloc(heap,page,padsize); // TODO: inline _mi_page_malloc
36 mi_assert_internal(p != NULL);
37 mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
38 if (zero) _mi_block_zero_init(page,p,size);
39 return p;
40 }
41 }
42
43 // use regular allocation if it is guaranteed to fit the alignment constraints
44 if (offset==0 && alignment<=padsize && padsize<=MI_MEDIUM_OBJ_SIZE_MAX && (padsize&align_mask)==0) {
45 void* p = _mi_heap_malloc_zero(heap, size, zero);
46 mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
47 return p;
48 }
49
50 // otherwise over-allocate
51 void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero);
52 if (p == NULL) return NULL;
53
54 // .. and align within the allocation
55 uintptr_t adjust = alignment - (((uintptr_t)p + offset) & align_mask);
56 mi_assert_internal(adjust <= alignment);
57 void* aligned_p = (adjust == alignment ? p : (void*)((uintptr_t)p + adjust));
58 if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true);
59 mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
60 mi_assert_internal( p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p),_mi_ptr_page(aligned_p),aligned_p) );
61 return aligned_p;
62 }
63
64
mi_heap_malloc_aligned_at(mi_heap_t * heap,size_t size,size_t alignment,size_t offset)65 mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
66 return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false);
67 }
68
mi_heap_malloc_aligned(mi_heap_t * heap,size_t size,size_t alignment)69 mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
70 return mi_heap_malloc_aligned_at(heap, size, alignment, 0);
71 }
72
mi_heap_zalloc_aligned_at(mi_heap_t * heap,size_t size,size_t alignment,size_t offset)73 mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
74 return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true);
75 }
76
mi_heap_zalloc_aligned(mi_heap_t * heap,size_t size,size_t alignment)77 mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
78 return mi_heap_zalloc_aligned_at(heap, size, alignment, 0);
79 }
80
mi_heap_calloc_aligned_at(mi_heap_t * heap,size_t count,size_t size,size_t alignment,size_t offset)81 mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
82 size_t total;
83 if (mi_count_size_overflow(count, size, &total)) return NULL;
84 return mi_heap_zalloc_aligned_at(heap, total, alignment, offset);
85 }
86
mi_heap_calloc_aligned(mi_heap_t * heap,size_t count,size_t size,size_t alignment)87 mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
88 return mi_heap_calloc_aligned_at(heap,count,size,alignment,0);
89 }
90
mi_malloc_aligned_at(size_t size,size_t alignment,size_t offset)91 mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
92 return mi_heap_malloc_aligned_at(mi_get_default_heap(), size, alignment, offset);
93 }
94
mi_malloc_aligned(size_t size,size_t alignment)95 mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
96 return mi_heap_malloc_aligned(mi_get_default_heap(), size, alignment);
97 }
98
mi_zalloc_aligned_at(size_t size,size_t alignment,size_t offset)99 mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
100 return mi_heap_zalloc_aligned_at(mi_get_default_heap(), size, alignment, offset);
101 }
102
mi_zalloc_aligned(size_t size,size_t alignment)103 mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
104 return mi_heap_zalloc_aligned(mi_get_default_heap(), size, alignment);
105 }
106
mi_calloc_aligned_at(size_t count,size_t size,size_t alignment,size_t offset)107 mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
108 return mi_heap_calloc_aligned_at(mi_get_default_heap(), count, size, alignment, offset);
109 }
110
mi_calloc_aligned(size_t count,size_t size,size_t alignment)111 mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {
112 return mi_heap_calloc_aligned(mi_get_default_heap(), count, size, alignment);
113 }
114
115
mi_heap_realloc_zero_aligned_at(mi_heap_t * heap,void * p,size_t newsize,size_t alignment,size_t offset,bool zero)116 static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
117 mi_assert(alignment > 0);
118 if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero);
119 if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero);
120 size_t size = mi_usable_size(p);
121 if (newsize <= size && newsize >= (size - (size / 2))
122 && (((uintptr_t)p + offset) % alignment) == 0) {
123 return p; // reallocation still fits, is aligned and not more than 50% waste
124 }
125 else {
126 void* newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset);
127 if (newp != NULL) {
128 if (zero && newsize > size) {
129 const mi_page_t* page = _mi_ptr_page(newp);
130 if (page->is_zero) {
131 // already zero initialized
132 mi_assert_expensive(mi_mem_is_zero(newp,newsize));
133 }
134 else {
135 // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
136 size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
137 memset((uint8_t*)newp + start, 0, newsize - start);
138 }
139 }
140 _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
141 mi_free(p); // only free if successful
142 }
143 return newp;
144 }
145 }
146
mi_heap_realloc_zero_aligned(mi_heap_t * heap,void * p,size_t newsize,size_t alignment,bool zero)147 static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {
148 mi_assert(alignment > 0);
149 if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero);
150 size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL)
151 return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero);
152 }
153
mi_heap_realloc_aligned_at(mi_heap_t * heap,void * p,size_t newsize,size_t alignment,size_t offset)154 void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
155 return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false);
156 }
157
mi_heap_realloc_aligned(mi_heap_t * heap,void * p,size_t newsize,size_t alignment)158 void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
159 return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false);
160 }
161
mi_heap_rezalloc_aligned_at(mi_heap_t * heap,void * p,size_t newsize,size_t alignment,size_t offset)162 void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
163 return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true);
164 }
165
mi_heap_rezalloc_aligned(mi_heap_t * heap,void * p,size_t newsize,size_t alignment)166 void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
167 return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true);
168 }
169
mi_heap_recalloc_aligned_at(mi_heap_t * heap,void * p,size_t newcount,size_t size,size_t alignment,size_t offset)170 void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
171 size_t total;
172 if (mi_count_size_overflow(newcount, size, &total)) return NULL;
173 return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset);
174 }
175
mi_heap_recalloc_aligned(mi_heap_t * heap,void * p,size_t newcount,size_t size,size_t alignment)176 void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
177 size_t total;
178 if (mi_count_size_overflow(newcount, size, &total)) return NULL;
179 return mi_heap_rezalloc_aligned(heap, p, total, alignment);
180 }
181
mi_realloc_aligned_at(void * p,size_t newsize,size_t alignment,size_t offset)182 void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
183 return mi_heap_realloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset);
184 }
185
mi_realloc_aligned(void * p,size_t newsize,size_t alignment)186 void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
187 return mi_heap_realloc_aligned(mi_get_default_heap(), p, newsize, alignment);
188 }
189
mi_rezalloc_aligned_at(void * p,size_t newsize,size_t alignment,size_t offset)190 void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
191 return mi_heap_rezalloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset);
192 }
193
mi_rezalloc_aligned(void * p,size_t newsize,size_t alignment)194 void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
195 return mi_heap_rezalloc_aligned(mi_get_default_heap(), p, newsize, alignment);
196 }
197
mi_recalloc_aligned_at(void * p,size_t newcount,size_t size,size_t alignment,size_t offset)198 void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
199 return mi_heap_recalloc_aligned_at(mi_get_default_heap(), p, newcount, size, alignment, offset);
200 }
201
mi_recalloc_aligned(void * p,size_t newcount,size_t size,size_t alignment)202 void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
203 return mi_heap_recalloc_aligned(mi_get_default_heap(), p, newcount, size, alignment);
204 }
205
206