1 //===-- dfsan_allocator.cpp -------------------------- --------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of DataflowSanitizer.
10 //
11 // DataflowSanitizer allocator.
12 //===----------------------------------------------------------------------===//
13
14 #include "dfsan_allocator.h"
15
16 #include "dfsan.h"
17 #include "dfsan_flags.h"
18 #include "dfsan_thread.h"
19 #include "sanitizer_common/sanitizer_allocator.h"
20 #include "sanitizer_common/sanitizer_allocator_checks.h"
21 #include "sanitizer_common/sanitizer_allocator_interface.h"
22 #include "sanitizer_common/sanitizer_allocator_report.h"
23 #include "sanitizer_common/sanitizer_errno.h"
24
25 namespace __dfsan {
26
27 struct Metadata {
28 uptr requested_size;
29 };
30
31 struct DFsanMapUnmapCallback {
OnMap__dfsan::DFsanMapUnmapCallback32 void OnMap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
OnUnmap__dfsan::DFsanMapUnmapCallback33 void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
34 };
35
36 static const uptr kAllocatorSpace = 0x700000000000ULL;
37 static const uptr kMaxAllowedMallocSize = 8UL << 30;
38
39 struct AP64 { // Allocator64 parameters. Deliberately using a short name.
40 static const uptr kSpaceBeg = kAllocatorSpace;
41 static const uptr kSpaceSize = 0x40000000000; // 4T.
42 static const uptr kMetadataSize = sizeof(Metadata);
43 typedef DefaultSizeClassMap SizeClassMap;
44 typedef DFsanMapUnmapCallback MapUnmapCallback;
45 static const uptr kFlags = 0;
46 using AddressSpaceView = LocalAddressSpaceView;
47 };
48
49 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
50
51 typedef CombinedAllocator<PrimaryAllocator> Allocator;
52 typedef Allocator::AllocatorCache AllocatorCache;
53
54 static Allocator allocator;
55 static AllocatorCache fallback_allocator_cache;
56 static StaticSpinMutex fallback_mutex;
57
58 static uptr max_malloc_size;
59
dfsan_allocator_init()60 void dfsan_allocator_init() {
61 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
62 allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
63 if (common_flags()->max_allocation_size_mb)
64 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
65 kMaxAllowedMallocSize);
66 else
67 max_malloc_size = kMaxAllowedMallocSize;
68 }
69
GetAllocatorCache(DFsanThreadLocalMallocStorage * ms)70 AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
71 CHECK(ms);
72 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
73 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
74 }
75
CommitBack()76 void DFsanThreadLocalMallocStorage::CommitBack() {
77 allocator.SwallowCache(GetAllocatorCache(this));
78 }
79
DFsanAllocate(uptr size,uptr alignment,bool zeroise)80 static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
81 if (size > max_malloc_size) {
82 if (AllocatorMayReturnNull()) {
83 Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
84 size);
85 return nullptr;
86 }
87 BufferedStackTrace stack;
88 ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
89 }
90 DFsanThread *t = GetCurrentThread();
91 void *allocated;
92 if (t) {
93 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
94 allocated = allocator.Allocate(cache, size, alignment);
95 } else {
96 SpinMutexLock l(&fallback_mutex);
97 AllocatorCache *cache = &fallback_allocator_cache;
98 allocated = allocator.Allocate(cache, size, alignment);
99 }
100 if (UNLIKELY(!allocated)) {
101 SetAllocatorOutOfMemory();
102 if (AllocatorMayReturnNull())
103 return nullptr;
104 BufferedStackTrace stack;
105 ReportOutOfMemory(size, &stack);
106 }
107 Metadata *meta =
108 reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
109 meta->requested_size = size;
110 if (zeroise) {
111 internal_memset(allocated, 0, size);
112 dfsan_set_label(0, allocated, size);
113 } else if (flags().zero_in_malloc) {
114 dfsan_set_label(0, allocated, size);
115 }
116 return allocated;
117 }
118
dfsan_deallocate(void * p)119 void dfsan_deallocate(void *p) {
120 CHECK(p);
121 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
122 uptr size = meta->requested_size;
123 meta->requested_size = 0;
124 if (flags().zero_in_free)
125 dfsan_set_label(0, p, size);
126 DFsanThread *t = GetCurrentThread();
127 if (t) {
128 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
129 allocator.Deallocate(cache, p);
130 } else {
131 SpinMutexLock l(&fallback_mutex);
132 AllocatorCache *cache = &fallback_allocator_cache;
133 allocator.Deallocate(cache, p);
134 }
135 }
136
DFsanReallocate(void * old_p,uptr new_size,uptr alignment)137 void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
138 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(old_p));
139 uptr old_size = meta->requested_size;
140 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
141 if (new_size <= actually_allocated_size) {
142 // We are not reallocating here.
143 meta->requested_size = new_size;
144 if (new_size > old_size && flags().zero_in_malloc)
145 dfsan_set_label(0, (char *)old_p + old_size, new_size - old_size);
146 return old_p;
147 }
148 uptr memcpy_size = Min(new_size, old_size);
149 void *new_p = DFsanAllocate(new_size, alignment, false /*zeroise*/);
150 if (new_p) {
151 dfsan_copy_memory(new_p, old_p, memcpy_size);
152 dfsan_deallocate(old_p);
153 }
154 return new_p;
155 }
156
DFsanCalloc(uptr nmemb,uptr size)157 void *DFsanCalloc(uptr nmemb, uptr size) {
158 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
159 if (AllocatorMayReturnNull())
160 return nullptr;
161 BufferedStackTrace stack;
162 ReportCallocOverflow(nmemb, size, &stack);
163 }
164 return DFsanAllocate(nmemb * size, sizeof(u64), true /*zeroise*/);
165 }
166
AllocationSize(const void * p)167 static uptr AllocationSize(const void *p) {
168 if (!p)
169 return 0;
170 const void *beg = allocator.GetBlockBegin(p);
171 if (beg != p)
172 return 0;
173 Metadata *b = (Metadata *)allocator.GetMetaData(p);
174 return b->requested_size;
175 }
176
dfsan_malloc(uptr size)177 void *dfsan_malloc(uptr size) {
178 return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
179 }
180
dfsan_calloc(uptr nmemb,uptr size)181 void *dfsan_calloc(uptr nmemb, uptr size) {
182 return SetErrnoOnNull(DFsanCalloc(nmemb, size));
183 }
184
dfsan_realloc(void * ptr,uptr size)185 void *dfsan_realloc(void *ptr, uptr size) {
186 if (!ptr)
187 return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
188 if (size == 0) {
189 dfsan_deallocate(ptr);
190 return nullptr;
191 }
192 return SetErrnoOnNull(DFsanReallocate(ptr, size, sizeof(u64)));
193 }
194
dfsan_reallocarray(void * ptr,uptr nmemb,uptr size)195 void *dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
196 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
197 errno = errno_ENOMEM;
198 if (AllocatorMayReturnNull())
199 return nullptr;
200 BufferedStackTrace stack;
201 ReportReallocArrayOverflow(nmemb, size, &stack);
202 }
203 return dfsan_realloc(ptr, nmemb * size);
204 }
205
dfsan_valloc(uptr size)206 void *dfsan_valloc(uptr size) {
207 return SetErrnoOnNull(
208 DFsanAllocate(size, GetPageSizeCached(), false /*zeroise*/));
209 }
210
dfsan_pvalloc(uptr size)211 void *dfsan_pvalloc(uptr size) {
212 uptr PageSize = GetPageSizeCached();
213 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
214 errno = errno_ENOMEM;
215 if (AllocatorMayReturnNull())
216 return nullptr;
217 BufferedStackTrace stack;
218 ReportPvallocOverflow(size, &stack);
219 }
220 // pvalloc(0) should allocate one page.
221 size = size ? RoundUpTo(size, PageSize) : PageSize;
222 return SetErrnoOnNull(DFsanAllocate(size, PageSize, false /*zeroise*/));
223 }
224
dfsan_aligned_alloc(uptr alignment,uptr size)225 void *dfsan_aligned_alloc(uptr alignment, uptr size) {
226 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
227 errno = errno_EINVAL;
228 if (AllocatorMayReturnNull())
229 return nullptr;
230 BufferedStackTrace stack;
231 ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
232 }
233 return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
234 }
235
dfsan_memalign(uptr alignment,uptr size)236 void *dfsan_memalign(uptr alignment, uptr size) {
237 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
238 errno = errno_EINVAL;
239 if (AllocatorMayReturnNull())
240 return nullptr;
241 BufferedStackTrace stack;
242 ReportInvalidAllocationAlignment(alignment, &stack);
243 }
244 return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
245 }
246
dfsan_posix_memalign(void ** memptr,uptr alignment,uptr size)247 int dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
248 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
249 if (AllocatorMayReturnNull())
250 return errno_EINVAL;
251 BufferedStackTrace stack;
252 ReportInvalidPosixMemalignAlignment(alignment, &stack);
253 }
254 void *ptr = DFsanAllocate(size, alignment, false /*zeroise*/);
255 if (UNLIKELY(!ptr))
256 // OOM error is already taken care of by DFsanAllocate.
257 return errno_ENOMEM;
258 CHECK(IsAligned((uptr)ptr, alignment));
259 *memptr = ptr;
260 return 0;
261 }
262
263 } // namespace __dfsan
264
265 using namespace __dfsan;
266
__sanitizer_get_current_allocated_bytes()267 uptr __sanitizer_get_current_allocated_bytes() {
268 uptr stats[AllocatorStatCount];
269 allocator.GetStats(stats);
270 return stats[AllocatorStatAllocated];
271 }
272
__sanitizer_get_heap_size()273 uptr __sanitizer_get_heap_size() {
274 uptr stats[AllocatorStatCount];
275 allocator.GetStats(stats);
276 return stats[AllocatorStatMapped];
277 }
278
__sanitizer_get_free_bytes()279 uptr __sanitizer_get_free_bytes() { return 1; }
280
__sanitizer_get_unmapped_bytes()281 uptr __sanitizer_get_unmapped_bytes() { return 1; }
282
__sanitizer_get_estimated_allocated_size(uptr size)283 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
284
__sanitizer_get_ownership(const void * p)285 int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
286
__sanitizer_get_allocated_size(const void * p)287 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
288