1 //===-- dfsan_allocator.cpp -------------------------- --------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of DataflowSanitizer.
10 //
11 // DataflowSanitizer allocator.
12 //===----------------------------------------------------------------------===//
13 
14 #include "dfsan_allocator.h"
15 
16 #include "dfsan.h"
17 #include "dfsan_flags.h"
18 #include "dfsan_thread.h"
19 #include "sanitizer_common/sanitizer_allocator.h"
20 #include "sanitizer_common/sanitizer_allocator_checks.h"
21 #include "sanitizer_common/sanitizer_allocator_interface.h"
22 #include "sanitizer_common/sanitizer_allocator_report.h"
23 #include "sanitizer_common/sanitizer_errno.h"
24 
25 namespace __dfsan {
26 
27 struct Metadata {
28   uptr requested_size;
29 };
30 
31 struct DFsanMapUnmapCallback {
32   void OnMap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
33   void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
34 };
35 
36 static const uptr kAllocatorSpace = 0x700000000000ULL;
37 static const uptr kMaxAllowedMallocSize = 8UL << 30;
38 
39 struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
40   static const uptr kSpaceBeg = kAllocatorSpace;
41   static const uptr kSpaceSize = 0x40000000000;  // 4T.
42   static const uptr kMetadataSize = sizeof(Metadata);
43   typedef DefaultSizeClassMap SizeClassMap;
44   typedef DFsanMapUnmapCallback MapUnmapCallback;
45   static const uptr kFlags = 0;
46   using AddressSpaceView = LocalAddressSpaceView;
47 };
48 
49 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
50 
51 typedef CombinedAllocator<PrimaryAllocator> Allocator;
52 typedef Allocator::AllocatorCache AllocatorCache;
53 
54 static Allocator allocator;
55 static AllocatorCache fallback_allocator_cache;
56 static StaticSpinMutex fallback_mutex;
57 
58 static uptr max_malloc_size;
59 
60 void dfsan_allocator_init() {
61   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
62   allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
63   if (common_flags()->max_allocation_size_mb)
64     max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
65                           kMaxAllowedMallocSize);
66   else
67     max_malloc_size = kMaxAllowedMallocSize;
68 }
69 
70 AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
71   CHECK(ms);
72   CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
73   return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
74 }
75 
76 void DFsanThreadLocalMallocStorage::CommitBack() {
77   allocator.SwallowCache(GetAllocatorCache(this));
78 }
79 
80 static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
81   if (size > max_malloc_size) {
82     if (AllocatorMayReturnNull()) {
83       Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
84              size);
85       return nullptr;
86     }
87     BufferedStackTrace stack;
88     ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
89   }
90   DFsanThread *t = GetCurrentThread();
91   void *allocated;
92   if (t) {
93     AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
94     allocated = allocator.Allocate(cache, size, alignment);
95   } else {
96     SpinMutexLock l(&fallback_mutex);
97     AllocatorCache *cache = &fallback_allocator_cache;
98     allocated = allocator.Allocate(cache, size, alignment);
99   }
100   if (UNLIKELY(!allocated)) {
101     SetAllocatorOutOfMemory();
102     if (AllocatorMayReturnNull())
103       return nullptr;
104     BufferedStackTrace stack;
105     ReportOutOfMemory(size, &stack);
106   }
107   Metadata *meta =
108       reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
109   meta->requested_size = size;
110   if (zeroise) {
111     internal_memset(allocated, 0, size);
112     dfsan_set_label(0, allocated, size);
113   } else if (flags().zero_in_malloc) {
114     dfsan_set_label(0, allocated, size);
115   }
116   return allocated;
117 }
118 
119 void dfsan_deallocate(void *p) {
120   CHECK(p);
121   Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
122   uptr size = meta->requested_size;
123   meta->requested_size = 0;
124   if (flags().zero_in_free)
125     dfsan_set_label(0, p, size);
126   DFsanThread *t = GetCurrentThread();
127   if (t) {
128     AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
129     allocator.Deallocate(cache, p);
130   } else {
131     SpinMutexLock l(&fallback_mutex);
132     AllocatorCache *cache = &fallback_allocator_cache;
133     allocator.Deallocate(cache, p);
134   }
135 }
136 
137 void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
138   Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(old_p));
139   uptr old_size = meta->requested_size;
140   uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
141   if (new_size <= actually_allocated_size) {
142     // We are not reallocating here.
143     meta->requested_size = new_size;
144     if (new_size > old_size && flags().zero_in_malloc)
145       dfsan_set_label(0, (char *)old_p + old_size, new_size - old_size);
146     return old_p;
147   }
148   uptr memcpy_size = Min(new_size, old_size);
149   void *new_p = DFsanAllocate(new_size, alignment, false /*zeroise*/);
150   if (new_p) {
151     dfsan_copy_memory(new_p, old_p, memcpy_size);
152     dfsan_deallocate(old_p);
153   }
154   return new_p;
155 }
156 
157 void *DFsanCalloc(uptr nmemb, uptr size) {
158   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
159     if (AllocatorMayReturnNull())
160       return nullptr;
161     BufferedStackTrace stack;
162     ReportCallocOverflow(nmemb, size, &stack);
163   }
164   return DFsanAllocate(nmemb * size, sizeof(u64), true /*zeroise*/);
165 }
166 
167 static uptr AllocationSize(const void *p) {
168   if (!p)
169     return 0;
170   const void *beg = allocator.GetBlockBegin(p);
171   if (beg != p)
172     return 0;
173   Metadata *b = (Metadata *)allocator.GetMetaData(p);
174   return b->requested_size;
175 }
176 
177 void *dfsan_malloc(uptr size) {
178   return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
179 }
180 
181 void *dfsan_calloc(uptr nmemb, uptr size) {
182   return SetErrnoOnNull(DFsanCalloc(nmemb, size));
183 }
184 
185 void *dfsan_realloc(void *ptr, uptr size) {
186   if (!ptr)
187     return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
188   if (size == 0) {
189     dfsan_deallocate(ptr);
190     return nullptr;
191   }
192   return SetErrnoOnNull(DFsanReallocate(ptr, size, sizeof(u64)));
193 }
194 
195 void *dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
196   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
197     errno = errno_ENOMEM;
198     if (AllocatorMayReturnNull())
199       return nullptr;
200     BufferedStackTrace stack;
201     ReportReallocArrayOverflow(nmemb, size, &stack);
202   }
203   return dfsan_realloc(ptr, nmemb * size);
204 }
205 
206 void *dfsan_valloc(uptr size) {
207   return SetErrnoOnNull(
208       DFsanAllocate(size, GetPageSizeCached(), false /*zeroise*/));
209 }
210 
211 void *dfsan_pvalloc(uptr size) {
212   uptr PageSize = GetPageSizeCached();
213   if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
214     errno = errno_ENOMEM;
215     if (AllocatorMayReturnNull())
216       return nullptr;
217     BufferedStackTrace stack;
218     ReportPvallocOverflow(size, &stack);
219   }
220   // pvalloc(0) should allocate one page.
221   size = size ? RoundUpTo(size, PageSize) : PageSize;
222   return SetErrnoOnNull(DFsanAllocate(size, PageSize, false /*zeroise*/));
223 }
224 
225 void *dfsan_aligned_alloc(uptr alignment, uptr size) {
226   if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
227     errno = errno_EINVAL;
228     if (AllocatorMayReturnNull())
229       return nullptr;
230     BufferedStackTrace stack;
231     ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
232   }
233   return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
234 }
235 
236 void *dfsan_memalign(uptr alignment, uptr size) {
237   if (UNLIKELY(!IsPowerOfTwo(alignment))) {
238     errno = errno_EINVAL;
239     if (AllocatorMayReturnNull())
240       return nullptr;
241     BufferedStackTrace stack;
242     ReportInvalidAllocationAlignment(alignment, &stack);
243   }
244   return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
245 }
246 
247 int dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
248   if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
249     if (AllocatorMayReturnNull())
250       return errno_EINVAL;
251     BufferedStackTrace stack;
252     ReportInvalidPosixMemalignAlignment(alignment, &stack);
253   }
254   void *ptr = DFsanAllocate(size, alignment, false /*zeroise*/);
255   if (UNLIKELY(!ptr))
256     // OOM error is already taken care of by DFsanAllocate.
257     return errno_ENOMEM;
258   CHECK(IsAligned((uptr)ptr, alignment));
259   *memptr = ptr;
260   return 0;
261 }
262 
263 }  // namespace __dfsan
264 
265 using namespace __dfsan;
266 
267 uptr __sanitizer_get_current_allocated_bytes() {
268   uptr stats[AllocatorStatCount];
269   allocator.GetStats(stats);
270   return stats[AllocatorStatAllocated];
271 }
272 
273 uptr __sanitizer_get_heap_size() {
274   uptr stats[AllocatorStatCount];
275   allocator.GetStats(stats);
276   return stats[AllocatorStatMapped];
277 }
278 
279 uptr __sanitizer_get_free_bytes() { return 1; }
280 
281 uptr __sanitizer_get_unmapped_bytes() { return 1; }
282 
283 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
284 
285 int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
286 
287 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
288