1 //===-- secondary.h ---------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef SCUDO_SECONDARY_H_
10 #define SCUDO_SECONDARY_H_
11
12 #include "chunk.h"
13 #include "common.h"
14 #include "list.h"
15 #include "memtag.h"
16 #include "mutex.h"
17 #include "options.h"
18 #include "stats.h"
19 #include "string_utils.h"
20
21 namespace scudo {
22
23 // This allocator wraps the platform allocation primitives, and as such is on
24 // the slower side and should preferably be used for larger sized allocations.
25 // Blocks allocated will be preceded and followed by a guard page, and hold
26 // their own header that is not checksummed: the guard pages and the Combined
27 // header should be enough for our purpose.
28
29 namespace LargeBlock {
30
31 struct alignas(Max<uptr>(archSupportsMemoryTagging()
32 ? archMemoryTagGranuleSize()
33 : 1,
34 1U << SCUDO_MIN_ALIGNMENT_LOG)) Header {
35 LargeBlock::Header *Prev;
36 LargeBlock::Header *Next;
37 uptr CommitBase;
38 uptr CommitSize;
39 uptr MapBase;
40 uptr MapSize;
41 [[no_unique_address]] MapPlatformData Data;
42 };
43
44 static_assert(sizeof(Header) % (1U << SCUDO_MIN_ALIGNMENT_LOG) == 0, "");
45 static_assert(!archSupportsMemoryTagging() ||
46 sizeof(Header) % archMemoryTagGranuleSize() == 0,
47 "");
48
getHeaderSize()49 constexpr uptr getHeaderSize() { return sizeof(Header); }
50
addHeaderTag(uptr Ptr)51 template <typename Config> static uptr addHeaderTag(uptr Ptr) {
52 if (allocatorSupportsMemoryTagging<Config>())
53 return addFixedTag(Ptr, 1);
54 return Ptr;
55 }
56
getHeader(uptr Ptr)57 template <typename Config> static Header *getHeader(uptr Ptr) {
58 return reinterpret_cast<Header *>(addHeaderTag<Config>(Ptr)) - 1;
59 }
60
getHeader(const void * Ptr)61 template <typename Config> static Header *getHeader(const void *Ptr) {
62 return getHeader<Config>(reinterpret_cast<uptr>(Ptr));
63 }
64
65 } // namespace LargeBlock
66
unmap(LargeBlock::Header * H)67 static void unmap(LargeBlock::Header *H) {
68 MapPlatformData Data = H->Data;
69 unmap(reinterpret_cast<void *>(H->MapBase), H->MapSize, UNMAP_ALL, &Data);
70 }
71
72 class MapAllocatorNoCache {
73 public:
init(UNUSED s32 ReleaseToOsInterval)74 void init(UNUSED s32 ReleaseToOsInterval) {}
retrieve(UNUSED Options Options,UNUSED uptr Size,UNUSED uptr Alignment,UNUSED LargeBlock::Header ** H,UNUSED bool * Zeroed)75 bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment,
76 UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) {
77 return false;
78 }
store(UNUSED Options Options,LargeBlock::Header * H)79 void store(UNUSED Options Options, LargeBlock::Header *H) { unmap(H); }
canCache(UNUSED uptr Size)80 bool canCache(UNUSED uptr Size) { return false; }
disable()81 void disable() {}
enable()82 void enable() {}
releaseToOS()83 void releaseToOS() {}
disableMemoryTagging()84 void disableMemoryTagging() {}
unmapTestOnly()85 void unmapTestOnly() {}
setOption(Option O,UNUSED sptr Value)86 bool setOption(Option O, UNUSED sptr Value) {
87 if (O == Option::ReleaseInterval || O == Option::MaxCacheEntriesCount ||
88 O == Option::MaxCacheEntrySize)
89 return false;
90 // Not supported by the Secondary Cache, but not an error either.
91 return true;
92 }
93 };
94
95 static const uptr MaxUnusedCachePages = 4U;
96
97 template <typename Config>
mapSecondary(Options Options,uptr CommitBase,uptr CommitSize,uptr AllocPos,uptr Flags,MapPlatformData * Data)98 void mapSecondary(Options Options, uptr CommitBase, uptr CommitSize,
99 uptr AllocPos, uptr Flags, MapPlatformData *Data) {
100 const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached();
101 if (useMemoryTagging<Config>(Options) && CommitSize > MaxUnusedCacheBytes) {
102 const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes);
103 map(reinterpret_cast<void *>(CommitBase), UntaggedPos - CommitBase,
104 "scudo:secondary", MAP_RESIZABLE | MAP_MEMTAG | Flags, Data);
105 map(reinterpret_cast<void *>(UntaggedPos),
106 CommitBase + CommitSize - UntaggedPos, "scudo:secondary",
107 MAP_RESIZABLE | Flags, Data);
108 } else {
109 map(reinterpret_cast<void *>(CommitBase), CommitSize, "scudo:secondary",
110 MAP_RESIZABLE | (useMemoryTagging<Config>(Options) ? MAP_MEMTAG : 0) |
111 Flags,
112 Data);
113 }
114 }
115
116 template <typename Config> class MapAllocatorCache {
117 public:
118 // Ensure the default maximum specified fits the array.
119 static_assert(Config::SecondaryCacheDefaultMaxEntriesCount <=
120 Config::SecondaryCacheEntriesArraySize,
121 "");
122
init(s32 ReleaseToOsInterval)123 void init(s32 ReleaseToOsInterval) {
124 DCHECK_EQ(EntriesCount, 0U);
125 setOption(Option::MaxCacheEntriesCount,
126 static_cast<sptr>(Config::SecondaryCacheDefaultMaxEntriesCount));
127 setOption(Option::MaxCacheEntrySize,
128 static_cast<sptr>(Config::SecondaryCacheDefaultMaxEntrySize));
129 setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
130 }
131
store(Options Options,LargeBlock::Header * H)132 void store(Options Options, LargeBlock::Header *H) {
133 if (!canCache(H->CommitSize))
134 return unmap(H);
135
136 bool EntryCached = false;
137 bool EmptyCache = false;
138 const s32 Interval = atomic_load_relaxed(&ReleaseToOsIntervalMs);
139 const u64 Time = getMonotonicTime();
140 const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
141 CachedBlock Entry;
142 Entry.CommitBase = H->CommitBase;
143 Entry.CommitSize = H->CommitSize;
144 Entry.MapBase = H->MapBase;
145 Entry.MapSize = H->MapSize;
146 Entry.BlockBegin = reinterpret_cast<uptr>(H + 1);
147 Entry.Data = H->Data;
148 Entry.Time = Time;
149 if (useMemoryTagging<Config>(Options)) {
150 if (Interval == 0 && !SCUDO_FUCHSIA) {
151 // Release the memory and make it inaccessible at the same time by
152 // creating a new MAP_NOACCESS mapping on top of the existing mapping.
153 // Fuchsia does not support replacing mappings by creating a new mapping
154 // on top so we just do the two syscalls there.
155 Entry.Time = 0;
156 mapSecondary<Config>(Options, Entry.CommitBase, Entry.CommitSize,
157 Entry.CommitBase, MAP_NOACCESS, &Entry.Data);
158 } else {
159 setMemoryPermission(Entry.CommitBase, Entry.CommitSize, MAP_NOACCESS,
160 &Entry.Data);
161 }
162 } else if (Interval == 0) {
163 releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data);
164 Entry.Time = 0;
165 }
166 do {
167 ScopedLock L(Mutex);
168 if (useMemoryTagging<Config>(Options) && QuarantinePos == -1U) {
169 // If we get here then memory tagging was disabled in between when we
170 // read Options and when we locked Mutex. We can't insert our entry into
171 // the quarantine or the cache because the permissions would be wrong so
172 // just unmap it.
173 break;
174 }
175 if (Config::SecondaryCacheQuarantineSize &&
176 useMemoryTagging<Config>(Options)) {
177 QuarantinePos =
178 (QuarantinePos + 1) % Max(Config::SecondaryCacheQuarantineSize, 1u);
179 if (!Quarantine[QuarantinePos].CommitBase) {
180 Quarantine[QuarantinePos] = Entry;
181 return;
182 }
183 CachedBlock PrevEntry = Quarantine[QuarantinePos];
184 Quarantine[QuarantinePos] = Entry;
185 if (OldestTime == 0)
186 OldestTime = Entry.Time;
187 Entry = PrevEntry;
188 }
189 if (EntriesCount >= MaxCount) {
190 if (IsFullEvents++ == 4U)
191 EmptyCache = true;
192 } else {
193 for (u32 I = 0; I < MaxCount; I++) {
194 if (Entries[I].CommitBase)
195 continue;
196 if (I != 0)
197 Entries[I] = Entries[0];
198 Entries[0] = Entry;
199 EntriesCount++;
200 if (OldestTime == 0)
201 OldestTime = Entry.Time;
202 EntryCached = true;
203 break;
204 }
205 }
206 } while (0);
207 if (EmptyCache)
208 empty();
209 else if (Interval >= 0)
210 releaseOlderThan(Time - static_cast<u64>(Interval) * 1000000);
211 if (!EntryCached)
212 unmap(reinterpret_cast<void *>(Entry.MapBase), Entry.MapSize, UNMAP_ALL,
213 &Entry.Data);
214 }
215
retrieve(Options Options,uptr Size,uptr Alignment,LargeBlock::Header ** H,bool * Zeroed)216 bool retrieve(Options Options, uptr Size, uptr Alignment,
217 LargeBlock::Header **H, bool *Zeroed) {
218 const uptr PageSize = getPageSizeCached();
219 const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
220 bool Found = false;
221 CachedBlock Entry;
222 uptr HeaderPos;
223 {
224 ScopedLock L(Mutex);
225 if (EntriesCount == 0)
226 return false;
227 for (u32 I = 0; I < MaxCount; I++) {
228 const uptr CommitBase = Entries[I].CommitBase;
229 if (!CommitBase)
230 continue;
231 const uptr CommitSize = Entries[I].CommitSize;
232 const uptr AllocPos =
233 roundDownTo(CommitBase + CommitSize - Size, Alignment);
234 HeaderPos =
235 AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize();
236 if (HeaderPos > CommitBase + CommitSize)
237 continue;
238 if (HeaderPos < CommitBase ||
239 AllocPos > CommitBase + PageSize * MaxUnusedCachePages)
240 continue;
241 Found = true;
242 Entry = Entries[I];
243 Entries[I].CommitBase = 0;
244 break;
245 }
246 }
247 if (Found) {
248 *H = reinterpret_cast<LargeBlock::Header *>(
249 LargeBlock::addHeaderTag<Config>(HeaderPos));
250 *Zeroed = Entry.Time == 0;
251 if (useMemoryTagging<Config>(Options))
252 setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0, &Entry.Data);
253 uptr NewBlockBegin = reinterpret_cast<uptr>(*H + 1);
254 if (useMemoryTagging<Config>(Options)) {
255 if (*Zeroed)
256 storeTags(LargeBlock::addHeaderTag<Config>(Entry.CommitBase),
257 NewBlockBegin);
258 else if (Entry.BlockBegin < NewBlockBegin)
259 storeTags(Entry.BlockBegin, NewBlockBegin);
260 else
261 storeTags(untagPointer(NewBlockBegin),
262 untagPointer(Entry.BlockBegin));
263 }
264 (*H)->CommitBase = Entry.CommitBase;
265 (*H)->CommitSize = Entry.CommitSize;
266 (*H)->MapBase = Entry.MapBase;
267 (*H)->MapSize = Entry.MapSize;
268 (*H)->Data = Entry.Data;
269 EntriesCount--;
270 }
271 return Found;
272 }
273
canCache(uptr Size)274 bool canCache(uptr Size) {
275 return atomic_load_relaxed(&MaxEntriesCount) != 0U &&
276 Size <= atomic_load_relaxed(&MaxEntrySize);
277 }
278
setOption(Option O,sptr Value)279 bool setOption(Option O, sptr Value) {
280 if (O == Option::ReleaseInterval) {
281 const s32 Interval =
282 Max(Min(static_cast<s32>(Value),
283 Config::SecondaryCacheMaxReleaseToOsIntervalMs),
284 Config::SecondaryCacheMinReleaseToOsIntervalMs);
285 atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
286 return true;
287 }
288 if (O == Option::MaxCacheEntriesCount) {
289 const u32 MaxCount = static_cast<u32>(Value);
290 if (MaxCount > Config::SecondaryCacheEntriesArraySize)
291 return false;
292 atomic_store_relaxed(&MaxEntriesCount, MaxCount);
293 return true;
294 }
295 if (O == Option::MaxCacheEntrySize) {
296 atomic_store_relaxed(&MaxEntrySize, static_cast<uptr>(Value));
297 return true;
298 }
299 // Not supported by the Secondary Cache, but not an error either.
300 return true;
301 }
302
releaseToOS()303 void releaseToOS() { releaseOlderThan(UINT64_MAX); }
304
disableMemoryTagging()305 void disableMemoryTagging() {
306 ScopedLock L(Mutex);
307 for (u32 I = 0; I != Config::SecondaryCacheQuarantineSize; ++I) {
308 if (Quarantine[I].CommitBase) {
309 unmap(reinterpret_cast<void *>(Quarantine[I].MapBase),
310 Quarantine[I].MapSize, UNMAP_ALL, &Quarantine[I].Data);
311 Quarantine[I].CommitBase = 0;
312 }
313 }
314 const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
315 for (u32 I = 0; I < MaxCount; I++)
316 if (Entries[I].CommitBase)
317 setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0,
318 &Entries[I].Data);
319 QuarantinePos = -1U;
320 }
321
disable()322 void disable() { Mutex.lock(); }
323
enable()324 void enable() { Mutex.unlock(); }
325
unmapTestOnly()326 void unmapTestOnly() { empty(); }
327
328 private:
empty()329 void empty() {
330 struct {
331 void *MapBase;
332 uptr MapSize;
333 MapPlatformData Data;
334 } MapInfo[Config::SecondaryCacheEntriesArraySize];
335 uptr N = 0;
336 {
337 ScopedLock L(Mutex);
338 for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) {
339 if (!Entries[I].CommitBase)
340 continue;
341 MapInfo[N].MapBase = reinterpret_cast<void *>(Entries[I].MapBase);
342 MapInfo[N].MapSize = Entries[I].MapSize;
343 MapInfo[N].Data = Entries[I].Data;
344 Entries[I].CommitBase = 0;
345 N++;
346 }
347 EntriesCount = 0;
348 IsFullEvents = 0;
349 }
350 for (uptr I = 0; I < N; I++)
351 unmap(MapInfo[I].MapBase, MapInfo[I].MapSize, UNMAP_ALL,
352 &MapInfo[I].Data);
353 }
354
355 struct CachedBlock {
356 uptr CommitBase;
357 uptr CommitSize;
358 uptr MapBase;
359 uptr MapSize;
360 uptr BlockBegin;
361 [[no_unique_address]] MapPlatformData Data;
362 u64 Time;
363 };
364
releaseIfOlderThan(CachedBlock & Entry,u64 Time)365 void releaseIfOlderThan(CachedBlock &Entry, u64 Time) {
366 if (!Entry.CommitBase || !Entry.Time)
367 return;
368 if (Entry.Time > Time) {
369 if (OldestTime == 0 || Entry.Time < OldestTime)
370 OldestTime = Entry.Time;
371 return;
372 }
373 releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data);
374 Entry.Time = 0;
375 }
376
releaseOlderThan(u64 Time)377 void releaseOlderThan(u64 Time) {
378 ScopedLock L(Mutex);
379 if (!EntriesCount || OldestTime == 0 || OldestTime > Time)
380 return;
381 OldestTime = 0;
382 for (uptr I = 0; I < Config::SecondaryCacheQuarantineSize; I++)
383 releaseIfOlderThan(Quarantine[I], Time);
384 for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++)
385 releaseIfOlderThan(Entries[I], Time);
386 }
387
388 HybridMutex Mutex;
389 u32 EntriesCount = 0;
390 u32 QuarantinePos = 0;
391 atomic_u32 MaxEntriesCount = {};
392 atomic_uptr MaxEntrySize = {};
393 u64 OldestTime = 0;
394 u32 IsFullEvents = 0;
395 atomic_s32 ReleaseToOsIntervalMs = {};
396
397 CachedBlock Entries[Config::SecondaryCacheEntriesArraySize] = {};
398 CachedBlock Quarantine[Config::SecondaryCacheQuarantineSize] = {};
399 };
400
401 template <typename Config> class MapAllocator {
402 public:
403 void init(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
404 DCHECK_EQ(AllocatedBytes, 0U);
405 DCHECK_EQ(FreedBytes, 0U);
406 Cache.init(ReleaseToOsInterval);
407 Stats.init();
408 if (LIKELY(S))
409 S->link(&Stats);
410 }
411
412 void *allocate(Options Options, uptr Size, uptr AlignmentHint = 0,
413 uptr *BlockEnd = nullptr,
414 FillContentsMode FillContents = NoFill);
415
416 void deallocate(Options Options, void *Ptr);
417
getBlockEnd(void * Ptr)418 static uptr getBlockEnd(void *Ptr) {
419 auto *B = LargeBlock::getHeader<Config>(Ptr);
420 return B->CommitBase + B->CommitSize;
421 }
422
getBlockSize(void * Ptr)423 static uptr getBlockSize(void *Ptr) {
424 return getBlockEnd(Ptr) - reinterpret_cast<uptr>(Ptr);
425 }
426
427 void getStats(ScopedString *Str) const;
428
disable()429 void disable() {
430 Mutex.lock();
431 Cache.disable();
432 }
433
enable()434 void enable() {
435 Cache.enable();
436 Mutex.unlock();
437 }
438
iterateOverBlocks(F Callback)439 template <typename F> void iterateOverBlocks(F Callback) const {
440 for (const auto &H : InUseBlocks) {
441 uptr Ptr = reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize();
442 if (allocatorSupportsMemoryTagging<Config>())
443 Ptr = untagPointer(Ptr);
444 Callback(Ptr);
445 }
446 }
447
canCache(uptr Size)448 uptr canCache(uptr Size) { return Cache.canCache(Size); }
449
setOption(Option O,sptr Value)450 bool setOption(Option O, sptr Value) { return Cache.setOption(O, Value); }
451
releaseToOS()452 void releaseToOS() { Cache.releaseToOS(); }
453
disableMemoryTagging()454 void disableMemoryTagging() { Cache.disableMemoryTagging(); }
455
unmapTestOnly()456 void unmapTestOnly() { Cache.unmapTestOnly(); }
457
458 private:
459 typename Config::SecondaryCache Cache;
460
461 HybridMutex Mutex;
462 DoublyLinkedList<LargeBlock::Header> InUseBlocks;
463 uptr AllocatedBytes = 0;
464 uptr FreedBytes = 0;
465 uptr LargestSize = 0;
466 u32 NumberOfAllocs = 0;
467 u32 NumberOfFrees = 0;
468 LocalStats Stats;
469 };
470
471 // As with the Primary, the size passed to this function includes any desired
472 // alignment, so that the frontend can align the user allocation. The hint
473 // parameter allows us to unmap spurious memory when dealing with larger
474 // (greater than a page) alignments on 32-bit platforms.
475 // Due to the sparsity of address space available on those platforms, requesting
476 // an allocation from the Secondary with a large alignment would end up wasting
477 // VA space (even though we are not committing the whole thing), hence the need
478 // to trim off some of the reserved space.
479 // For allocations requested with an alignment greater than or equal to a page,
480 // the committed memory will amount to something close to Size - AlignmentHint
481 // (pending rounding and headers).
482 template <typename Config>
allocate(Options Options,uptr Size,uptr Alignment,uptr * BlockEndPtr,FillContentsMode FillContents)483 void *MapAllocator<Config>::allocate(Options Options, uptr Size, uptr Alignment,
484 uptr *BlockEndPtr,
485 FillContentsMode FillContents) {
486 if (Options.get(OptionBit::AddLargeAllocationSlack))
487 Size += 1UL << SCUDO_MIN_ALIGNMENT_LOG;
488 Alignment = Max(Alignment, 1UL << SCUDO_MIN_ALIGNMENT_LOG);
489 const uptr PageSize = getPageSizeCached();
490 uptr RoundedSize =
491 roundUpTo(roundUpTo(Size, Alignment) + LargeBlock::getHeaderSize() +
492 Chunk::getHeaderSize(),
493 PageSize);
494 if (Alignment > PageSize)
495 RoundedSize += Alignment - PageSize;
496
497 if (Alignment < PageSize && Cache.canCache(RoundedSize)) {
498 LargeBlock::Header *H;
499 bool Zeroed;
500 if (Cache.retrieve(Options, Size, Alignment, &H, &Zeroed)) {
501 const uptr BlockEnd = H->CommitBase + H->CommitSize;
502 if (BlockEndPtr)
503 *BlockEndPtr = BlockEnd;
504 uptr HInt = reinterpret_cast<uptr>(H);
505 if (allocatorSupportsMemoryTagging<Config>())
506 HInt = untagPointer(HInt);
507 const uptr PtrInt = HInt + LargeBlock::getHeaderSize();
508 void *Ptr = reinterpret_cast<void *>(PtrInt);
509 if (FillContents && !Zeroed)
510 memset(Ptr, FillContents == ZeroFill ? 0 : PatternFillByte,
511 BlockEnd - PtrInt);
512 const uptr BlockSize = BlockEnd - HInt;
513 {
514 ScopedLock L(Mutex);
515 InUseBlocks.push_back(H);
516 AllocatedBytes += BlockSize;
517 NumberOfAllocs++;
518 Stats.add(StatAllocated, BlockSize);
519 Stats.add(StatMapped, H->MapSize);
520 }
521 return Ptr;
522 }
523 }
524
525 MapPlatformData Data = {};
526 const uptr MapSize = RoundedSize + 2 * PageSize;
527 uptr MapBase = reinterpret_cast<uptr>(
528 map(nullptr, MapSize, nullptr, MAP_NOACCESS | MAP_ALLOWNOMEM, &Data));
529 if (UNLIKELY(!MapBase))
530 return nullptr;
531 uptr CommitBase = MapBase + PageSize;
532 uptr MapEnd = MapBase + MapSize;
533
534 // In the unlikely event of alignments larger than a page, adjust the amount
535 // of memory we want to commit, and trim the extra memory.
536 if (UNLIKELY(Alignment >= PageSize)) {
537 // For alignments greater than or equal to a page, the user pointer (eg: the
538 // pointer that is returned by the C or C++ allocation APIs) ends up on a
539 // page boundary , and our headers will live in the preceding page.
540 CommitBase = roundUpTo(MapBase + PageSize + 1, Alignment) - PageSize;
541 const uptr NewMapBase = CommitBase - PageSize;
542 DCHECK_GE(NewMapBase, MapBase);
543 // We only trim the extra memory on 32-bit platforms: 64-bit platforms
544 // are less constrained memory wise, and that saves us two syscalls.
545 if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) {
546 unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data);
547 MapBase = NewMapBase;
548 }
549 const uptr NewMapEnd =
550 CommitBase + PageSize + roundUpTo(Size, PageSize) + PageSize;
551 DCHECK_LE(NewMapEnd, MapEnd);
552 if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) {
553 unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data);
554 MapEnd = NewMapEnd;
555 }
556 }
557
558 const uptr CommitSize = MapEnd - PageSize - CommitBase;
559 const uptr AllocPos = roundDownTo(CommitBase + CommitSize - Size, Alignment);
560 mapSecondary<Config>(Options, CommitBase, CommitSize, AllocPos, 0, &Data);
561 const uptr HeaderPos =
562 AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize();
563 LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(
564 LargeBlock::addHeaderTag<Config>(HeaderPos));
565 if (useMemoryTagging<Config>(Options))
566 storeTags(LargeBlock::addHeaderTag<Config>(CommitBase),
567 reinterpret_cast<uptr>(H + 1));
568 H->MapBase = MapBase;
569 H->MapSize = MapEnd - MapBase;
570 H->CommitBase = CommitBase;
571 H->CommitSize = CommitSize;
572 H->Data = Data;
573 if (BlockEndPtr)
574 *BlockEndPtr = CommitBase + CommitSize;
575 {
576 ScopedLock L(Mutex);
577 InUseBlocks.push_back(H);
578 AllocatedBytes += CommitSize;
579 if (LargestSize < CommitSize)
580 LargestSize = CommitSize;
581 NumberOfAllocs++;
582 Stats.add(StatAllocated, CommitSize);
583 Stats.add(StatMapped, H->MapSize);
584 }
585 return reinterpret_cast<void *>(HeaderPos + LargeBlock::getHeaderSize());
586 }
587
588 template <typename Config>
deallocate(Options Options,void * Ptr)589 void MapAllocator<Config>::deallocate(Options Options, void *Ptr) {
590 LargeBlock::Header *H = LargeBlock::getHeader<Config>(Ptr);
591 const uptr CommitSize = H->CommitSize;
592 {
593 ScopedLock L(Mutex);
594 InUseBlocks.remove(H);
595 FreedBytes += CommitSize;
596 NumberOfFrees++;
597 Stats.sub(StatAllocated, CommitSize);
598 Stats.sub(StatMapped, H->MapSize);
599 }
600 Cache.store(Options, H);
601 }
602
603 template <typename Config>
getStats(ScopedString * Str)604 void MapAllocator<Config>::getStats(ScopedString *Str) const {
605 Str->append(
606 "Stats: MapAllocator: allocated %zu times (%zuK), freed %zu times "
607 "(%zuK), remains %zu (%zuK) max %zuM\n",
608 NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10,
609 NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10,
610 LargestSize >> 20);
611 }
612
613 } // namespace scudo
614
615 #endif // SCUDO_SECONDARY_H_
616