10b57cec5SDimitry Andric //===--- JITLinkMemoryManager.cpp - JITLinkMemoryManager implementation ---===//
20b57cec5SDimitry Andric //
3349cc55cSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4349cc55cSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5349cc55cSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric
90b57cec5SDimitry Andric #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
10349cc55cSDimitry Andric #include "llvm/ExecutionEngine/JITLink/JITLink.h"
11349cc55cSDimitry Andric #include "llvm/Support/FormatVariadic.h"
120b57cec5SDimitry Andric #include "llvm/Support/Process.h"
130b57cec5SDimitry Andric
14349cc55cSDimitry Andric #define DEBUG_TYPE "jitlink"
15349cc55cSDimitry Andric
16349cc55cSDimitry Andric using namespace llvm;
17349cc55cSDimitry Andric
180b57cec5SDimitry Andric namespace llvm {
190b57cec5SDimitry Andric namespace jitlink {
200b57cec5SDimitry Andric
210b57cec5SDimitry Andric JITLinkMemoryManager::~JITLinkMemoryManager() = default;
22349cc55cSDimitry Andric JITLinkMemoryManager::InFlightAlloc::~InFlightAlloc() = default;
230b57cec5SDimitry Andric
BasicLayout(LinkGraph & G)24349cc55cSDimitry Andric BasicLayout::BasicLayout(LinkGraph &G) : G(G) {
25349cc55cSDimitry Andric
26349cc55cSDimitry Andric for (auto &Sec : G.sections()) {
2706c3fb27SDimitry Andric // Skip empty sections, and sections with NoAlloc lifetime policies.
2806c3fb27SDimitry Andric if (Sec.blocks().empty() ||
295f757f3fSDimitry Andric Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
30349cc55cSDimitry Andric continue;
31349cc55cSDimitry Andric
325f757f3fSDimitry Andric auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemLifetime()}];
33349cc55cSDimitry Andric for (auto *B : Sec.blocks())
34349cc55cSDimitry Andric if (LLVM_LIKELY(!B->isZeroFill()))
35349cc55cSDimitry Andric Seg.ContentBlocks.push_back(B);
36349cc55cSDimitry Andric else
37349cc55cSDimitry Andric Seg.ZeroFillBlocks.push_back(B);
38349cc55cSDimitry Andric }
39349cc55cSDimitry Andric
40349cc55cSDimitry Andric // Build Segments map.
41349cc55cSDimitry Andric auto CompareBlocks = [](const Block *LHS, const Block *RHS) {
42349cc55cSDimitry Andric // Sort by section, address and size
43349cc55cSDimitry Andric if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())
44349cc55cSDimitry Andric return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();
45349cc55cSDimitry Andric if (LHS->getAddress() != RHS->getAddress())
46349cc55cSDimitry Andric return LHS->getAddress() < RHS->getAddress();
47349cc55cSDimitry Andric return LHS->getSize() < RHS->getSize();
48349cc55cSDimitry Andric };
49349cc55cSDimitry Andric
50349cc55cSDimitry Andric LLVM_DEBUG(dbgs() << "Generated BasicLayout for " << G.getName() << ":\n");
51349cc55cSDimitry Andric for (auto &KV : Segments) {
52349cc55cSDimitry Andric auto &Seg = KV.second;
53349cc55cSDimitry Andric
54349cc55cSDimitry Andric llvm::sort(Seg.ContentBlocks, CompareBlocks);
55349cc55cSDimitry Andric llvm::sort(Seg.ZeroFillBlocks, CompareBlocks);
56349cc55cSDimitry Andric
57349cc55cSDimitry Andric for (auto *B : Seg.ContentBlocks) {
58349cc55cSDimitry Andric Seg.ContentSize = alignToBlock(Seg.ContentSize, *B);
59349cc55cSDimitry Andric Seg.ContentSize += B->getSize();
60349cc55cSDimitry Andric Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));
61349cc55cSDimitry Andric }
62349cc55cSDimitry Andric
63349cc55cSDimitry Andric uint64_t SegEndOffset = Seg.ContentSize;
64349cc55cSDimitry Andric for (auto *B : Seg.ZeroFillBlocks) {
65349cc55cSDimitry Andric SegEndOffset = alignToBlock(SegEndOffset, *B);
66349cc55cSDimitry Andric SegEndOffset += B->getSize();
67349cc55cSDimitry Andric Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));
68349cc55cSDimitry Andric }
69349cc55cSDimitry Andric Seg.ZeroFillSize = SegEndOffset - Seg.ContentSize;
70349cc55cSDimitry Andric
71349cc55cSDimitry Andric LLVM_DEBUG({
72349cc55cSDimitry Andric dbgs() << " Seg " << KV.first
73349cc55cSDimitry Andric << ": content-size=" << formatv("{0:x}", Seg.ContentSize)
74349cc55cSDimitry Andric << ", zero-fill-size=" << formatv("{0:x}", Seg.ZeroFillSize)
75349cc55cSDimitry Andric << ", align=" << formatv("{0:x}", Seg.Alignment.value()) << "\n";
76349cc55cSDimitry Andric });
77349cc55cSDimitry Andric }
78349cc55cSDimitry Andric }
79349cc55cSDimitry Andric
80349cc55cSDimitry Andric Expected<BasicLayout::ContiguousPageBasedLayoutSizes>
getContiguousPageBasedLayoutSizes(uint64_t PageSize)81349cc55cSDimitry Andric BasicLayout::getContiguousPageBasedLayoutSizes(uint64_t PageSize) {
82349cc55cSDimitry Andric ContiguousPageBasedLayoutSizes SegsSizes;
83349cc55cSDimitry Andric
84349cc55cSDimitry Andric for (auto &KV : segments()) {
85349cc55cSDimitry Andric auto &AG = KV.first;
86349cc55cSDimitry Andric auto &Seg = KV.second;
87349cc55cSDimitry Andric
88349cc55cSDimitry Andric if (Seg.Alignment > PageSize)
89349cc55cSDimitry Andric return make_error<StringError>("Segment alignment greater than page size",
90349cc55cSDimitry Andric inconvertibleErrorCode());
91349cc55cSDimitry Andric
92349cc55cSDimitry Andric uint64_t SegSize = alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);
935f757f3fSDimitry Andric if (AG.getMemLifetime() == orc::MemLifetime::Standard)
94349cc55cSDimitry Andric SegsSizes.StandardSegs += SegSize;
95349cc55cSDimitry Andric else
96349cc55cSDimitry Andric SegsSizes.FinalizeSegs += SegSize;
97349cc55cSDimitry Andric }
98349cc55cSDimitry Andric
99349cc55cSDimitry Andric return SegsSizes;
100349cc55cSDimitry Andric }
101349cc55cSDimitry Andric
apply()102349cc55cSDimitry Andric Error BasicLayout::apply() {
103349cc55cSDimitry Andric for (auto &KV : Segments) {
104349cc55cSDimitry Andric auto &Seg = KV.second;
105349cc55cSDimitry Andric
106349cc55cSDimitry Andric assert(!(Seg.ContentBlocks.empty() && Seg.ZeroFillBlocks.empty()) &&
107349cc55cSDimitry Andric "Empty section recorded?");
108349cc55cSDimitry Andric
109349cc55cSDimitry Andric for (auto *B : Seg.ContentBlocks) {
110349cc55cSDimitry Andric // Align addr and working-mem-offset.
111349cc55cSDimitry Andric Seg.Addr = alignToBlock(Seg.Addr, *B);
112349cc55cSDimitry Andric Seg.NextWorkingMemOffset = alignToBlock(Seg.NextWorkingMemOffset, *B);
113349cc55cSDimitry Andric
114349cc55cSDimitry Andric // Update block addr.
115349cc55cSDimitry Andric B->setAddress(Seg.Addr);
116349cc55cSDimitry Andric Seg.Addr += B->getSize();
117349cc55cSDimitry Andric
118349cc55cSDimitry Andric // Copy content to working memory, then update content to point at working
119349cc55cSDimitry Andric // memory.
120349cc55cSDimitry Andric memcpy(Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getContent().data(),
121349cc55cSDimitry Andric B->getSize());
122349cc55cSDimitry Andric B->setMutableContent(
123349cc55cSDimitry Andric {Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getSize()});
124349cc55cSDimitry Andric Seg.NextWorkingMemOffset += B->getSize();
125349cc55cSDimitry Andric }
126349cc55cSDimitry Andric
127349cc55cSDimitry Andric for (auto *B : Seg.ZeroFillBlocks) {
128349cc55cSDimitry Andric // Align addr.
129349cc55cSDimitry Andric Seg.Addr = alignToBlock(Seg.Addr, *B);
130349cc55cSDimitry Andric // Update block addr.
131349cc55cSDimitry Andric B->setAddress(Seg.Addr);
132349cc55cSDimitry Andric Seg.Addr += B->getSize();
133349cc55cSDimitry Andric }
134349cc55cSDimitry Andric
135349cc55cSDimitry Andric Seg.ContentBlocks.clear();
136349cc55cSDimitry Andric Seg.ZeroFillBlocks.clear();
137349cc55cSDimitry Andric }
138349cc55cSDimitry Andric
139349cc55cSDimitry Andric return Error::success();
140349cc55cSDimitry Andric }
141349cc55cSDimitry Andric
graphAllocActions()14204eeddc0SDimitry Andric orc::shared::AllocActions &BasicLayout::graphAllocActions() {
143349cc55cSDimitry Andric return G.allocActions();
144349cc55cSDimitry Andric }
145349cc55cSDimitry Andric
Create(JITLinkMemoryManager & MemMgr,const JITLinkDylib * JD,SegmentMap Segments,OnCreatedFunction OnCreated)146349cc55cSDimitry Andric void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,
147349cc55cSDimitry Andric const JITLinkDylib *JD, SegmentMap Segments,
148349cc55cSDimitry Andric OnCreatedFunction OnCreated) {
149349cc55cSDimitry Andric
15006c3fb27SDimitry Andric static_assert(orc::AllocGroup::NumGroups == 32,
151349cc55cSDimitry Andric "AllocGroup has changed. Section names below must be updated");
152349cc55cSDimitry Andric StringRef AGSectionNames[] = {
153349cc55cSDimitry Andric "__---.standard", "__R--.standard", "__-W-.standard", "__RW-.standard",
154349cc55cSDimitry Andric "__--X.standard", "__R-X.standard", "__-WX.standard", "__RWX.standard",
155349cc55cSDimitry Andric "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize",
156349cc55cSDimitry Andric "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"};
157349cc55cSDimitry Andric
1585f757f3fSDimitry Andric auto G = std::make_unique<LinkGraph>("", Triple(), 0,
1595f757f3fSDimitry Andric llvm::endianness::native, nullptr);
160bdd1243dSDimitry Andric orc::AllocGroupSmallMap<Block *> ContentBlocks;
161349cc55cSDimitry Andric
16204eeddc0SDimitry Andric orc::ExecutorAddr NextAddr(0x100000);
163349cc55cSDimitry Andric for (auto &KV : Segments) {
164349cc55cSDimitry Andric auto &AG = KV.first;
165349cc55cSDimitry Andric auto &Seg = KV.second;
166349cc55cSDimitry Andric
1675f757f3fSDimitry Andric assert(AG.getMemLifetime() != orc::MemLifetime::NoAlloc &&
16806c3fb27SDimitry Andric "NoAlloc segments are not supported by SimpleSegmentAlloc");
16906c3fb27SDimitry Andric
170349cc55cSDimitry Andric auto AGSectionName =
171349cc55cSDimitry Andric AGSectionNames[static_cast<unsigned>(AG.getMemProt()) |
1725f757f3fSDimitry Andric static_cast<bool>(AG.getMemLifetime()) << 3];
173349cc55cSDimitry Andric
174349cc55cSDimitry Andric auto &Sec = G->createSection(AGSectionName, AG.getMemProt());
1755f757f3fSDimitry Andric Sec.setMemLifetime(AG.getMemLifetime());
176349cc55cSDimitry Andric
177349cc55cSDimitry Andric if (Seg.ContentSize != 0) {
17804eeddc0SDimitry Andric NextAddr =
17904eeddc0SDimitry Andric orc::ExecutorAddr(alignTo(NextAddr.getValue(), Seg.ContentAlign));
180349cc55cSDimitry Andric auto &B =
181349cc55cSDimitry Andric G->createMutableContentBlock(Sec, G->allocateBuffer(Seg.ContentSize),
182349cc55cSDimitry Andric NextAddr, Seg.ContentAlign.value(), 0);
183349cc55cSDimitry Andric ContentBlocks[AG] = &B;
184349cc55cSDimitry Andric NextAddr += Seg.ContentSize;
185349cc55cSDimitry Andric }
186349cc55cSDimitry Andric }
187349cc55cSDimitry Andric
188349cc55cSDimitry Andric // GRef declared separately since order-of-argument-eval isn't specified.
189349cc55cSDimitry Andric auto &GRef = *G;
190349cc55cSDimitry Andric MemMgr.allocate(JD, GRef,
191349cc55cSDimitry Andric [G = std::move(G), ContentBlocks = std::move(ContentBlocks),
192349cc55cSDimitry Andric OnCreated = std::move(OnCreated)](
193349cc55cSDimitry Andric JITLinkMemoryManager::AllocResult Alloc) mutable {
194349cc55cSDimitry Andric if (!Alloc)
195349cc55cSDimitry Andric OnCreated(Alloc.takeError());
196349cc55cSDimitry Andric else
197349cc55cSDimitry Andric OnCreated(SimpleSegmentAlloc(std::move(G),
198349cc55cSDimitry Andric std::move(ContentBlocks),
199349cc55cSDimitry Andric std::move(*Alloc)));
200349cc55cSDimitry Andric });
201349cc55cSDimitry Andric }
202349cc55cSDimitry Andric
203349cc55cSDimitry Andric Expected<SimpleSegmentAlloc>
Create(JITLinkMemoryManager & MemMgr,const JITLinkDylib * JD,SegmentMap Segments)204349cc55cSDimitry Andric SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD,
205349cc55cSDimitry Andric SegmentMap Segments) {
206349cc55cSDimitry Andric std::promise<MSVCPExpected<SimpleSegmentAlloc>> AllocP;
207349cc55cSDimitry Andric auto AllocF = AllocP.get_future();
208349cc55cSDimitry Andric Create(MemMgr, JD, std::move(Segments),
209349cc55cSDimitry Andric [&](Expected<SimpleSegmentAlloc> Result) {
210349cc55cSDimitry Andric AllocP.set_value(std::move(Result));
211349cc55cSDimitry Andric });
212349cc55cSDimitry Andric return AllocF.get();
213349cc55cSDimitry Andric }
214349cc55cSDimitry Andric
215349cc55cSDimitry Andric SimpleSegmentAlloc::SimpleSegmentAlloc(SimpleSegmentAlloc &&) = default;
216349cc55cSDimitry Andric SimpleSegmentAlloc &
217349cc55cSDimitry Andric SimpleSegmentAlloc::operator=(SimpleSegmentAlloc &&) = default;
21881ad6265SDimitry Andric SimpleSegmentAlloc::~SimpleSegmentAlloc() = default;
219349cc55cSDimitry Andric
220bdd1243dSDimitry Andric SimpleSegmentAlloc::SegmentInfo
getSegInfo(orc::AllocGroup AG)221bdd1243dSDimitry Andric SimpleSegmentAlloc::getSegInfo(orc::AllocGroup AG) {
222349cc55cSDimitry Andric auto I = ContentBlocks.find(AG);
223349cc55cSDimitry Andric if (I != ContentBlocks.end()) {
224349cc55cSDimitry Andric auto &B = *I->second;
225349cc55cSDimitry Andric return {B.getAddress(), B.getAlreadyMutableContent()};
226349cc55cSDimitry Andric }
227349cc55cSDimitry Andric return {};
228349cc55cSDimitry Andric }
229349cc55cSDimitry Andric
SimpleSegmentAlloc(std::unique_ptr<LinkGraph> G,orc::AllocGroupSmallMap<Block * > ContentBlocks,std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc)230349cc55cSDimitry Andric SimpleSegmentAlloc::SimpleSegmentAlloc(
231bdd1243dSDimitry Andric std::unique_ptr<LinkGraph> G,
232bdd1243dSDimitry Andric orc::AllocGroupSmallMap<Block *> ContentBlocks,
233349cc55cSDimitry Andric std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc)
234349cc55cSDimitry Andric : G(std::move(G)), ContentBlocks(std::move(ContentBlocks)),
235349cc55cSDimitry Andric Alloc(std::move(Alloc)) {}
236349cc55cSDimitry Andric
237349cc55cSDimitry Andric class InProcessMemoryManager::IPInFlightAlloc
238349cc55cSDimitry Andric : public JITLinkMemoryManager::InFlightAlloc {
2390b57cec5SDimitry Andric public:
IPInFlightAlloc(InProcessMemoryManager & MemMgr,LinkGraph & G,BasicLayout BL,sys::MemoryBlock StandardSegments,sys::MemoryBlock FinalizationSegments)240349cc55cSDimitry Andric IPInFlightAlloc(InProcessMemoryManager &MemMgr, LinkGraph &G, BasicLayout BL,
241349cc55cSDimitry Andric sys::MemoryBlock StandardSegments,
242349cc55cSDimitry Andric sys::MemoryBlock FinalizationSegments)
24306c3fb27SDimitry Andric : MemMgr(MemMgr), G(&G), BL(std::move(BL)),
244349cc55cSDimitry Andric StandardSegments(std::move(StandardSegments)),
245349cc55cSDimitry Andric FinalizationSegments(std::move(FinalizationSegments)) {}
246349cc55cSDimitry Andric
~IPInFlightAlloc()24706c3fb27SDimitry Andric ~IPInFlightAlloc() {
24806c3fb27SDimitry Andric assert(!G && "InFlight alloc neither abandoned nor finalized");
24906c3fb27SDimitry Andric }
25006c3fb27SDimitry Andric
finalize(OnFinalizedFunction OnFinalized)251349cc55cSDimitry Andric void finalize(OnFinalizedFunction OnFinalized) override {
252349cc55cSDimitry Andric
253349cc55cSDimitry Andric // Apply memory protections to all segments.
254349cc55cSDimitry Andric if (auto Err = applyProtections()) {
255349cc55cSDimitry Andric OnFinalized(std::move(Err));
256349cc55cSDimitry Andric return;
2570b57cec5SDimitry Andric }
258349cc55cSDimitry Andric
259349cc55cSDimitry Andric // Run finalization actions.
26006c3fb27SDimitry Andric auto DeallocActions = runFinalizeActions(G->allocActions());
26104eeddc0SDimitry Andric if (!DeallocActions) {
26204eeddc0SDimitry Andric OnFinalized(DeallocActions.takeError());
263349cc55cSDimitry Andric return;
2640b57cec5SDimitry Andric }
265349cc55cSDimitry Andric
266349cc55cSDimitry Andric // Release the finalize segments slab.
267349cc55cSDimitry Andric if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments)) {
268349cc55cSDimitry Andric OnFinalized(errorCodeToError(EC));
269349cc55cSDimitry Andric return;
2708bcb0991SDimitry Andric }
271349cc55cSDimitry Andric
27206c3fb27SDimitry Andric #ifndef NDEBUG
27306c3fb27SDimitry Andric // Set 'G' to null to flag that we've been successfully finalized.
27406c3fb27SDimitry Andric // This allows us to assert at destruction time that a call has been made
27506c3fb27SDimitry Andric // to either finalize or abandon.
27606c3fb27SDimitry Andric G = nullptr;
27706c3fb27SDimitry Andric #endif
27806c3fb27SDimitry Andric
279349cc55cSDimitry Andric // Continue with finalized allocation.
280349cc55cSDimitry Andric OnFinalized(MemMgr.createFinalizedAlloc(std::move(StandardSegments),
28104eeddc0SDimitry Andric std::move(*DeallocActions)));
282349cc55cSDimitry Andric }
283349cc55cSDimitry Andric
abandon(OnAbandonedFunction OnAbandoned)284349cc55cSDimitry Andric void abandon(OnAbandonedFunction OnAbandoned) override {
285349cc55cSDimitry Andric Error Err = Error::success();
286349cc55cSDimitry Andric if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments))
287349cc55cSDimitry Andric Err = joinErrors(std::move(Err), errorCodeToError(EC));
288349cc55cSDimitry Andric if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))
289349cc55cSDimitry Andric Err = joinErrors(std::move(Err), errorCodeToError(EC));
29006c3fb27SDimitry Andric
29106c3fb27SDimitry Andric #ifndef NDEBUG
29206c3fb27SDimitry Andric // Set 'G' to null to flag that we've been successfully finalized.
29306c3fb27SDimitry Andric // This allows us to assert at destruction time that a call has been made
29406c3fb27SDimitry Andric // to either finalize or abandon.
29506c3fb27SDimitry Andric G = nullptr;
29606c3fb27SDimitry Andric #endif
29706c3fb27SDimitry Andric
298349cc55cSDimitry Andric OnAbandoned(std::move(Err));
2990b57cec5SDimitry Andric }
3000b57cec5SDimitry Andric
3010b57cec5SDimitry Andric private:
applyProtections()3020b57cec5SDimitry Andric Error applyProtections() {
303349cc55cSDimitry Andric for (auto &KV : BL.segments()) {
304349cc55cSDimitry Andric const auto &AG = KV.first;
305349cc55cSDimitry Andric auto &Seg = KV.second;
306349cc55cSDimitry Andric
307349cc55cSDimitry Andric auto Prot = toSysMemoryProtectionFlags(AG.getMemProt());
308349cc55cSDimitry Andric
309349cc55cSDimitry Andric uint64_t SegSize =
310349cc55cSDimitry Andric alignTo(Seg.ContentSize + Seg.ZeroFillSize, MemMgr.PageSize);
311349cc55cSDimitry Andric sys::MemoryBlock MB(Seg.WorkingMem, SegSize);
312349cc55cSDimitry Andric if (auto EC = sys::Memory::protectMappedMemory(MB, Prot))
3130b57cec5SDimitry Andric return errorCodeToError(EC);
3140b57cec5SDimitry Andric if (Prot & sys::Memory::MF_EXEC)
315349cc55cSDimitry Andric sys::Memory::InvalidateInstructionCache(MB.base(), MB.allocatedSize());
3160b57cec5SDimitry Andric }
3170b57cec5SDimitry Andric return Error::success();
3180b57cec5SDimitry Andric }
3190b57cec5SDimitry Andric
320349cc55cSDimitry Andric InProcessMemoryManager &MemMgr;
32106c3fb27SDimitry Andric LinkGraph *G;
322349cc55cSDimitry Andric BasicLayout BL;
323349cc55cSDimitry Andric sys::MemoryBlock StandardSegments;
324349cc55cSDimitry Andric sys::MemoryBlock FinalizationSegments;
3250b57cec5SDimitry Andric };
3260b57cec5SDimitry Andric
327349cc55cSDimitry Andric Expected<std::unique_ptr<InProcessMemoryManager>>
Create()328349cc55cSDimitry Andric InProcessMemoryManager::Create() {
329349cc55cSDimitry Andric if (auto PageSize = sys::Process::getPageSize())
330349cc55cSDimitry Andric return std::make_unique<InProcessMemoryManager>(*PageSize);
331349cc55cSDimitry Andric else
332349cc55cSDimitry Andric return PageSize.takeError();
333349cc55cSDimitry Andric }
3348bcb0991SDimitry Andric
allocate(const JITLinkDylib * JD,LinkGraph & G,OnAllocatedFunction OnAllocated)335349cc55cSDimitry Andric void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G,
336349cc55cSDimitry Andric OnAllocatedFunction OnAllocated) {
337349cc55cSDimitry Andric
338349cc55cSDimitry Andric // FIXME: Just check this once on startup.
339349cc55cSDimitry Andric if (!isPowerOf2_64((uint64_t)PageSize)) {
340349cc55cSDimitry Andric OnAllocated(make_error<StringError>("Page size is not a power of 2",
341349cc55cSDimitry Andric inconvertibleErrorCode()));
342349cc55cSDimitry Andric return;
343349cc55cSDimitry Andric }
344349cc55cSDimitry Andric
345349cc55cSDimitry Andric BasicLayout BL(G);
346349cc55cSDimitry Andric
347349cc55cSDimitry Andric /// Scan the request and calculate the group and total sizes.
348349cc55cSDimitry Andric /// Check that segment size is no larger than a page.
349349cc55cSDimitry Andric auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(PageSize);
350349cc55cSDimitry Andric if (!SegsSizes) {
351349cc55cSDimitry Andric OnAllocated(SegsSizes.takeError());
352349cc55cSDimitry Andric return;
353349cc55cSDimitry Andric }
354349cc55cSDimitry Andric
355349cc55cSDimitry Andric /// Check that the total size requested (including zero fill) is not larger
356349cc55cSDimitry Andric /// than a size_t.
357349cc55cSDimitry Andric if (SegsSizes->total() > std::numeric_limits<size_t>::max()) {
358349cc55cSDimitry Andric OnAllocated(make_error<JITLinkError>(
359349cc55cSDimitry Andric "Total requested size " + formatv("{0:x}", SegsSizes->total()) +
360349cc55cSDimitry Andric " for graph " + G.getName() + " exceeds address space"));
361349cc55cSDimitry Andric return;
362349cc55cSDimitry Andric }
363349cc55cSDimitry Andric
364349cc55cSDimitry Andric // Allocate one slab for the whole thing (to make sure everything is
365349cc55cSDimitry Andric // in-range), then partition into standard and finalization blocks.
366349cc55cSDimitry Andric //
367349cc55cSDimitry Andric // FIXME: Make two separate allocations in the future to reduce
368349cc55cSDimitry Andric // fragmentation: finalization segments will usually be a single page, and
369349cc55cSDimitry Andric // standard segments are likely to be more than one page. Where multiple
370349cc55cSDimitry Andric // allocations are in-flight at once (likely) the current approach will leave
371349cc55cSDimitry Andric // a lot of single-page holes.
372349cc55cSDimitry Andric sys::MemoryBlock Slab;
373349cc55cSDimitry Andric sys::MemoryBlock StandardSegsMem;
374349cc55cSDimitry Andric sys::MemoryBlock FinalizeSegsMem;
375349cc55cSDimitry Andric {
3760b57cec5SDimitry Andric const sys::Memory::ProtectionFlags ReadWrite =
3770b57cec5SDimitry Andric static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
3780b57cec5SDimitry Andric sys::Memory::MF_WRITE);
3790b57cec5SDimitry Andric
3800b57cec5SDimitry Andric std::error_code EC;
381349cc55cSDimitry Andric Slab = sys::Memory::allocateMappedMemory(SegsSizes->total(), nullptr,
382349cc55cSDimitry Andric ReadWrite, EC);
3830b57cec5SDimitry Andric
384349cc55cSDimitry Andric if (EC) {
385349cc55cSDimitry Andric OnAllocated(errorCodeToError(EC));
386349cc55cSDimitry Andric return;
3870b57cec5SDimitry Andric }
388fe6060f1SDimitry Andric
389349cc55cSDimitry Andric // Zero-fill the whole slab up-front.
390349cc55cSDimitry Andric memset(Slab.base(), 0, Slab.allocatedSize());
391349cc55cSDimitry Andric
392349cc55cSDimitry Andric StandardSegsMem = {Slab.base(),
393349cc55cSDimitry Andric static_cast<size_t>(SegsSizes->StandardSegs)};
394349cc55cSDimitry Andric FinalizeSegsMem = {(void *)((char *)Slab.base() + SegsSizes->StandardSegs),
395349cc55cSDimitry Andric static_cast<size_t>(SegsSizes->FinalizeSegs)};
396349cc55cSDimitry Andric }
397349cc55cSDimitry Andric
39804eeddc0SDimitry Andric auto NextStandardSegAddr = orc::ExecutorAddr::fromPtr(StandardSegsMem.base());
39904eeddc0SDimitry Andric auto NextFinalizeSegAddr = orc::ExecutorAddr::fromPtr(FinalizeSegsMem.base());
400349cc55cSDimitry Andric
401349cc55cSDimitry Andric LLVM_DEBUG({
402349cc55cSDimitry Andric dbgs() << "InProcessMemoryManager allocated:\n";
403349cc55cSDimitry Andric if (SegsSizes->StandardSegs)
404349cc55cSDimitry Andric dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextStandardSegAddr,
405349cc55cSDimitry Andric NextStandardSegAddr + StandardSegsMem.allocatedSize())
406349cc55cSDimitry Andric << " to stardard segs\n";
407349cc55cSDimitry Andric else
408349cc55cSDimitry Andric dbgs() << " no standard segs\n";
409349cc55cSDimitry Andric if (SegsSizes->FinalizeSegs)
410349cc55cSDimitry Andric dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextFinalizeSegAddr,
411349cc55cSDimitry Andric NextFinalizeSegAddr + FinalizeSegsMem.allocatedSize())
412349cc55cSDimitry Andric << " to finalize segs\n";
413349cc55cSDimitry Andric else
414349cc55cSDimitry Andric dbgs() << " no finalize segs\n";
415349cc55cSDimitry Andric });
416349cc55cSDimitry Andric
417349cc55cSDimitry Andric // Build ProtMap, assign addresses.
418349cc55cSDimitry Andric for (auto &KV : BL.segments()) {
419349cc55cSDimitry Andric auto &AG = KV.first;
420349cc55cSDimitry Andric auto &Seg = KV.second;
421349cc55cSDimitry Andric
4225f757f3fSDimitry Andric auto &SegAddr = (AG.getMemLifetime() == orc::MemLifetime::Standard)
423349cc55cSDimitry Andric ? NextStandardSegAddr
424349cc55cSDimitry Andric : NextFinalizeSegAddr;
425349cc55cSDimitry Andric
42604eeddc0SDimitry Andric Seg.WorkingMem = SegAddr.toPtr<char *>();
427349cc55cSDimitry Andric Seg.Addr = SegAddr;
428349cc55cSDimitry Andric
429349cc55cSDimitry Andric SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);
430349cc55cSDimitry Andric }
431349cc55cSDimitry Andric
432349cc55cSDimitry Andric if (auto Err = BL.apply()) {
433349cc55cSDimitry Andric OnAllocated(std::move(Err));
434349cc55cSDimitry Andric return;
435349cc55cSDimitry Andric }
436349cc55cSDimitry Andric
437349cc55cSDimitry Andric OnAllocated(std::make_unique<IPInFlightAlloc>(*this, G, std::move(BL),
438349cc55cSDimitry Andric std::move(StandardSegsMem),
439349cc55cSDimitry Andric std::move(FinalizeSegsMem)));
440349cc55cSDimitry Andric }
441349cc55cSDimitry Andric
deallocate(std::vector<FinalizedAlloc> Allocs,OnDeallocatedFunction OnDeallocated)442349cc55cSDimitry Andric void InProcessMemoryManager::deallocate(std::vector<FinalizedAlloc> Allocs,
443349cc55cSDimitry Andric OnDeallocatedFunction OnDeallocated) {
444349cc55cSDimitry Andric std::vector<sys::MemoryBlock> StandardSegmentsList;
44504eeddc0SDimitry Andric std::vector<std::vector<orc::shared::WrapperFunctionCall>> DeallocActionsList;
446349cc55cSDimitry Andric
447349cc55cSDimitry Andric {
448349cc55cSDimitry Andric std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);
449349cc55cSDimitry Andric for (auto &Alloc : Allocs) {
45004eeddc0SDimitry Andric auto *FA = Alloc.release().toPtr<FinalizedAllocInfo *>();
451349cc55cSDimitry Andric StandardSegmentsList.push_back(std::move(FA->StandardSegments));
452349cc55cSDimitry Andric if (!FA->DeallocActions.empty())
453349cc55cSDimitry Andric DeallocActionsList.push_back(std::move(FA->DeallocActions));
454349cc55cSDimitry Andric FA->~FinalizedAllocInfo();
455349cc55cSDimitry Andric FinalizedAllocInfos.Deallocate(FA);
456349cc55cSDimitry Andric }
457349cc55cSDimitry Andric }
458349cc55cSDimitry Andric
459349cc55cSDimitry Andric Error DeallocErr = Error::success();
460349cc55cSDimitry Andric
461349cc55cSDimitry Andric while (!DeallocActionsList.empty()) {
462349cc55cSDimitry Andric auto &DeallocActions = DeallocActionsList.back();
463349cc55cSDimitry Andric auto &StandardSegments = StandardSegmentsList.back();
464349cc55cSDimitry Andric
465349cc55cSDimitry Andric /// Run any deallocate calls.
466349cc55cSDimitry Andric while (!DeallocActions.empty()) {
46704eeddc0SDimitry Andric if (auto Err = DeallocActions.back().runWithSPSRetErrorMerged())
468349cc55cSDimitry Andric DeallocErr = joinErrors(std::move(DeallocErr), std::move(Err));
469349cc55cSDimitry Andric DeallocActions.pop_back();
470349cc55cSDimitry Andric }
471349cc55cSDimitry Andric
472349cc55cSDimitry Andric /// Release the standard segments slab.
473349cc55cSDimitry Andric if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))
474349cc55cSDimitry Andric DeallocErr = joinErrors(std::move(DeallocErr), errorCodeToError(EC));
475349cc55cSDimitry Andric
476349cc55cSDimitry Andric DeallocActionsList.pop_back();
477349cc55cSDimitry Andric StandardSegmentsList.pop_back();
478349cc55cSDimitry Andric }
479349cc55cSDimitry Andric
480349cc55cSDimitry Andric OnDeallocated(std::move(DeallocErr));
481349cc55cSDimitry Andric }
482349cc55cSDimitry Andric
483349cc55cSDimitry Andric JITLinkMemoryManager::FinalizedAlloc
createFinalizedAlloc(sys::MemoryBlock StandardSegments,std::vector<orc::shared::WrapperFunctionCall> DeallocActions)484349cc55cSDimitry Andric InProcessMemoryManager::createFinalizedAlloc(
485349cc55cSDimitry Andric sys::MemoryBlock StandardSegments,
48604eeddc0SDimitry Andric std::vector<orc::shared::WrapperFunctionCall> DeallocActions) {
487349cc55cSDimitry Andric std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);
488349cc55cSDimitry Andric auto *FA = FinalizedAllocInfos.Allocate<FinalizedAllocInfo>();
489349cc55cSDimitry Andric new (FA) FinalizedAllocInfo(
490349cc55cSDimitry Andric {std::move(StandardSegments), std::move(DeallocActions)});
49104eeddc0SDimitry Andric return FinalizedAlloc(orc::ExecutorAddr::fromPtr(FA));
4920b57cec5SDimitry Andric }
4930b57cec5SDimitry Andric
4940b57cec5SDimitry Andric } // end namespace jitlink
4950b57cec5SDimitry Andric } // end namespace llvm
496