1*d415bd75Srobert //===- SimpleExecuorMemoryManagare.cpp - Simple executor-side memory mgmt -===//
2*d415bd75Srobert //
3*d415bd75Srobert // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4*d415bd75Srobert // See https://llvm.org/LICENSE.txt for license information.
5*d415bd75Srobert // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6*d415bd75Srobert //
7*d415bd75Srobert //===----------------------------------------------------------------------===//
8*d415bd75Srobert
9*d415bd75Srobert #include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h"
10*d415bd75Srobert
11*d415bd75Srobert #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12*d415bd75Srobert #include "llvm/Support/FormatVariadic.h"
13*d415bd75Srobert
14*d415bd75Srobert #define DEBUG_TYPE "orc"
15*d415bd75Srobert
16*d415bd75Srobert namespace llvm {
17*d415bd75Srobert namespace orc {
18*d415bd75Srobert namespace rt_bootstrap {
19*d415bd75Srobert
~SimpleExecutorMemoryManager()20*d415bd75Srobert SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() {
21*d415bd75Srobert assert(Allocations.empty() && "shutdown not called?");
22*d415bd75Srobert }
23*d415bd75Srobert
allocate(uint64_t Size)24*d415bd75Srobert Expected<ExecutorAddr> SimpleExecutorMemoryManager::allocate(uint64_t Size) {
25*d415bd75Srobert std::error_code EC;
26*d415bd75Srobert auto MB = sys::Memory::allocateMappedMemory(
27*d415bd75Srobert Size, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
28*d415bd75Srobert if (EC)
29*d415bd75Srobert return errorCodeToError(EC);
30*d415bd75Srobert std::lock_guard<std::mutex> Lock(M);
31*d415bd75Srobert assert(!Allocations.count(MB.base()) && "Duplicate allocation addr");
32*d415bd75Srobert Allocations[MB.base()].Size = Size;
33*d415bd75Srobert return ExecutorAddr::fromPtr(MB.base());
34*d415bd75Srobert }
35*d415bd75Srobert
finalize(tpctypes::FinalizeRequest & FR)36*d415bd75Srobert Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) {
37*d415bd75Srobert ExecutorAddr Base(~0ULL);
38*d415bd75Srobert std::vector<shared::WrapperFunctionCall> DeallocationActions;
39*d415bd75Srobert size_t SuccessfulFinalizationActions = 0;
40*d415bd75Srobert
41*d415bd75Srobert if (FR.Segments.empty()) {
42*d415bd75Srobert // NOTE: Finalizing nothing is currently a no-op. Should it be an error?
43*d415bd75Srobert if (FR.Actions.empty())
44*d415bd75Srobert return Error::success();
45*d415bd75Srobert else
46*d415bd75Srobert return make_error<StringError>("Finalization actions attached to empty "
47*d415bd75Srobert "finalization request",
48*d415bd75Srobert inconvertibleErrorCode());
49*d415bd75Srobert }
50*d415bd75Srobert
51*d415bd75Srobert for (auto &Seg : FR.Segments)
52*d415bd75Srobert Base = std::min(Base, Seg.Addr);
53*d415bd75Srobert
54*d415bd75Srobert for (auto &ActPair : FR.Actions)
55*d415bd75Srobert if (ActPair.Dealloc)
56*d415bd75Srobert DeallocationActions.push_back(ActPair.Dealloc);
57*d415bd75Srobert
58*d415bd75Srobert // Get the Allocation for this finalization.
59*d415bd75Srobert size_t AllocSize = 0;
60*d415bd75Srobert {
61*d415bd75Srobert std::lock_guard<std::mutex> Lock(M);
62*d415bd75Srobert auto I = Allocations.find(Base.toPtr<void *>());
63*d415bd75Srobert if (I == Allocations.end())
64*d415bd75Srobert return make_error<StringError>("Attempt to finalize unrecognized "
65*d415bd75Srobert "allocation " +
66*d415bd75Srobert formatv("{0:x}", Base.getValue()),
67*d415bd75Srobert inconvertibleErrorCode());
68*d415bd75Srobert AllocSize = I->second.Size;
69*d415bd75Srobert I->second.DeallocationActions = std::move(DeallocationActions);
70*d415bd75Srobert }
71*d415bd75Srobert ExecutorAddr AllocEnd = Base + ExecutorAddrDiff(AllocSize);
72*d415bd75Srobert
73*d415bd75Srobert // Bail-out function: this will run deallocation actions corresponding to any
74*d415bd75Srobert // completed finalization actions, then deallocate memory.
75*d415bd75Srobert auto BailOut = [&](Error Err) {
76*d415bd75Srobert std::pair<void *, Allocation> AllocToDestroy;
77*d415bd75Srobert
78*d415bd75Srobert // Get allocation to destory.
79*d415bd75Srobert {
80*d415bd75Srobert std::lock_guard<std::mutex> Lock(M);
81*d415bd75Srobert auto I = Allocations.find(Base.toPtr<void *>());
82*d415bd75Srobert
83*d415bd75Srobert // Check for missing allocation (effective a double free).
84*d415bd75Srobert if (I == Allocations.end())
85*d415bd75Srobert return joinErrors(
86*d415bd75Srobert std::move(Err),
87*d415bd75Srobert make_error<StringError>("No allocation entry found "
88*d415bd75Srobert "for " +
89*d415bd75Srobert formatv("{0:x}", Base.getValue()),
90*d415bd75Srobert inconvertibleErrorCode()));
91*d415bd75Srobert AllocToDestroy = std::move(*I);
92*d415bd75Srobert Allocations.erase(I);
93*d415bd75Srobert }
94*d415bd75Srobert
95*d415bd75Srobert // Run deallocation actions for all completed finalization actions.
96*d415bd75Srobert while (SuccessfulFinalizationActions)
97*d415bd75Srobert Err =
98*d415bd75Srobert joinErrors(std::move(Err), FR.Actions[--SuccessfulFinalizationActions]
99*d415bd75Srobert .Dealloc.runWithSPSRetErrorMerged());
100*d415bd75Srobert
101*d415bd75Srobert // Deallocate memory.
102*d415bd75Srobert sys::MemoryBlock MB(AllocToDestroy.first, AllocToDestroy.second.Size);
103*d415bd75Srobert if (auto EC = sys::Memory::releaseMappedMemory(MB))
104*d415bd75Srobert Err = joinErrors(std::move(Err), errorCodeToError(EC));
105*d415bd75Srobert
106*d415bd75Srobert return Err;
107*d415bd75Srobert };
108*d415bd75Srobert
109*d415bd75Srobert // Copy content and apply permissions.
110*d415bd75Srobert for (auto &Seg : FR.Segments) {
111*d415bd75Srobert
112*d415bd75Srobert // Check segment ranges.
113*d415bd75Srobert if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size()))
114*d415bd75Srobert return BailOut(make_error<StringError>(
115*d415bd75Srobert formatv("Segment {0:x} content size ({1:x} bytes) "
116*d415bd75Srobert "exceeds segment size ({2:x} bytes)",
117*d415bd75Srobert Seg.Addr.getValue(), Seg.Content.size(), Seg.Size),
118*d415bd75Srobert inconvertibleErrorCode()));
119*d415bd75Srobert ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size);
120*d415bd75Srobert if (LLVM_UNLIKELY(Seg.Addr < Base || SegEnd > AllocEnd))
121*d415bd75Srobert return BailOut(make_error<StringError>(
122*d415bd75Srobert formatv("Segment {0:x} -- {1:x} crosses boundary of "
123*d415bd75Srobert "allocation {2:x} -- {3:x}",
124*d415bd75Srobert Seg.Addr.getValue(), SegEnd.getValue(), Base.getValue(),
125*d415bd75Srobert AllocEnd.getValue()),
126*d415bd75Srobert inconvertibleErrorCode()));
127*d415bd75Srobert
128*d415bd75Srobert char *Mem = Seg.Addr.toPtr<char *>();
129*d415bd75Srobert if (!Seg.Content.empty())
130*d415bd75Srobert memcpy(Mem, Seg.Content.data(), Seg.Content.size());
131*d415bd75Srobert memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size());
132*d415bd75Srobert assert(Seg.Size <= std::numeric_limits<size_t>::max());
133*d415bd75Srobert if (auto EC = sys::Memory::protectMappedMemory(
134*d415bd75Srobert {Mem, static_cast<size_t>(Seg.Size)},
135*d415bd75Srobert toSysMemoryProtectionFlags(Seg.AG.getMemProt())))
136*d415bd75Srobert return BailOut(errorCodeToError(EC));
137*d415bd75Srobert if ((Seg.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
138*d415bd75Srobert sys::Memory::InvalidateInstructionCache(Mem, Seg.Size);
139*d415bd75Srobert }
140*d415bd75Srobert
141*d415bd75Srobert // Run finalization actions.
142*d415bd75Srobert for (auto &ActPair : FR.Actions) {
143*d415bd75Srobert if (auto Err = ActPair.Finalize.runWithSPSRetErrorMerged())
144*d415bd75Srobert return BailOut(std::move(Err));
145*d415bd75Srobert ++SuccessfulFinalizationActions;
146*d415bd75Srobert }
147*d415bd75Srobert
148*d415bd75Srobert return Error::success();
149*d415bd75Srobert }
150*d415bd75Srobert
deallocate(const std::vector<ExecutorAddr> & Bases)151*d415bd75Srobert Error SimpleExecutorMemoryManager::deallocate(
152*d415bd75Srobert const std::vector<ExecutorAddr> &Bases) {
153*d415bd75Srobert std::vector<std::pair<void *, Allocation>> AllocPairs;
154*d415bd75Srobert AllocPairs.reserve(Bases.size());
155*d415bd75Srobert
156*d415bd75Srobert // Get allocation to destory.
157*d415bd75Srobert Error Err = Error::success();
158*d415bd75Srobert {
159*d415bd75Srobert std::lock_guard<std::mutex> Lock(M);
160*d415bd75Srobert for (auto &Base : Bases) {
161*d415bd75Srobert auto I = Allocations.find(Base.toPtr<void *>());
162*d415bd75Srobert
163*d415bd75Srobert // Check for missing allocation (effective a double free).
164*d415bd75Srobert if (I != Allocations.end()) {
165*d415bd75Srobert AllocPairs.push_back(std::move(*I));
166*d415bd75Srobert Allocations.erase(I);
167*d415bd75Srobert } else
168*d415bd75Srobert Err = joinErrors(
169*d415bd75Srobert std::move(Err),
170*d415bd75Srobert make_error<StringError>("No allocation entry found "
171*d415bd75Srobert "for " +
172*d415bd75Srobert formatv("{0:x}", Base.getValue()),
173*d415bd75Srobert inconvertibleErrorCode()));
174*d415bd75Srobert }
175*d415bd75Srobert }
176*d415bd75Srobert
177*d415bd75Srobert while (!AllocPairs.empty()) {
178*d415bd75Srobert auto &P = AllocPairs.back();
179*d415bd75Srobert Err = joinErrors(std::move(Err), deallocateImpl(P.first, P.second));
180*d415bd75Srobert AllocPairs.pop_back();
181*d415bd75Srobert }
182*d415bd75Srobert
183*d415bd75Srobert return Err;
184*d415bd75Srobert }
185*d415bd75Srobert
shutdown()186*d415bd75Srobert Error SimpleExecutorMemoryManager::shutdown() {
187*d415bd75Srobert
188*d415bd75Srobert AllocationsMap AM;
189*d415bd75Srobert {
190*d415bd75Srobert std::lock_guard<std::mutex> Lock(M);
191*d415bd75Srobert AM = std::move(Allocations);
192*d415bd75Srobert }
193*d415bd75Srobert
194*d415bd75Srobert Error Err = Error::success();
195*d415bd75Srobert for (auto &KV : AM)
196*d415bd75Srobert Err = joinErrors(std::move(Err), deallocateImpl(KV.first, KV.second));
197*d415bd75Srobert return Err;
198*d415bd75Srobert }
199*d415bd75Srobert
addBootstrapSymbols(StringMap<ExecutorAddr> & M)200*d415bd75Srobert void SimpleExecutorMemoryManager::addBootstrapSymbols(
201*d415bd75Srobert StringMap<ExecutorAddr> &M) {
202*d415bd75Srobert M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(this);
203*d415bd75Srobert M[rt::SimpleExecutorMemoryManagerReserveWrapperName] =
204*d415bd75Srobert ExecutorAddr::fromPtr(&reserveWrapper);
205*d415bd75Srobert M[rt::SimpleExecutorMemoryManagerFinalizeWrapperName] =
206*d415bd75Srobert ExecutorAddr::fromPtr(&finalizeWrapper);
207*d415bd75Srobert M[rt::SimpleExecutorMemoryManagerDeallocateWrapperName] =
208*d415bd75Srobert ExecutorAddr::fromPtr(&deallocateWrapper);
209*d415bd75Srobert }
210*d415bd75Srobert
deallocateImpl(void * Base,Allocation & A)211*d415bd75Srobert Error SimpleExecutorMemoryManager::deallocateImpl(void *Base, Allocation &A) {
212*d415bd75Srobert Error Err = Error::success();
213*d415bd75Srobert
214*d415bd75Srobert while (!A.DeallocationActions.empty()) {
215*d415bd75Srobert Err = joinErrors(std::move(Err),
216*d415bd75Srobert A.DeallocationActions.back().runWithSPSRetErrorMerged());
217*d415bd75Srobert A.DeallocationActions.pop_back();
218*d415bd75Srobert }
219*d415bd75Srobert
220*d415bd75Srobert sys::MemoryBlock MB(Base, A.Size);
221*d415bd75Srobert if (auto EC = sys::Memory::releaseMappedMemory(MB))
222*d415bd75Srobert Err = joinErrors(std::move(Err), errorCodeToError(EC));
223*d415bd75Srobert
224*d415bd75Srobert return Err;
225*d415bd75Srobert }
226*d415bd75Srobert
227*d415bd75Srobert llvm::orc::shared::CWrapperFunctionResult
reserveWrapper(const char * ArgData,size_t ArgSize)228*d415bd75Srobert SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData,
229*d415bd75Srobert size_t ArgSize) {
230*d415bd75Srobert return shared::WrapperFunction<
231*d415bd75Srobert rt::SPSSimpleExecutorMemoryManagerReserveSignature>::
232*d415bd75Srobert handle(ArgData, ArgSize,
233*d415bd75Srobert shared::makeMethodWrapperHandler(
234*d415bd75Srobert &SimpleExecutorMemoryManager::allocate))
235*d415bd75Srobert .release();
236*d415bd75Srobert }
237*d415bd75Srobert
238*d415bd75Srobert llvm::orc::shared::CWrapperFunctionResult
finalizeWrapper(const char * ArgData,size_t ArgSize)239*d415bd75Srobert SimpleExecutorMemoryManager::finalizeWrapper(const char *ArgData,
240*d415bd75Srobert size_t ArgSize) {
241*d415bd75Srobert return shared::WrapperFunction<
242*d415bd75Srobert rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>::
243*d415bd75Srobert handle(ArgData, ArgSize,
244*d415bd75Srobert shared::makeMethodWrapperHandler(
245*d415bd75Srobert &SimpleExecutorMemoryManager::finalize))
246*d415bd75Srobert .release();
247*d415bd75Srobert }
248*d415bd75Srobert
249*d415bd75Srobert llvm::orc::shared::CWrapperFunctionResult
deallocateWrapper(const char * ArgData,size_t ArgSize)250*d415bd75Srobert SimpleExecutorMemoryManager::deallocateWrapper(const char *ArgData,
251*d415bd75Srobert size_t ArgSize) {
252*d415bd75Srobert return shared::WrapperFunction<
253*d415bd75Srobert rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>::
254*d415bd75Srobert handle(ArgData, ArgSize,
255*d415bd75Srobert shared::makeMethodWrapperHandler(
256*d415bd75Srobert &SimpleExecutorMemoryManager::deallocate))
257*d415bd75Srobert .release();
258*d415bd75Srobert }
259*d415bd75Srobert
260*d415bd75Srobert } // namespace rt_bootstrap
261*d415bd75Srobert } // end namespace orc
262*d415bd75Srobert } // end namespace llvm
263