1 //===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h"
10 #include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h"
11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12 #include "llvm/Support/Alignment.h"
13 #include "llvm/Support/FormatVariadic.h"
14
15 #define DEBUG_TYPE "orc"
16
17 namespace llvm {
18 namespace orc {
19
20 Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>>
CreateWithDefaultBootstrapSymbols(ExecutorProcessControl & EPC)21 EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols(
22 ExecutorProcessControl &EPC) {
23 SymbolAddrs SAs;
24 if (auto Err = EPC.getBootstrapSymbols(
25 {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName},
26 {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName},
27 {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName},
28 {SAs.Deallocate,
29 rt::SimpleExecutorMemoryManagerDeallocateWrapperName},
30 {SAs.RegisterEHFrame,
31 rt::RegisterEHFrameSectionCustomDirectWrapperName},
32 {SAs.DeregisterEHFrame,
33 rt::DeregisterEHFrameSectionCustomDirectWrapperName}}))
34 return std::move(Err);
35 return std::make_unique<EPCGenericRTDyldMemoryManager>(EPC, std::move(SAs));
36 }
37
EPCGenericRTDyldMemoryManager(ExecutorProcessControl & EPC,SymbolAddrs SAs)38 EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager(
39 ExecutorProcessControl &EPC, SymbolAddrs SAs)
40 : EPC(EPC), SAs(std::move(SAs)) {
41 LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n");
42 }
43
~EPCGenericRTDyldMemoryManager()44 EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() {
45 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n");
46 if (!ErrMsg.empty())
47 errs() << "Destroying with existing errors:\n" << ErrMsg << "\n";
48
49 Error Err = Error::success();
50 if (auto Err2 = EPC.callSPSWrapper<
51 rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>(
52 SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) {
53 // FIXME: Report errors through EPC once that functionality is available.
54 logAllUnhandledErrors(std::move(Err2), errs(), "");
55 return;
56 }
57
58 if (Err)
59 logAllUnhandledErrors(std::move(Err), errs(), "");
60 }
61
allocateCodeSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName)62 uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection(
63 uintptr_t Size, unsigned Alignment, unsigned SectionID,
64 StringRef SectionName) {
65 std::lock_guard<std::mutex> Lock(M);
66 LLVM_DEBUG({
67 dbgs() << "Allocator " << (void *)this << " allocating code section "
68 << SectionName << ": size = " << formatv("{0:x}", Size)
69 << " bytes, alignment = " << Alignment << "\n";
70 });
71 auto &Seg = Unmapped.back().CodeAllocs;
72 Seg.emplace_back(Size, Alignment);
73 return reinterpret_cast<uint8_t *>(
74 alignAddr(Seg.back().Contents.get(), Align(Alignment)));
75 }
76
allocateDataSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName,bool IsReadOnly)77 uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection(
78 uintptr_t Size, unsigned Alignment, unsigned SectionID,
79 StringRef SectionName, bool IsReadOnly) {
80 std::lock_guard<std::mutex> Lock(M);
81 LLVM_DEBUG({
82 dbgs() << "Allocator " << (void *)this << " allocating "
83 << (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName
84 << ": size = " << formatv("{0:x}", Size) << " bytes, alignment "
85 << Alignment << ")\n";
86 });
87
88 auto &Seg =
89 IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs;
90
91 Seg.emplace_back(Size, Alignment);
92 return reinterpret_cast<uint8_t *>(
93 alignAddr(Seg.back().Contents.get(), Align(Alignment)));
94 }
95
reserveAllocationSpace(uintptr_t CodeSize,uint32_t CodeAlign,uintptr_t RODataSize,uint32_t RODataAlign,uintptr_t RWDataSize,uint32_t RWDataAlign)96 void EPCGenericRTDyldMemoryManager::reserveAllocationSpace(
97 uintptr_t CodeSize, uint32_t CodeAlign, uintptr_t RODataSize,
98 uint32_t RODataAlign, uintptr_t RWDataSize, uint32_t RWDataAlign) {
99
100 {
101 std::lock_guard<std::mutex> Lock(M);
102 // If there's already an error then bail out.
103 if (!ErrMsg.empty())
104 return;
105
106 if (!isPowerOf2_32(CodeAlign) || CodeAlign > EPC.getPageSize()) {
107 ErrMsg = "Invalid code alignment in reserveAllocationSpace";
108 return;
109 }
110 if (!isPowerOf2_32(RODataAlign) || RODataAlign > EPC.getPageSize()) {
111 ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace";
112 return;
113 }
114 if (!isPowerOf2_32(RWDataAlign) || RWDataAlign > EPC.getPageSize()) {
115 ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace";
116 return;
117 }
118 }
119
120 uint64_t TotalSize = 0;
121 TotalSize += alignTo(CodeSize, EPC.getPageSize());
122 TotalSize += alignTo(RODataSize, EPC.getPageSize());
123 TotalSize += alignTo(RWDataSize, EPC.getPageSize());
124
125 LLVM_DEBUG({
126 dbgs() << "Allocator " << (void *)this << " reserving "
127 << formatv("{0:x}", TotalSize) << " bytes.\n";
128 });
129
130 Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr()));
131 if (auto Err = EPC.callSPSWrapper<
132 rt::SPSSimpleExecutorMemoryManagerReserveSignature>(
133 SAs.Reserve, TargetAllocAddr, SAs.Instance, TotalSize)) {
134 std::lock_guard<std::mutex> Lock(M);
135 ErrMsg = toString(std::move(Err));
136 return;
137 }
138 if (!TargetAllocAddr) {
139 std::lock_guard<std::mutex> Lock(M);
140 ErrMsg = toString(TargetAllocAddr.takeError());
141 return;
142 }
143
144 std::lock_guard<std::mutex> Lock(M);
145 Unmapped.push_back(AllocGroup());
146 Unmapped.back().RemoteCode = {
147 *TargetAllocAddr, ExecutorAddrDiff(alignTo(CodeSize, EPC.getPageSize()))};
148 Unmapped.back().RemoteROData = {
149 Unmapped.back().RemoteCode.End,
150 ExecutorAddrDiff(alignTo(RODataSize, EPC.getPageSize()))};
151 Unmapped.back().RemoteRWData = {
152 Unmapped.back().RemoteROData.End,
153 ExecutorAddrDiff(alignTo(RWDataSize, EPC.getPageSize()))};
154 }
155
needsToReserveAllocationSpace()156 bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() {
157 return true;
158 }
159
registerEHFrames(uint8_t * Addr,uint64_t LoadAddr,size_t Size)160 void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr,
161 uint64_t LoadAddr,
162 size_t Size) {
163 LLVM_DEBUG({
164 dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame "
165 << formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n";
166 });
167 std::lock_guard<std::mutex> Lock(M);
168 // Bail out early if there's already an error.
169 if (!ErrMsg.empty())
170 return;
171
172 ExecutorAddr LA(LoadAddr);
173 for (auto &Alloc : llvm::reverse(Unfinalized)) {
174 if (Alloc.RemoteCode.contains(LA) || Alloc.RemoteROData.contains(LA) ||
175 Alloc.RemoteRWData.contains(LA)) {
176 Alloc.UnfinalizedEHFrames.push_back({LA, Size});
177 return;
178 }
179 }
180 ErrMsg = "eh-frame does not lie inside unfinalized alloc";
181 }
182
deregisterEHFrames()183 void EPCGenericRTDyldMemoryManager::deregisterEHFrames() {
184 // This is a no-op for us: We've registered a deallocation action for it.
185 }
186
notifyObjectLoaded(RuntimeDyld & Dyld,const object::ObjectFile & Obj)187 void EPCGenericRTDyldMemoryManager::notifyObjectLoaded(
188 RuntimeDyld &Dyld, const object::ObjectFile &Obj) {
189 std::lock_guard<std::mutex> Lock(M);
190 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n");
191 for (auto &ObjAllocs : Unmapped) {
192 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs,
193 ObjAllocs.RemoteCode.Start);
194 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs,
195 ObjAllocs.RemoteROData.Start);
196 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs,
197 ObjAllocs.RemoteRWData.Start);
198 Unfinalized.push_back(std::move(ObjAllocs));
199 }
200 Unmapped.clear();
201 }
202
finalizeMemory(std::string * ErrMsg)203 bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) {
204 LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n");
205
206 // If there's an error then bail out here.
207 std::vector<AllocGroup> Allocs;
208 {
209 std::lock_guard<std::mutex> Lock(M);
210 if (ErrMsg && !this->ErrMsg.empty()) {
211 *ErrMsg = std::move(this->ErrMsg);
212 return true;
213 }
214 std::swap(Allocs, Unfinalized);
215 }
216
217 // Loop over unfinalized objects to make finalization requests.
218 for (auto &ObjAllocs : Allocs) {
219
220 tpctypes::WireProtectionFlags SegProts[3] = {
221 tpctypes::toWireProtectionFlags(
222 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
223 sys::Memory::MF_EXEC)),
224 tpctypes::toWireProtectionFlags(sys::Memory::MF_READ),
225 tpctypes::toWireProtectionFlags(
226 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
227 sys::Memory::MF_WRITE))};
228
229 ExecutorAddrRange *RemoteAddrs[3] = {&ObjAllocs.RemoteCode,
230 &ObjAllocs.RemoteROData,
231 &ObjAllocs.RemoteRWData};
232
233 std::vector<Alloc> *SegSections[3] = {&ObjAllocs.CodeAllocs,
234 &ObjAllocs.RODataAllocs,
235 &ObjAllocs.RWDataAllocs};
236
237 tpctypes::FinalizeRequest FR;
238 std::unique_ptr<char[]> AggregateContents[3];
239
240 for (unsigned I = 0; I != 3; ++I) {
241 FR.Segments.push_back({});
242 auto &Seg = FR.Segments.back();
243 Seg.Prot = SegProts[I];
244 Seg.Addr = RemoteAddrs[I]->Start;
245 for (auto &SecAlloc : *SegSections[I]) {
246 Seg.Size = alignTo(Seg.Size, SecAlloc.Align);
247 Seg.Size += SecAlloc.Size;
248 }
249 AggregateContents[I] = std::make_unique<char[]>(Seg.Size);
250 size_t SecOffset = 0;
251 for (auto &SecAlloc : *SegSections[I]) {
252 SecOffset = alignTo(SecOffset, SecAlloc.Align);
253 memcpy(&AggregateContents[I][SecOffset],
254 reinterpret_cast<const char *>(
255 alignAddr(SecAlloc.Contents.get(), Align(SecAlloc.Align))),
256 SecAlloc.Size);
257 SecOffset += SecAlloc.Size;
258 // FIXME: Can we reset SecAlloc.Content here, now that it's copied into
259 // the aggregated content?
260 }
261 Seg.Content = {AggregateContents[I].get(), SecOffset};
262 }
263
264 for (auto &Frame : ObjAllocs.UnfinalizedEHFrames)
265 FR.Actions.push_back({{SAs.RegisterEHFrame, Frame.Addr, Frame.Size},
266 {SAs.DeregisterEHFrame, Frame.Addr, Frame.Size}});
267
268 // We'll also need to make an extra allocation for the eh-frame wrapper call
269 // arguments.
270 Error FinalizeErr = Error::success();
271 if (auto Err = EPC.callSPSWrapper<
272 rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>(
273 SAs.Finalize, FinalizeErr, SAs.Instance, std::move(FR))) {
274 std::lock_guard<std::mutex> Lock(M);
275 this->ErrMsg = toString(std::move(Err));
276 dbgs() << "Serialization error: " << this->ErrMsg << "\n";
277 if (ErrMsg)
278 *ErrMsg = this->ErrMsg;
279 return true;
280 }
281 if (FinalizeErr) {
282 std::lock_guard<std::mutex> Lock(M);
283 this->ErrMsg = toString(std::move(FinalizeErr));
284 dbgs() << "Finalization error: " << this->ErrMsg << "\n";
285 if (ErrMsg)
286 *ErrMsg = this->ErrMsg;
287 return true;
288 }
289 }
290
291 return false;
292 }
293
mapAllocsToRemoteAddrs(RuntimeDyld & Dyld,std::vector<Alloc> & Allocs,ExecutorAddr NextAddr)294 void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs(
295 RuntimeDyld &Dyld, std::vector<Alloc> &Allocs, ExecutorAddr NextAddr) {
296 for (auto &Alloc : Allocs) {
297 NextAddr.setValue(alignTo(NextAddr.getValue(), Alloc.Align));
298 LLVM_DEBUG({
299 dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> "
300 << format("0x%016" PRIx64, NextAddr.getValue()) << "\n";
301 });
302 Dyld.mapSectionAddress(reinterpret_cast<const void *>(alignAddr(
303 Alloc.Contents.get(), Align(Alloc.Align))),
304 NextAddr.getValue());
305 Alloc.RemoteAddr = NextAddr;
306 // Only advance NextAddr if it was non-null to begin with,
307 // otherwise leave it as null.
308 if (NextAddr)
309 NextAddr += ExecutorAddrDiff(Alloc.Size);
310 }
311 }
312
313 } // end namespace orc
314 } // end namespace llvm
315