1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
10 
11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
12 #include "llvm/Support/WindowsError.h"
13 
14 #include <algorithm>
15 
16 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
17 #include <fcntl.h>
18 #include <sys/mman.h>
19 #include <unistd.h>
20 #elif defined(_WIN32)
21 #include <windows.h>
22 #endif
23 
24 namespace llvm {
25 namespace orc {
26 
27 MemoryMapper::~MemoryMapper() {}
28 
29 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize)
30     : PageSize(PageSize) {}
31 
32 Expected<std::unique_ptr<InProcessMemoryMapper>>
33 InProcessMemoryMapper::Create() {
34   auto PageSize = sys::Process::getPageSize();
35   if (!PageSize)
36     return PageSize.takeError();
37   return std::make_unique<InProcessMemoryMapper>(*PageSize);
38 }
39 
40 void InProcessMemoryMapper::reserve(size_t NumBytes,
41                                     OnReservedFunction OnReserved) {
42   std::error_code EC;
43   auto MB = sys::Memory::allocateMappedMemory(
44       NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
45 
46   if (EC)
47     return OnReserved(errorCodeToError(EC));
48 
49   {
50     std::lock_guard<std::mutex> Lock(Mutex);
51     Reservations[MB.base()].Size = MB.allocatedSize();
52   }
53 
54   OnReserved(
55       ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize()));
56 }
57 
58 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
59   return Addr.toPtr<char *>();
60 }
61 
62 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
63                                        OnInitializedFunction OnInitialized) {
64   ExecutorAddr MinAddr(~0ULL);
65   ExecutorAddr MaxAddr(0);
66 
67   // FIXME: Release finalize lifetime segments.
68   for (auto &Segment : AI.Segments) {
69     auto Base = AI.MappingBase + Segment.Offset;
70     auto Size = Segment.ContentSize + Segment.ZeroFillSize;
71 
72     if (Base < MinAddr)
73       MinAddr = Base;
74 
75     if (Base + Size > MaxAddr)
76       MaxAddr = Base + Size;
77 
78     std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0,
79                 Segment.ZeroFillSize);
80 
81     if (auto EC = sys::Memory::protectMappedMemory(
82             {Base.toPtr<void *>(), Size},
83             toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) {
84       return OnInitialized(errorCodeToError(EC));
85     }
86     if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
87       sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size);
88   }
89 
90   auto DeinitializeActions = shared::runFinalizeActions(AI.Actions);
91   if (!DeinitializeActions)
92     return OnInitialized(DeinitializeActions.takeError());
93 
94   {
95     std::lock_guard<std::mutex> Lock(Mutex);
96 
97     // This is the maximum range whose permission have been possibly modified
98     Allocations[MinAddr].Size = MaxAddr - MinAddr;
99     Allocations[MinAddr].DeinitializationActions =
100         std::move(*DeinitializeActions);
101     Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr);
102   }
103 
104   OnInitialized(MinAddr);
105 }
106 
107 void InProcessMemoryMapper::deinitialize(
108     ArrayRef<ExecutorAddr> Bases,
109     MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
110   Error AllErr = Error::success();
111 
112   {
113     std::lock_guard<std::mutex> Lock(Mutex);
114 
115     for (auto Base : llvm::reverse(Bases)) {
116 
117       if (Error Err = shared::runDeallocActions(
118               Allocations[Base].DeinitializationActions)) {
119         AllErr = joinErrors(std::move(AllErr), std::move(Err));
120       }
121 
122       // Reset protections to read/write so the area can be reused
123       if (auto EC = sys::Memory::protectMappedMemory(
124               {Base.toPtr<void *>(), Allocations[Base].Size},
125               sys::Memory::ProtectionFlags::MF_READ |
126                   sys::Memory::ProtectionFlags::MF_WRITE)) {
127         AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC));
128       }
129 
130       Allocations.erase(Base);
131     }
132   }
133 
134   OnDeinitialized(std::move(AllErr));
135 }
136 
137 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
138                                     OnReleasedFunction OnReleased) {
139   Error Err = Error::success();
140 
141   for (auto Base : Bases) {
142     std::vector<ExecutorAddr> AllocAddrs;
143     size_t Size;
144     {
145       std::lock_guard<std::mutex> Lock(Mutex);
146       auto &R = Reservations[Base.toPtr<void *>()];
147       Size = R.Size;
148       AllocAddrs.swap(R.Allocations);
149     }
150 
151     // deinitialize sub allocations
152     std::promise<MSVCPError> P;
153     auto F = P.get_future();
154     deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
155     if (Error E = F.get()) {
156       Err = joinErrors(std::move(Err), std::move(E));
157     }
158 
159     // free the memory
160     auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
161 
162     auto EC = sys::Memory::releaseMappedMemory(MB);
163     if (EC) {
164       Err = joinErrors(std::move(Err), errorCodeToError(EC));
165     }
166 
167     std::lock_guard<std::mutex> Lock(Mutex);
168     Reservations.erase(Base.toPtr<void *>());
169   }
170 
171   OnReleased(std::move(Err));
172 }
173 
174 InProcessMemoryMapper::~InProcessMemoryMapper() {
175   std::vector<ExecutorAddr> ReservationAddrs;
176   {
177     std::lock_guard<std::mutex> Lock(Mutex);
178 
179     ReservationAddrs.reserve(Reservations.size());
180     for (const auto &R : Reservations) {
181       ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
182     }
183   }
184 
185   std::promise<MSVCPError> P;
186   auto F = P.get_future();
187   release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
188   cantFail(F.get());
189 }
190 
191 // SharedMemoryMapper
192 
193 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC,
194                                        SymbolAddrs SAs, size_t PageSize)
195     : EPC(EPC), SAs(SAs), PageSize(PageSize) {
196 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
197   llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
198 #endif
199 }
200 
201 Expected<std::unique_ptr<SharedMemoryMapper>>
202 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) {
203 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
204   auto PageSize = sys::Process::getPageSize();
205   if (!PageSize)
206     return PageSize.takeError();
207 
208   return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize);
209 #else
210   return make_error<StringError>(
211       "SharedMemoryMapper is not supported on this platform yet",
212       inconvertibleErrorCode());
213 #endif
214 }
215 
216 void SharedMemoryMapper::reserve(size_t NumBytes,
217                                  OnReservedFunction OnReserved) {
218 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
219 
220   EPC.callSPSWrapperAsync<
221       rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
222       SAs.Reserve,
223       [this, NumBytes, OnReserved = std::move(OnReserved)](
224           Error SerializationErr,
225           Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
226         if (SerializationErr) {
227           cantFail(Result.takeError());
228           return OnReserved(std::move(SerializationErr));
229         }
230 
231         if (!Result)
232           return OnReserved(Result.takeError());
233 
234         ExecutorAddr RemoteAddr;
235         std::string SharedMemoryName;
236         std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result);
237 
238         void *LocalAddr = nullptr;
239 
240 #if defined(LLVM_ON_UNIX)
241 
242         int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700);
243         if (SharedMemoryFile < 0) {
244           return OnReserved(errorCodeToError(
245               std::error_code(errno, std::generic_category())));
246         }
247 
248         // this prevents other processes from accessing it by name
249         shm_unlink(SharedMemoryName.c_str());
250 
251         LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
252                          SharedMemoryFile, 0);
253         if (LocalAddr == MAP_FAILED) {
254           return OnReserved(errorCodeToError(
255               std::error_code(errno, std::generic_category())));
256         }
257 
258         close(SharedMemoryFile);
259 
260 #elif defined(_WIN32)
261 
262         std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
263                                           SharedMemoryName.end());
264         HANDLE SharedMemoryFile = OpenFileMappingW(
265             FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
266         if (!SharedMemoryFile)
267           return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
268 
269         LocalAddr =
270             MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
271         if (!LocalAddr) {
272           CloseHandle(SharedMemoryFile);
273           return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
274         }
275 
276         CloseHandle(SharedMemoryFile);
277 
278 #endif
279         {
280           std::lock_guard<std::mutex> Lock(Mutex);
281           Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}});
282         }
283 
284         OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
285       },
286       SAs.Instance, static_cast<uint64_t>(NumBytes));
287 
288 #else
289   OnReserved(make_error<StringError>(
290       "SharedMemoryMapper is not supported on this platform yet",
291       inconvertibleErrorCode()));
292 #endif
293 }
294 
295 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
296   auto R = Reservations.upper_bound(Addr);
297   assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
298   R--;
299 
300   ExecutorAddrDiff Offset = Addr - R->first;
301 
302   return static_cast<char *>(R->second.LocalAddr) + Offset;
303 }
304 
305 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
306                                     OnInitializedFunction OnInitialized) {
307   auto Reservation = Reservations.upper_bound(AI.MappingBase);
308   assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
309   Reservation--;
310 
311   auto AllocationOffset = AI.MappingBase - Reservation->first;
312 
313   tpctypes::SharedMemoryFinalizeRequest FR;
314 
315   AI.Actions.swap(FR.Actions);
316 
317   FR.Segments.reserve(AI.Segments.size());
318 
319   for (auto Segment : AI.Segments) {
320     char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
321                  AllocationOffset + Segment.Offset;
322     std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize);
323 
324     tpctypes::SharedMemorySegFinalizeRequest SegReq;
325     SegReq.AG = Segment.AG;
326     SegReq.Addr = AI.MappingBase + Segment.Offset;
327     SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
328 
329     FR.Segments.push_back(SegReq);
330   }
331 
332   EPC.callSPSWrapperAsync<
333       rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
334       SAs.Initialize,
335       [OnInitialized = std::move(OnInitialized)](
336           Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
337         if (SerializationErr) {
338           cantFail(Result.takeError());
339           return OnInitialized(std::move(SerializationErr));
340         }
341 
342         OnInitialized(std::move(Result));
343       },
344       SAs.Instance, Reservation->first, std::move(FR));
345 }
346 
347 void SharedMemoryMapper::deinitialize(
348     ArrayRef<ExecutorAddr> Allocations,
349     MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
350   EPC.callSPSWrapperAsync<
351       rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
352       SAs.Deinitialize,
353       [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
354                                                      Error Result) mutable {
355         if (SerializationErr) {
356           cantFail(std::move(Result));
357           return OnDeinitialized(std::move(SerializationErr));
358         }
359 
360         OnDeinitialized(std::move(Result));
361       },
362       SAs.Instance, Allocations);
363 }
364 
365 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
366                                  OnReleasedFunction OnReleased) {
367 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
368   Error Err = Error::success();
369 
370   {
371     std::lock_guard<std::mutex> Lock(Mutex);
372 
373     for (auto Base : Bases) {
374 
375 #if defined(LLVM_ON_UNIX)
376 
377       if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
378         Err = joinErrors(std::move(Err), errorCodeToError(std::error_code(
379                                              errno, std::generic_category())));
380 
381 #elif defined(_WIN32)
382 
383       if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
384         Err = joinErrors(std::move(Err),
385                          errorCodeToError(mapWindowsError(GetLastError())));
386 
387 #endif
388 
389       Reservations.erase(Base);
390     }
391   }
392 
393   EPC.callSPSWrapperAsync<
394       rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
395       SAs.Release,
396       [OnReleased = std::move(OnReleased),
397        Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
398         if (SerializationErr) {
399           cantFail(std::move(Result));
400           return OnReleased(
401               joinErrors(std::move(Err), std::move(SerializationErr)));
402         }
403 
404         return OnReleased(joinErrors(std::move(Err), std::move(Result)));
405       },
406       SAs.Instance, Bases);
407 #else
408   OnReleased(make_error<StringError>(
409       "SharedMemoryMapper is not supported on this platform yet",
410       inconvertibleErrorCode()));
411 #endif
412 }
413 
414 SharedMemoryMapper::~SharedMemoryMapper() {
415   std::lock_guard<std::mutex> Lock(Mutex);
416   for (const auto &R : Reservations) {
417 
418 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
419 
420     munmap(R.second.LocalAddr, R.second.Size);
421 
422 #elif defined(_WIN32)
423 
424     UnmapViewOfFile(R.second.LocalAddr);
425 
426 #else
427 
428     (void)R;
429 
430 #endif
431   }
432 }
433 
434 } // namespace orc
435 
436 } // namespace llvm
437