1349cc55cSDimitry Andric //===- elfnix_platform.cpp ------------------------------------------------===//
2349cc55cSDimitry Andric //
3349cc55cSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4349cc55cSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5349cc55cSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6349cc55cSDimitry Andric //
7349cc55cSDimitry Andric //===----------------------------------------------------------------------===//
8349cc55cSDimitry Andric //
9349cc55cSDimitry Andric // This file contains code required to load the rest of the ELF-on-*IX runtime.
10349cc55cSDimitry Andric //
11349cc55cSDimitry Andric //===----------------------------------------------------------------------===//
12349cc55cSDimitry Andric
13349cc55cSDimitry Andric #include "elfnix_platform.h"
14349cc55cSDimitry Andric #include "common.h"
15*06c3fb27SDimitry Andric #include "compiler.h"
16349cc55cSDimitry Andric #include "error.h"
17349cc55cSDimitry Andric #include "wrapper_function_utils.h"
18349cc55cSDimitry Andric
1981ad6265SDimitry Andric #include <algorithm>
20349cc55cSDimitry Andric #include <map>
21349cc55cSDimitry Andric #include <mutex>
22349cc55cSDimitry Andric #include <sstream>
23bdd1243dSDimitry Andric #include <string_view>
24349cc55cSDimitry Andric #include <unordered_map>
25349cc55cSDimitry Andric #include <vector>
26349cc55cSDimitry Andric
27349cc55cSDimitry Andric using namespace __orc_rt;
28349cc55cSDimitry Andric using namespace __orc_rt::elfnix;
29349cc55cSDimitry Andric
30349cc55cSDimitry Andric // Declare function tags for functions in the JIT process.
31349cc55cSDimitry Andric ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
32349cc55cSDimitry Andric ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
33349cc55cSDimitry Andric ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
34349cc55cSDimitry Andric
3581ad6265SDimitry Andric // eh-frame registration functions, made available via aliases
3681ad6265SDimitry Andric // installed by the Platform
37*06c3fb27SDimitry Andric extern "C" void __register_frame(const void *);
38*06c3fb27SDimitry Andric extern "C" void __deregister_frame(const void *);
39*06c3fb27SDimitry Andric
40*06c3fb27SDimitry Andric extern "C" void
41*06c3fb27SDimitry Andric __unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
42*06c3fb27SDimitry Andric extern "C" void
43*06c3fb27SDimitry Andric __unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
44349cc55cSDimitry Andric
45349cc55cSDimitry Andric namespace {
46349cc55cSDimitry Andric
validatePointerSectionExtent(const char * SectionName,const ExecutorAddrRange & SE)47349cc55cSDimitry Andric Error validatePointerSectionExtent(const char *SectionName,
48349cc55cSDimitry Andric const ExecutorAddrRange &SE) {
49bdd1243dSDimitry Andric if (SE.size() % sizeof(uintptr_t)) {
50349cc55cSDimitry Andric std::ostringstream ErrMsg;
51349cc55cSDimitry Andric ErrMsg << std::hex << "Size of " << SectionName << " 0x"
52349cc55cSDimitry Andric << SE.Start.getValue() << " -- 0x" << SE.End.getValue()
53349cc55cSDimitry Andric << " is not a pointer multiple";
54349cc55cSDimitry Andric return make_error<StringError>(ErrMsg.str());
55349cc55cSDimitry Andric }
56349cc55cSDimitry Andric return Error::success();
57349cc55cSDimitry Andric }
58349cc55cSDimitry Andric
runInitArray(const std::vector<ExecutorAddrRange> & InitArraySections,const ELFNixJITDylibInitializers & MOJDIs)59349cc55cSDimitry Andric Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
60349cc55cSDimitry Andric const ELFNixJITDylibInitializers &MOJDIs) {
61349cc55cSDimitry Andric
62349cc55cSDimitry Andric for (const auto &ModInits : InitArraySections) {
63349cc55cSDimitry Andric if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
64349cc55cSDimitry Andric return Err;
65349cc55cSDimitry Andric
66349cc55cSDimitry Andric using InitFunc = void (*)();
67349cc55cSDimitry Andric for (auto *Init : ModInits.toSpan<InitFunc>())
68349cc55cSDimitry Andric (*Init)();
69349cc55cSDimitry Andric }
70349cc55cSDimitry Andric
71349cc55cSDimitry Andric return Error::success();
72349cc55cSDimitry Andric }
73753f127fSDimitry Andric
74349cc55cSDimitry Andric struct TLSInfoEntry {
75349cc55cSDimitry Andric unsigned long Key = 0;
76349cc55cSDimitry Andric unsigned long DataAddress = 0;
77349cc55cSDimitry Andric };
78349cc55cSDimitry Andric
79753f127fSDimitry Andric struct TLSDescriptor {
80753f127fSDimitry Andric void (*Resolver)(void *);
81753f127fSDimitry Andric TLSInfoEntry *InfoEntry;
82753f127fSDimitry Andric };
83753f127fSDimitry Andric
84349cc55cSDimitry Andric class ELFNixPlatformRuntimeState {
85349cc55cSDimitry Andric private:
86349cc55cSDimitry Andric struct AtExitEntry {
87349cc55cSDimitry Andric void (*Func)(void *);
88349cc55cSDimitry Andric void *Arg;
89349cc55cSDimitry Andric };
90349cc55cSDimitry Andric
91349cc55cSDimitry Andric using AtExitsVector = std::vector<AtExitEntry>;
92349cc55cSDimitry Andric
93349cc55cSDimitry Andric struct PerJITDylibState {
94349cc55cSDimitry Andric void *Header = nullptr;
95349cc55cSDimitry Andric size_t RefCount = 0;
96349cc55cSDimitry Andric bool AllowReinitialization = false;
97349cc55cSDimitry Andric AtExitsVector AtExits;
98349cc55cSDimitry Andric };
99349cc55cSDimitry Andric
100349cc55cSDimitry Andric public:
101349cc55cSDimitry Andric static void initialize(void *DSOHandle);
102349cc55cSDimitry Andric static ELFNixPlatformRuntimeState &get();
103349cc55cSDimitry Andric static void destroy();
104349cc55cSDimitry Andric
105*06c3fb27SDimitry Andric ELFNixPlatformRuntimeState(void *DSOHandle);
106349cc55cSDimitry Andric
107349cc55cSDimitry Andric // Delete copy and move constructors.
108349cc55cSDimitry Andric ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
109349cc55cSDimitry Andric ELFNixPlatformRuntimeState &
110349cc55cSDimitry Andric operator=(const ELFNixPlatformRuntimeState &) = delete;
111349cc55cSDimitry Andric ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
112349cc55cSDimitry Andric ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
113349cc55cSDimitry Andric
114349cc55cSDimitry Andric Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
115349cc55cSDimitry Andric Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
116349cc55cSDimitry Andric
117349cc55cSDimitry Andric const char *dlerror();
118bdd1243dSDimitry Andric void *dlopen(std::string_view Name, int Mode);
119349cc55cSDimitry Andric int dlclose(void *DSOHandle);
120bdd1243dSDimitry Andric void *dlsym(void *DSOHandle, std::string_view Symbol);
121349cc55cSDimitry Andric
122349cc55cSDimitry Andric int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
123349cc55cSDimitry Andric void runAtExits(void *DSOHandle);
124349cc55cSDimitry Andric
125349cc55cSDimitry Andric /// Returns the base address of the section containing ThreadData.
126349cc55cSDimitry Andric Expected<std::pair<const char *, size_t>>
127349cc55cSDimitry Andric getThreadDataSectionFor(const char *ThreadData);
128349cc55cSDimitry Andric
getPlatformJDDSOHandle()129349cc55cSDimitry Andric void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
130349cc55cSDimitry Andric
131349cc55cSDimitry Andric private:
132349cc55cSDimitry Andric PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
133bdd1243dSDimitry Andric PerJITDylibState *getJITDylibStateByName(std::string_view Path);
134349cc55cSDimitry Andric PerJITDylibState &
135349cc55cSDimitry Andric getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
136349cc55cSDimitry Andric
137349cc55cSDimitry Andric Error registerThreadDataSection(span<const char> ThreadDataSection);
138349cc55cSDimitry Andric
139349cc55cSDimitry Andric Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
140bdd1243dSDimitry Andric std::string_view Symbol);
141349cc55cSDimitry Andric
142349cc55cSDimitry Andric Expected<ELFNixJITDylibInitializerSequence>
143bdd1243dSDimitry Andric getJITDylibInitializersByName(std::string_view Path);
144bdd1243dSDimitry Andric Expected<void *> dlopenInitialize(std::string_view Path, int Mode);
145349cc55cSDimitry Andric Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
146349cc55cSDimitry Andric
147349cc55cSDimitry Andric static ELFNixPlatformRuntimeState *MOPS;
148349cc55cSDimitry Andric
149349cc55cSDimitry Andric void *PlatformJDDSOHandle;
150349cc55cSDimitry Andric
151*06c3fb27SDimitry Andric // Frame registration functions:
152*06c3fb27SDimitry Andric void (*registerEHFrameSection)(const void *) = nullptr;
153*06c3fb27SDimitry Andric void (*deregisterEHFrameSection)(const void *) = nullptr;
154*06c3fb27SDimitry Andric
155349cc55cSDimitry Andric // FIXME: Move to thread-state.
156349cc55cSDimitry Andric std::string DLFcnError;
157349cc55cSDimitry Andric
158349cc55cSDimitry Andric std::recursive_mutex JDStatesMutex;
159349cc55cSDimitry Andric std::unordered_map<void *, PerJITDylibState> JDStates;
160349cc55cSDimitry Andric std::unordered_map<std::string, void *> JDNameToHeader;
161349cc55cSDimitry Andric
162349cc55cSDimitry Andric std::mutex ThreadDataSectionsMutex;
163349cc55cSDimitry Andric std::map<const char *, size_t> ThreadDataSections;
164349cc55cSDimitry Andric };
165349cc55cSDimitry Andric
166349cc55cSDimitry Andric ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
167349cc55cSDimitry Andric
initialize(void * DSOHandle)168349cc55cSDimitry Andric void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
169349cc55cSDimitry Andric assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
170349cc55cSDimitry Andric MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
171349cc55cSDimitry Andric }
172349cc55cSDimitry Andric
get()173349cc55cSDimitry Andric ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
174349cc55cSDimitry Andric assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
175349cc55cSDimitry Andric return *MOPS;
176349cc55cSDimitry Andric }
177349cc55cSDimitry Andric
destroy()178349cc55cSDimitry Andric void ELFNixPlatformRuntimeState::destroy() {
179349cc55cSDimitry Andric assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
180349cc55cSDimitry Andric delete MOPS;
181349cc55cSDimitry Andric }
182349cc55cSDimitry Andric
ELFNixPlatformRuntimeState(void * DSOHandle)183*06c3fb27SDimitry Andric ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle)
184*06c3fb27SDimitry Andric : PlatformJDDSOHandle(DSOHandle) {
185*06c3fb27SDimitry Andric if (__unw_add_dynamic_eh_frame_section &&
186*06c3fb27SDimitry Andric __unw_remove_dynamic_eh_frame_section) {
187*06c3fb27SDimitry Andric registerEHFrameSection = __unw_add_dynamic_eh_frame_section;
188*06c3fb27SDimitry Andric deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section;
189*06c3fb27SDimitry Andric } else {
190*06c3fb27SDimitry Andric registerEHFrameSection = __register_frame;
191*06c3fb27SDimitry Andric deregisterEHFrameSection = __deregister_frame;
192*06c3fb27SDimitry Andric }
193*06c3fb27SDimitry Andric }
194*06c3fb27SDimitry Andric
registerObjectSections(ELFNixPerObjectSectionsToRegister POSR)195349cc55cSDimitry Andric Error ELFNixPlatformRuntimeState::registerObjectSections(
196349cc55cSDimitry Andric ELFNixPerObjectSectionsToRegister POSR) {
197349cc55cSDimitry Andric if (POSR.EHFrameSection.Start)
198*06c3fb27SDimitry Andric registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
199349cc55cSDimitry Andric
200349cc55cSDimitry Andric if (POSR.ThreadDataSection.Start) {
201349cc55cSDimitry Andric if (auto Err = registerThreadDataSection(
202349cc55cSDimitry Andric POSR.ThreadDataSection.toSpan<const char>()))
203349cc55cSDimitry Andric return Err;
204349cc55cSDimitry Andric }
205349cc55cSDimitry Andric
206349cc55cSDimitry Andric return Error::success();
207349cc55cSDimitry Andric }
208349cc55cSDimitry Andric
deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR)209349cc55cSDimitry Andric Error ELFNixPlatformRuntimeState::deregisterObjectSections(
210349cc55cSDimitry Andric ELFNixPerObjectSectionsToRegister POSR) {
211349cc55cSDimitry Andric if (POSR.EHFrameSection.Start)
212*06c3fb27SDimitry Andric deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
213349cc55cSDimitry Andric
214349cc55cSDimitry Andric return Error::success();
215349cc55cSDimitry Andric }
216349cc55cSDimitry Andric
dlerror()217349cc55cSDimitry Andric const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
218349cc55cSDimitry Andric
dlopen(std::string_view Path,int Mode)219bdd1243dSDimitry Andric void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
220349cc55cSDimitry Andric std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
221349cc55cSDimitry Andric
222349cc55cSDimitry Andric // Use fast path if all JITDylibs are already loaded and don't require
223349cc55cSDimitry Andric // re-running initializers.
224349cc55cSDimitry Andric if (auto *JDS = getJITDylibStateByName(Path)) {
225349cc55cSDimitry Andric if (!JDS->AllowReinitialization) {
226349cc55cSDimitry Andric ++JDS->RefCount;
227349cc55cSDimitry Andric return JDS->Header;
228349cc55cSDimitry Andric }
229349cc55cSDimitry Andric }
230349cc55cSDimitry Andric
231349cc55cSDimitry Andric auto H = dlopenInitialize(Path, Mode);
232349cc55cSDimitry Andric if (!H) {
233349cc55cSDimitry Andric DLFcnError = toString(H.takeError());
234349cc55cSDimitry Andric return nullptr;
235349cc55cSDimitry Andric }
236349cc55cSDimitry Andric
237349cc55cSDimitry Andric return *H;
238349cc55cSDimitry Andric }
239349cc55cSDimitry Andric
dlclose(void * DSOHandle)240349cc55cSDimitry Andric int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
241349cc55cSDimitry Andric runAtExits(DSOHandle);
242349cc55cSDimitry Andric return 0;
243349cc55cSDimitry Andric }
244349cc55cSDimitry Andric
dlsym(void * DSOHandle,std::string_view Symbol)245bdd1243dSDimitry Andric void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle,
246bdd1243dSDimitry Andric std::string_view Symbol) {
247349cc55cSDimitry Andric auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
248349cc55cSDimitry Andric if (!Addr) {
249349cc55cSDimitry Andric DLFcnError = toString(Addr.takeError());
250349cc55cSDimitry Andric return 0;
251349cc55cSDimitry Andric }
252349cc55cSDimitry Andric
253349cc55cSDimitry Andric return Addr->toPtr<void *>();
254349cc55cSDimitry Andric }
255349cc55cSDimitry Andric
registerAtExit(void (* F)(void *),void * Arg,void * DSOHandle)256349cc55cSDimitry Andric int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
257349cc55cSDimitry Andric void *DSOHandle) {
258349cc55cSDimitry Andric // FIXME: Handle out-of-memory errors, returning -1 if OOM.
259349cc55cSDimitry Andric std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
260349cc55cSDimitry Andric auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
261349cc55cSDimitry Andric assert(JDS && "JITDylib state not initialized");
262349cc55cSDimitry Andric JDS->AtExits.push_back({F, Arg});
263349cc55cSDimitry Andric return 0;
264349cc55cSDimitry Andric }
265349cc55cSDimitry Andric
runAtExits(void * DSOHandle)266349cc55cSDimitry Andric void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
267349cc55cSDimitry Andric // FIXME: Should atexits be allowed to run concurrently with access to
268349cc55cSDimitry Andric // JDState?
269349cc55cSDimitry Andric AtExitsVector V;
270349cc55cSDimitry Andric {
271349cc55cSDimitry Andric std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
272349cc55cSDimitry Andric auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
273349cc55cSDimitry Andric assert(JDS && "JITDlybi state not initialized");
274349cc55cSDimitry Andric std::swap(V, JDS->AtExits);
275349cc55cSDimitry Andric }
276349cc55cSDimitry Andric
277349cc55cSDimitry Andric while (!V.empty()) {
278349cc55cSDimitry Andric auto &AE = V.back();
279349cc55cSDimitry Andric AE.Func(AE.Arg);
280349cc55cSDimitry Andric V.pop_back();
281349cc55cSDimitry Andric }
282349cc55cSDimitry Andric }
283349cc55cSDimitry Andric
284349cc55cSDimitry Andric Expected<std::pair<const char *, size_t>>
getThreadDataSectionFor(const char * ThreadData)285349cc55cSDimitry Andric ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
286349cc55cSDimitry Andric std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
287349cc55cSDimitry Andric auto I = ThreadDataSections.upper_bound(ThreadData);
288349cc55cSDimitry Andric // Check that we have a valid entry conovering this address.
289349cc55cSDimitry Andric if (I == ThreadDataSections.begin())
290349cc55cSDimitry Andric return make_error<StringError>("No thread local data section for key");
291349cc55cSDimitry Andric I = std::prev(I);
292349cc55cSDimitry Andric if (ThreadData >= I->first + I->second)
293349cc55cSDimitry Andric return make_error<StringError>("No thread local data section for key");
294349cc55cSDimitry Andric return *I;
295349cc55cSDimitry Andric }
296349cc55cSDimitry Andric
297349cc55cSDimitry Andric ELFNixPlatformRuntimeState::PerJITDylibState *
getJITDylibStateByHeaderAddr(void * DSOHandle)298349cc55cSDimitry Andric ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
299349cc55cSDimitry Andric auto I = JDStates.find(DSOHandle);
300349cc55cSDimitry Andric if (I == JDStates.end())
301349cc55cSDimitry Andric return nullptr;
302349cc55cSDimitry Andric return &I->second;
303349cc55cSDimitry Andric }
304349cc55cSDimitry Andric
305349cc55cSDimitry Andric ELFNixPlatformRuntimeState::PerJITDylibState *
getJITDylibStateByName(std::string_view Name)306bdd1243dSDimitry Andric ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
307349cc55cSDimitry Andric // FIXME: Avoid creating string copy here.
308349cc55cSDimitry Andric auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
309349cc55cSDimitry Andric if (I == JDNameToHeader.end())
310349cc55cSDimitry Andric return nullptr;
311349cc55cSDimitry Andric void *H = I->second;
312349cc55cSDimitry Andric auto J = JDStates.find(H);
313349cc55cSDimitry Andric assert(J != JDStates.end() &&
314349cc55cSDimitry Andric "JITDylib has name map entry but no header map entry");
315349cc55cSDimitry Andric return &J->second;
316349cc55cSDimitry Andric }
317349cc55cSDimitry Andric
318349cc55cSDimitry Andric ELFNixPlatformRuntimeState::PerJITDylibState &
getOrCreateJITDylibState(ELFNixJITDylibInitializers & MOJDIs)319349cc55cSDimitry Andric ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
320349cc55cSDimitry Andric ELFNixJITDylibInitializers &MOJDIs) {
321349cc55cSDimitry Andric void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
322349cc55cSDimitry Andric
323349cc55cSDimitry Andric auto &JDS = JDStates[Header];
324349cc55cSDimitry Andric
325349cc55cSDimitry Andric // If this entry hasn't been created yet.
326349cc55cSDimitry Andric if (!JDS.Header) {
327349cc55cSDimitry Andric assert(!JDNameToHeader.count(MOJDIs.Name) &&
328349cc55cSDimitry Andric "JITDylib has header map entry but no name map entry");
329349cc55cSDimitry Andric JDNameToHeader[MOJDIs.Name] = Header;
330349cc55cSDimitry Andric JDS.Header = Header;
331349cc55cSDimitry Andric }
332349cc55cSDimitry Andric
333349cc55cSDimitry Andric return JDS;
334349cc55cSDimitry Andric }
335349cc55cSDimitry Andric
registerThreadDataSection(span<const char> ThreadDataSection)336349cc55cSDimitry Andric Error ELFNixPlatformRuntimeState::registerThreadDataSection(
337349cc55cSDimitry Andric span<const char> ThreadDataSection) {
338349cc55cSDimitry Andric std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
339349cc55cSDimitry Andric auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
340349cc55cSDimitry Andric if (I != ThreadDataSections.begin()) {
341349cc55cSDimitry Andric auto J = std::prev(I);
342349cc55cSDimitry Andric if (J->first + J->second > ThreadDataSection.data())
343349cc55cSDimitry Andric return make_error<StringError>("Overlapping .tdata sections");
344349cc55cSDimitry Andric }
345349cc55cSDimitry Andric ThreadDataSections.insert(
346349cc55cSDimitry Andric I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
347349cc55cSDimitry Andric return Error::success();
348349cc55cSDimitry Andric }
349349cc55cSDimitry Andric
350349cc55cSDimitry Andric Expected<ExecutorAddr>
lookupSymbolInJITDylib(void * DSOHandle,std::string_view Sym)351349cc55cSDimitry Andric ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
352bdd1243dSDimitry Andric std::string_view Sym) {
353349cc55cSDimitry Andric Expected<ExecutorAddr> Result((ExecutorAddr()));
354349cc55cSDimitry Andric if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
355349cc55cSDimitry Andric SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag,
356349cc55cSDimitry Andric Result,
357349cc55cSDimitry Andric ExecutorAddr::fromPtr(DSOHandle),
358349cc55cSDimitry Andric Sym))
359349cc55cSDimitry Andric return std::move(Err);
360349cc55cSDimitry Andric return Result;
361349cc55cSDimitry Andric }
362349cc55cSDimitry Andric
363349cc55cSDimitry Andric Expected<ELFNixJITDylibInitializerSequence>
getJITDylibInitializersByName(std::string_view Path)364bdd1243dSDimitry Andric ELFNixPlatformRuntimeState::getJITDylibInitializersByName(
365bdd1243dSDimitry Andric std::string_view Path) {
366349cc55cSDimitry Andric Expected<ELFNixJITDylibInitializerSequence> Result(
367349cc55cSDimitry Andric (ELFNixJITDylibInitializerSequence()));
368349cc55cSDimitry Andric std::string PathStr(Path.data(), Path.size());
369349cc55cSDimitry Andric if (auto Err =
370349cc55cSDimitry Andric WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
371349cc55cSDimitry Andric SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
372349cc55cSDimitry Andric Path))
373349cc55cSDimitry Andric return std::move(Err);
374349cc55cSDimitry Andric return Result;
375349cc55cSDimitry Andric }
376349cc55cSDimitry Andric
377bdd1243dSDimitry Andric Expected<void *>
dlopenInitialize(std::string_view Path,int Mode)378bdd1243dSDimitry Andric ELFNixPlatformRuntimeState::dlopenInitialize(std::string_view Path, int Mode) {
379349cc55cSDimitry Andric // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
380349cc55cSDimitry Andric // reinitialization. We need to call in to the JIT to see if there's any new
381349cc55cSDimitry Andric // work pending.
382349cc55cSDimitry Andric auto InitSeq = getJITDylibInitializersByName(Path);
383349cc55cSDimitry Andric if (!InitSeq)
384349cc55cSDimitry Andric return InitSeq.takeError();
385349cc55cSDimitry Andric
386349cc55cSDimitry Andric // Init sequences should be non-empty.
387349cc55cSDimitry Andric if (InitSeq->empty())
388349cc55cSDimitry Andric return make_error<StringError>(
389349cc55cSDimitry Andric "__orc_rt_elfnix_get_initializers returned an "
390349cc55cSDimitry Andric "empty init sequence");
391349cc55cSDimitry Andric
392349cc55cSDimitry Andric // Otherwise register and run initializers for each JITDylib.
393349cc55cSDimitry Andric for (auto &MOJDIs : *InitSeq)
394349cc55cSDimitry Andric if (auto Err = initializeJITDylib(MOJDIs))
395349cc55cSDimitry Andric return std::move(Err);
396349cc55cSDimitry Andric
397349cc55cSDimitry Andric // Return the header for the last item in the list.
398349cc55cSDimitry Andric auto *JDS = getJITDylibStateByHeaderAddr(
399349cc55cSDimitry Andric InitSeq->back().DSOHandleAddress.toPtr<void *>());
400349cc55cSDimitry Andric assert(JDS && "Missing state entry for JD");
401349cc55cSDimitry Andric return JDS->Header;
402349cc55cSDimitry Andric }
403349cc55cSDimitry Andric
getPriority(const std::string & name)40481ad6265SDimitry Andric long getPriority(const std::string &name) {
40581ad6265SDimitry Andric auto pos = name.find_last_not_of("0123456789");
40681ad6265SDimitry Andric if (pos == name.size() - 1)
40781ad6265SDimitry Andric return 65535;
40881ad6265SDimitry Andric else
40981ad6265SDimitry Andric return std::strtol(name.c_str() + pos + 1, nullptr, 10);
41081ad6265SDimitry Andric }
41181ad6265SDimitry Andric
initializeJITDylib(ELFNixJITDylibInitializers & MOJDIs)412349cc55cSDimitry Andric Error ELFNixPlatformRuntimeState::initializeJITDylib(
413349cc55cSDimitry Andric ELFNixJITDylibInitializers &MOJDIs) {
414349cc55cSDimitry Andric
415349cc55cSDimitry Andric auto &JDS = getOrCreateJITDylibState(MOJDIs);
416349cc55cSDimitry Andric ++JDS.RefCount;
417349cc55cSDimitry Andric
41881ad6265SDimitry Andric using SectionList = std::vector<ExecutorAddrRange>;
41981ad6265SDimitry Andric std::sort(MOJDIs.InitSections.begin(), MOJDIs.InitSections.end(),
42081ad6265SDimitry Andric [](const std::pair<std::string, SectionList> &LHS,
42181ad6265SDimitry Andric const std::pair<std::string, SectionList> &RHS) -> bool {
42281ad6265SDimitry Andric return getPriority(LHS.first) < getPriority(RHS.first);
42381ad6265SDimitry Andric });
42481ad6265SDimitry Andric for (auto &Entry : MOJDIs.InitSections)
42581ad6265SDimitry Andric if (auto Err = runInitArray(Entry.second, MOJDIs))
426349cc55cSDimitry Andric return Err;
427349cc55cSDimitry Andric
428349cc55cSDimitry Andric return Error::success();
429349cc55cSDimitry Andric }
430349cc55cSDimitry Andric class ELFNixPlatformRuntimeTLVManager {
431349cc55cSDimitry Andric public:
432349cc55cSDimitry Andric void *getInstance(const char *ThreadData);
433349cc55cSDimitry Andric
434349cc55cSDimitry Andric private:
435349cc55cSDimitry Andric std::unordered_map<const char *, char *> Instances;
436349cc55cSDimitry Andric std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
437349cc55cSDimitry Andric };
438349cc55cSDimitry Andric
getInstance(const char * ThreadData)439349cc55cSDimitry Andric void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
440349cc55cSDimitry Andric auto I = Instances.find(ThreadData);
441349cc55cSDimitry Andric if (I != Instances.end())
442349cc55cSDimitry Andric return I->second;
443349cc55cSDimitry Andric auto TDS =
444349cc55cSDimitry Andric ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
445349cc55cSDimitry Andric if (!TDS) {
446349cc55cSDimitry Andric __orc_rt_log_error(toString(TDS.takeError()).c_str());
447349cc55cSDimitry Andric return nullptr;
448349cc55cSDimitry Andric }
449349cc55cSDimitry Andric
450349cc55cSDimitry Andric auto &Allocated = AllocatedSections[TDS->first];
451349cc55cSDimitry Andric if (!Allocated) {
452349cc55cSDimitry Andric Allocated = std::make_unique<char[]>(TDS->second);
453349cc55cSDimitry Andric memcpy(Allocated.get(), TDS->first, TDS->second);
454349cc55cSDimitry Andric }
455349cc55cSDimitry Andric size_t ThreadDataDelta = ThreadData - TDS->first;
456349cc55cSDimitry Andric assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
457349cc55cSDimitry Andric
458349cc55cSDimitry Andric char *Instance = Allocated.get() + ThreadDataDelta;
459349cc55cSDimitry Andric Instances[ThreadData] = Instance;
460349cc55cSDimitry Andric return Instance;
461349cc55cSDimitry Andric }
462349cc55cSDimitry Andric
destroyELFNixTLVMgr(void * ELFNixTLVMgr)463349cc55cSDimitry Andric void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
464349cc55cSDimitry Andric delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
465349cc55cSDimitry Andric }
466349cc55cSDimitry Andric
467349cc55cSDimitry Andric } // end anonymous namespace
468349cc55cSDimitry Andric
469349cc55cSDimitry Andric //------------------------------------------------------------------------------
470349cc55cSDimitry Andric // JIT entry points
471349cc55cSDimitry Andric //------------------------------------------------------------------------------
472349cc55cSDimitry Andric
473*06c3fb27SDimitry Andric ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_platform_bootstrap(char * ArgData,size_t ArgSize)474349cc55cSDimitry Andric __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
475349cc55cSDimitry Andric return WrapperFunction<void(uint64_t)>::handle(
476349cc55cSDimitry Andric ArgData, ArgSize,
477349cc55cSDimitry Andric [](uint64_t &DSOHandle) {
478349cc55cSDimitry Andric ELFNixPlatformRuntimeState::initialize(
479349cc55cSDimitry Andric reinterpret_cast<void *>(DSOHandle));
480349cc55cSDimitry Andric })
481349cc55cSDimitry Andric .release();
482349cc55cSDimitry Andric }
483349cc55cSDimitry Andric
484*06c3fb27SDimitry Andric ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_platform_shutdown(char * ArgData,size_t ArgSize)485349cc55cSDimitry Andric __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
486349cc55cSDimitry Andric ELFNixPlatformRuntimeState::destroy();
487349cc55cSDimitry Andric return WrapperFunctionResult().release();
488349cc55cSDimitry Andric }
489349cc55cSDimitry Andric
490349cc55cSDimitry Andric /// Wrapper function for registering metadata on a per-object basis.
491*06c3fb27SDimitry Andric ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_register_object_sections(char * ArgData,size_t ArgSize)492349cc55cSDimitry Andric __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
493349cc55cSDimitry Andric return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
494349cc55cSDimitry Andric handle(ArgData, ArgSize,
495349cc55cSDimitry Andric [](ELFNixPerObjectSectionsToRegister &POSR) {
496349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get().registerObjectSections(
497349cc55cSDimitry Andric std::move(POSR));
498349cc55cSDimitry Andric })
499349cc55cSDimitry Andric .release();
500349cc55cSDimitry Andric }
501349cc55cSDimitry Andric
502349cc55cSDimitry Andric /// Wrapper for releasing per-object metadat.
503*06c3fb27SDimitry Andric ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_deregister_object_sections(char * ArgData,size_t ArgSize)504349cc55cSDimitry Andric __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
505349cc55cSDimitry Andric return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
506349cc55cSDimitry Andric handle(ArgData, ArgSize,
507349cc55cSDimitry Andric [](ELFNixPerObjectSectionsToRegister &POSR) {
508349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get()
509349cc55cSDimitry Andric .deregisterObjectSections(std::move(POSR));
510349cc55cSDimitry Andric })
511349cc55cSDimitry Andric .release();
512349cc55cSDimitry Andric }
513349cc55cSDimitry Andric
514349cc55cSDimitry Andric //------------------------------------------------------------------------------
515349cc55cSDimitry Andric // TLV support
516349cc55cSDimitry Andric //------------------------------------------------------------------------------
517349cc55cSDimitry Andric
__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry * D)518349cc55cSDimitry Andric ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
519349cc55cSDimitry Andric auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
520349cc55cSDimitry Andric pthread_getspecific(D->Key));
521349cc55cSDimitry Andric if (!TLVMgr)
522349cc55cSDimitry Andric TLVMgr = new ELFNixPlatformRuntimeTLVManager();
523349cc55cSDimitry Andric if (pthread_setspecific(D->Key, TLVMgr)) {
524349cc55cSDimitry Andric __orc_rt_log_error("Call to pthread_setspecific failed");
525349cc55cSDimitry Andric return nullptr;
526349cc55cSDimitry Andric }
527349cc55cSDimitry Andric
528349cc55cSDimitry Andric return TLVMgr->getInstance(
529349cc55cSDimitry Andric reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
530349cc55cSDimitry Andric }
531349cc55cSDimitry Andric
___orc_rt_elfnix_tlsdesc_resolver_impl(TLSDescriptor * D,const char * ThreadPointer)532753f127fSDimitry Andric ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
533753f127fSDimitry Andric TLSDescriptor *D, const char *ThreadPointer) {
534753f127fSDimitry Andric const char *TLVPtr = reinterpret_cast<const char *>(
535753f127fSDimitry Andric __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry));
536753f127fSDimitry Andric return TLVPtr - ThreadPointer;
537753f127fSDimitry Andric }
538753f127fSDimitry Andric
539*06c3fb27SDimitry Andric ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_create_pthread_key(char * ArgData,size_t ArgSize)540349cc55cSDimitry Andric __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
541349cc55cSDimitry Andric return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
542349cc55cSDimitry Andric ArgData, ArgSize,
543349cc55cSDimitry Andric []() -> Expected<uint64_t> {
544349cc55cSDimitry Andric pthread_key_t Key;
545349cc55cSDimitry Andric if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
546349cc55cSDimitry Andric __orc_rt_log_error("Call to pthread_key_create failed");
547349cc55cSDimitry Andric return make_error<StringError>(strerror(Err));
548349cc55cSDimitry Andric }
549349cc55cSDimitry Andric return static_cast<uint64_t>(Key);
550349cc55cSDimitry Andric })
551349cc55cSDimitry Andric .release();
552349cc55cSDimitry Andric }
553349cc55cSDimitry Andric
554349cc55cSDimitry Andric //------------------------------------------------------------------------------
555349cc55cSDimitry Andric // cxa_atexit support
556349cc55cSDimitry Andric //------------------------------------------------------------------------------
557349cc55cSDimitry Andric
__orc_rt_elfnix_cxa_atexit(void (* func)(void *),void * arg,void * dso_handle)558349cc55cSDimitry Andric int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
559349cc55cSDimitry Andric void *dso_handle) {
560349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
561349cc55cSDimitry Andric dso_handle);
562349cc55cSDimitry Andric }
563349cc55cSDimitry Andric
__orc_rt_elfnix_atexit(void (* func)(void *))564349cc55cSDimitry Andric int __orc_rt_elfnix_atexit(void (*func)(void *)) {
565349cc55cSDimitry Andric auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
566349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get().registerAtExit(
567349cc55cSDimitry Andric func, NULL, PlatformRTState.getPlatformJDDSOHandle());
568349cc55cSDimitry Andric }
569349cc55cSDimitry Andric
__orc_rt_elfnix_cxa_finalize(void * dso_handle)570349cc55cSDimitry Andric void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
571349cc55cSDimitry Andric ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
572349cc55cSDimitry Andric }
573349cc55cSDimitry Andric
574349cc55cSDimitry Andric //------------------------------------------------------------------------------
575349cc55cSDimitry Andric // JIT'd dlfcn alternatives.
576349cc55cSDimitry Andric //------------------------------------------------------------------------------
577349cc55cSDimitry Andric
__orc_rt_elfnix_jit_dlerror()578349cc55cSDimitry Andric const char *__orc_rt_elfnix_jit_dlerror() {
579349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get().dlerror();
580349cc55cSDimitry Andric }
581349cc55cSDimitry Andric
__orc_rt_elfnix_jit_dlopen(const char * path,int mode)582349cc55cSDimitry Andric void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
583349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
584349cc55cSDimitry Andric }
585349cc55cSDimitry Andric
__orc_rt_elfnix_jit_dlclose(void * dso_handle)586349cc55cSDimitry Andric int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
587349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
588349cc55cSDimitry Andric }
589349cc55cSDimitry Andric
__orc_rt_elfnix_jit_dlsym(void * dso_handle,const char * symbol)590349cc55cSDimitry Andric void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
591349cc55cSDimitry Andric return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
592349cc55cSDimitry Andric }
593349cc55cSDimitry Andric
594349cc55cSDimitry Andric //------------------------------------------------------------------------------
595349cc55cSDimitry Andric // ELFNix Run Program
596349cc55cSDimitry Andric //------------------------------------------------------------------------------
597349cc55cSDimitry Andric
__orc_rt_elfnix_run_program(const char * JITDylibName,const char * EntrySymbolName,int argc,char * argv[])598349cc55cSDimitry Andric ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
599349cc55cSDimitry Andric const char *JITDylibName, const char *EntrySymbolName, int argc,
600349cc55cSDimitry Andric char *argv[]) {
601349cc55cSDimitry Andric using MainTy = int (*)(int, char *[]);
602349cc55cSDimitry Andric
603349cc55cSDimitry Andric void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
604349cc55cSDimitry Andric __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
605349cc55cSDimitry Andric if (!H) {
606349cc55cSDimitry Andric __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
607349cc55cSDimitry Andric return -1;
608349cc55cSDimitry Andric }
609349cc55cSDimitry Andric
610349cc55cSDimitry Andric auto *Main =
611349cc55cSDimitry Andric reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
612349cc55cSDimitry Andric
613349cc55cSDimitry Andric if (!Main) {
614349cc55cSDimitry Andric __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
615349cc55cSDimitry Andric return -1;
616349cc55cSDimitry Andric }
617349cc55cSDimitry Andric
618349cc55cSDimitry Andric int Result = Main(argc, argv);
619349cc55cSDimitry Andric
620349cc55cSDimitry Andric if (__orc_rt_elfnix_jit_dlclose(H) == -1)
621349cc55cSDimitry Andric __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
622349cc55cSDimitry Andric
623349cc55cSDimitry Andric return Result;
624349cc55cSDimitry Andric }
625