1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Functionality for handling RTL plugins.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "rtl.h"
14 #include "device.h"
15 #include "private.h"
16
17 #if OMPT_SUPPORT
18 #include "ompt-target.h"
19 #endif
20
21 #include <cassert>
22 #include <cstdlib>
23 #include <cstring>
24 #include <dlfcn.h>
25 #include <mutex>
26 #include <string>
27
28 // List of all plugins that can support offloading.
29 static const char *RTLNames[] = {
30 /* PowerPC target */ "libomptarget.rtl.ppc64.so",
31 /* x86_64 target */ "libomptarget.rtl.x86_64.so",
32 /* CUDA target */ "libomptarget.rtl.cuda.so",
33 /* AArch64 target */ "libomptarget.rtl.aarch64.so",
34 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so",
35 /* AMDGPU target */ "libomptarget.rtl.amdgpu.so",
36 /* Remote target */ "libomptarget.rtl.rpc.so",
37 };
38
39 PluginManager *PM;
40
41 #if OMPTARGET_PROFILE_ENABLED
42 static char *ProfileTraceFile = nullptr;
43 #endif
44
init()45 __attribute__((constructor(101))) void init() {
46 DP("Init target library!\n");
47 PM = new PluginManager();
48
49 #ifdef OMPTARGET_PROFILE_ENABLED
50 ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE");
51 // TODO: add a configuration option for time granularity
52 if (ProfileTraceFile)
53 llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget");
54 #endif
55 }
56
deinit()57 __attribute__((destructor(101))) void deinit() {
58 DP("Deinit target library!\n");
59 delete PM;
60
61 #ifdef OMPTARGET_PROFILE_ENABLED
62 if (ProfileTraceFile) {
63 // TODO: add env var for file output
64 if (auto E = llvm::timeTraceProfilerWrite(ProfileTraceFile, "-"))
65 fprintf(stderr, "Error writing out the time trace\n");
66
67 llvm::timeTraceProfilerCleanup();
68 }
69 #endif
70 }
71
LoadRTLs()72 void RTLsTy::LoadRTLs() {
73 // Parse environment variable OMP_TARGET_OFFLOAD (if set)
74 PM->TargetOffloadPolicy =
75 (kmp_target_offload_kind_t)__kmpc_get_target_offload();
76 if (PM->TargetOffloadPolicy == tgt_disabled) {
77 return;
78 }
79
80 DP("Loading RTLs...\n");
81
82 // Attempt to open all the plugins and, if they exist, check if the interface
83 // is correct and if they are supporting any devices.
84 for (auto *Name : RTLNames) {
85 DP("Loading library '%s'...\n", Name);
86 void *dynlib_handle = dlopen(Name, RTLD_NOW);
87
88 if (!dynlib_handle) {
89 // Library does not exist or cannot be found.
90 DP("Unable to load library '%s': %s!\n", Name, dlerror());
91 continue;
92 }
93
94 DP("Successfully loaded library '%s'!\n", Name);
95
96 AllRTLs.emplace_back();
97
98 // Retrieve the RTL information from the runtime library.
99 RTLInfoTy &R = AllRTLs.back();
100
101 bool ValidPlugin = true;
102
103 if (!(*((void **)&R.is_valid_binary) =
104 dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary")))
105 ValidPlugin = false;
106 if (!(*((void **)&R.number_of_devices) =
107 dlsym(dynlib_handle, "__tgt_rtl_number_of_devices")))
108 ValidPlugin = false;
109 if (!(*((void **)&R.init_device) =
110 dlsym(dynlib_handle, "__tgt_rtl_init_device")))
111 ValidPlugin = false;
112 if (!(*((void **)&R.load_binary) =
113 dlsym(dynlib_handle, "__tgt_rtl_load_binary")))
114 ValidPlugin = false;
115 if (!(*((void **)&R.data_alloc) =
116 dlsym(dynlib_handle, "__tgt_rtl_data_alloc")))
117 ValidPlugin = false;
118 if (!(*((void **)&R.data_submit) =
119 dlsym(dynlib_handle, "__tgt_rtl_data_submit")))
120 ValidPlugin = false;
121 if (!(*((void **)&R.data_retrieve) =
122 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve")))
123 ValidPlugin = false;
124 if (!(*((void **)&R.data_delete) =
125 dlsym(dynlib_handle, "__tgt_rtl_data_delete")))
126 ValidPlugin = false;
127 if (!(*((void **)&R.run_region) =
128 dlsym(dynlib_handle, "__tgt_rtl_run_target_region")))
129 ValidPlugin = false;
130 if (!(*((void **)&R.run_team_region) =
131 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region")))
132 ValidPlugin = false;
133
134 // Invalid plugin
135 if (!ValidPlugin) {
136 DP("Invalid plugin as necessary interface is not found.\n");
137 AllRTLs.pop_back();
138 continue;
139 }
140
141 // No devices are supported by this RTL?
142 if (!(R.NumberOfDevices = R.number_of_devices())) {
143 // The RTL is invalid! Will pop the object from the RTLs list.
144 DP("No devices supported in this RTL\n");
145 AllRTLs.pop_back();
146 continue;
147 }
148
149 R.LibraryHandler = dynlib_handle;
150
151 #ifdef OMPTARGET_DEBUG
152 R.RTLName = Name;
153 #endif
154
155 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(),
156 R.NumberOfDevices);
157
158 // Optional functions
159 *((void **)&R.init_requires) =
160 dlsym(dynlib_handle, "__tgt_rtl_init_requires");
161 *((void **)&R.data_submit_async) =
162 dlsym(dynlib_handle, "__tgt_rtl_data_submit_async");
163 *((void **)&R.data_retrieve_async) =
164 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async");
165 *((void **)&R.run_region_async) =
166 dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async");
167 *((void **)&R.run_team_region_async) =
168 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async");
169 *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize");
170 *((void **)&R.data_exchange) =
171 dlsym(dynlib_handle, "__tgt_rtl_data_exchange");
172 *((void **)&R.data_exchange_async) =
173 dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async");
174 *((void **)&R.is_data_exchangable) =
175 dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable");
176 *((void **)&R.register_lib) =
177 dlsym(dynlib_handle, "__tgt_rtl_register_lib");
178 *((void **)&R.unregister_lib) =
179 dlsym(dynlib_handle, "__tgt_rtl_unregister_lib");
180 *((void **)&R.supports_empty_images) =
181 dlsym(dynlib_handle, "__tgt_rtl_supports_empty_images");
182 *((void **)&R.set_info_flag) =
183 dlsym(dynlib_handle, "__tgt_rtl_set_info_flag");
184 *((void **)&R.print_device_info) =
185 dlsym(dynlib_handle, "__tgt_rtl_print_device_info");
186 *((void **)&R.create_event) =
187 dlsym(dynlib_handle, "__tgt_rtl_create_event");
188 *((void **)&R.record_event) =
189 dlsym(dynlib_handle, "__tgt_rtl_record_event");
190 *((void **)&R.wait_event) = dlsym(dynlib_handle, "__tgt_rtl_wait_event");
191 *((void **)&R.sync_event) = dlsym(dynlib_handle, "__tgt_rtl_sync_event");
192 *((void **)&R.destroy_event) =
193 dlsym(dynlib_handle, "__tgt_rtl_destroy_event");
194 }
195
196 #if OMPT_SUPPORT
197 DP("OMPT_SUPPORT is enabled in libomptarget\n");
198 DP("Init OMPT for libomptarget\n");
199 if (libomp_start_tool) {
200 DP("Retrieve libomp_start_tool successfully\n");
201 if (!libomp_start_tool(&ompt_target_enabled)) {
202 DP("Turn off OMPT in libomptarget because libomp_start_tool returns "
203 "false\n");
204 memset(&ompt_target_enabled, 0, sizeof(ompt_target_enabled));
205 }
206 }
207 #endif
208
209 DP("RTLs loaded!\n");
210
211 return;
212 }
213
214 ////////////////////////////////////////////////////////////////////////////////
215 // Functionality for registering libs
216
RegisterImageIntoTranslationTable(TranslationTable & TT,RTLInfoTy & RTL,__tgt_device_image * image)217 static void RegisterImageIntoTranslationTable(TranslationTable &TT,
218 RTLInfoTy &RTL,
219 __tgt_device_image *image) {
220
221 // same size, as when we increase one, we also increase the other.
222 assert(TT.TargetsTable.size() == TT.TargetsImages.size() &&
223 "We should have as many images as we have tables!");
224
225 // Resize the Targets Table and Images to accommodate the new targets if
226 // required
227 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices;
228
229 if (TT.TargetsTable.size() < TargetsTableMinimumSize) {
230 TT.TargetsImages.resize(TargetsTableMinimumSize, 0);
231 TT.TargetsTable.resize(TargetsTableMinimumSize, 0);
232 }
233
234 // Register the image in all devices for this target type.
235 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) {
236 // If we are changing the image we are also invalidating the target table.
237 if (TT.TargetsImages[RTL.Idx + i] != image) {
238 TT.TargetsImages[RTL.Idx + i] = image;
239 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table.
240 }
241 }
242 }
243
244 ////////////////////////////////////////////////////////////////////////////////
245 // Functionality for registering Ctors/Dtors
246
RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc * desc,__tgt_device_image * img,RTLInfoTy * RTL)247 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc,
248 __tgt_device_image *img,
249 RTLInfoTy *RTL) {
250
251 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) {
252 DeviceTy &Device = *PM->Devices[RTL->Idx + i];
253 Device.PendingGlobalsMtx.lock();
254 Device.HasPendingGlobals = true;
255 for (__tgt_offload_entry *entry = img->EntriesBegin;
256 entry != img->EntriesEnd; ++entry) {
257 if (entry->flags & OMP_DECLARE_TARGET_CTOR) {
258 DP("Adding ctor " DPxMOD " to the pending list.\n",
259 DPxPTR(entry->addr));
260 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr);
261 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) {
262 // Dtors are pushed in reverse order so they are executed from end
263 // to beginning when unregistering the library!
264 DP("Adding dtor " DPxMOD " to the pending list.\n",
265 DPxPTR(entry->addr));
266 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr);
267 }
268
269 if (entry->flags & OMP_DECLARE_TARGET_LINK) {
270 DP("The \"link\" attribute is not yet supported!\n");
271 }
272 }
273 Device.PendingGlobalsMtx.unlock();
274 }
275 }
276
RegisterRequires(int64_t flags)277 void RTLsTy::RegisterRequires(int64_t flags) {
278 // TODO: add more elaborate check.
279 // Minimal check: only set requires flags if previous value
280 // is undefined. This ensures that only the first call to this
281 // function will set the requires flags. All subsequent calls
282 // will be checked for compatibility.
283 assert(flags != OMP_REQ_UNDEFINED &&
284 "illegal undefined flag for requires directive!");
285 if (RequiresFlags == OMP_REQ_UNDEFINED) {
286 RequiresFlags = flags;
287 return;
288 }
289
290 // If multiple compilation units are present enforce
291 // consistency across all of them for require clauses:
292 // - reverse_offload
293 // - unified_address
294 // - unified_shared_memory
295 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) !=
296 (flags & OMP_REQ_REVERSE_OFFLOAD)) {
297 FATAL_MESSAGE0(
298 1, "'#pragma omp requires reverse_offload' not used consistently!");
299 }
300 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) !=
301 (flags & OMP_REQ_UNIFIED_ADDRESS)) {
302 FATAL_MESSAGE0(
303 1, "'#pragma omp requires unified_address' not used consistently!");
304 }
305 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) !=
306 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) {
307 FATAL_MESSAGE0(
308 1,
309 "'#pragma omp requires unified_shared_memory' not used consistently!");
310 }
311
312 // TODO: insert any other missing checks
313
314 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n",
315 flags, RequiresFlags);
316 }
317
initRTLonce(RTLInfoTy & R)318 void RTLsTy::initRTLonce(RTLInfoTy &R) {
319 // If this RTL is not already in use, initialize it.
320 if (!R.isUsed && R.NumberOfDevices != 0) {
321 // Initialize the device information for the RTL we are about to use.
322 const size_t Start = PM->Devices.size();
323 PM->Devices.reserve(Start + R.NumberOfDevices);
324 for (int32_t device_id = 0; device_id < R.NumberOfDevices; device_id++) {
325 PM->Devices.push_back(std::make_unique<DeviceTy>(&R));
326 // global device ID
327 PM->Devices[Start + device_id]->DeviceID = Start + device_id;
328 // RTL local device ID
329 PM->Devices[Start + device_id]->RTLDeviceID = device_id;
330 }
331
332 // Initialize the index of this RTL and save it in the used RTLs.
333 R.Idx = (UsedRTLs.empty())
334 ? 0
335 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices;
336 assert((size_t)R.Idx == Start &&
337 "RTL index should equal the number of devices used so far.");
338 R.isUsed = true;
339 UsedRTLs.push_back(&R);
340
341 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx);
342 }
343 }
344
initAllRTLs()345 void RTLsTy::initAllRTLs() {
346 for (auto &R : AllRTLs)
347 initRTLonce(R);
348 }
349
RegisterLib(__tgt_bin_desc * desc)350 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) {
351 PM->RTLsMtx.lock();
352 // Register the images with the RTLs that understand them, if any.
353 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
354 // Obtain the image.
355 __tgt_device_image *img = &desc->DeviceImages[i];
356
357 RTLInfoTy *FoundRTL = nullptr;
358
359 // Scan the RTLs that have associated images until we find one that supports
360 // the current image.
361 for (auto &R : AllRTLs) {
362 if (!R.is_valid_binary(img)) {
363 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
364 DPxPTR(img->ImageStart), R.RTLName.c_str());
365 continue;
366 }
367
368 DP("Image " DPxMOD " is compatible with RTL %s!\n",
369 DPxPTR(img->ImageStart), R.RTLName.c_str());
370
371 initRTLonce(R);
372
373 // Initialize (if necessary) translation table for this library.
374 PM->TrlTblMtx.lock();
375 if (!PM->HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)) {
376 PM->HostEntriesBeginRegistrationOrder.push_back(desc->HostEntriesBegin);
377 TranslationTable &TransTable =
378 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
379 TransTable.HostTable.EntriesBegin = desc->HostEntriesBegin;
380 TransTable.HostTable.EntriesEnd = desc->HostEntriesEnd;
381 }
382
383 // Retrieve translation table for this library.
384 TranslationTable &TransTable =
385 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
386
387 DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(img->ImageStart),
388 R.RTLName.c_str());
389 RegisterImageIntoTranslationTable(TransTable, R, img);
390 PM->TrlTblMtx.unlock();
391 FoundRTL = &R;
392
393 // Load ctors/dtors for static objects
394 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL);
395
396 // if an RTL was found we are done - proceed to register the next image
397 break;
398 }
399
400 if (!FoundRTL) {
401 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart));
402 }
403 }
404 PM->RTLsMtx.unlock();
405
406 DP("Done registering entries!\n");
407 }
408
UnregisterLib(__tgt_bin_desc * desc)409 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) {
410 DP("Unloading target library!\n");
411
412 PM->RTLsMtx.lock();
413 // Find which RTL understands each image, if any.
414 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
415 // Obtain the image.
416 __tgt_device_image *img = &desc->DeviceImages[i];
417
418 RTLInfoTy *FoundRTL = NULL;
419
420 // Scan the RTLs that have associated images until we find one that supports
421 // the current image. We only need to scan RTLs that are already being used.
422 for (auto *R : UsedRTLs) {
423
424 assert(R->isUsed && "Expecting used RTLs.");
425
426 if (!R->is_valid_binary(img)) {
427 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n",
428 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
429 continue;
430 }
431
432 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n",
433 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
434
435 FoundRTL = R;
436
437 // Execute dtors for static objects if the device has been used, i.e.
438 // if its PendingCtors list has been emptied.
439 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) {
440 DeviceTy &Device = *PM->Devices[FoundRTL->Idx + i];
441 Device.PendingGlobalsMtx.lock();
442 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) {
443 AsyncInfoTy AsyncInfo(Device);
444 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) {
445 int rc = target(nullptr, Device, dtor, 0, nullptr, nullptr, nullptr,
446 nullptr, nullptr, nullptr, 1, 1, true /*team*/,
447 AsyncInfo);
448 if (rc != OFFLOAD_SUCCESS) {
449 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor));
450 }
451 }
452 // Remove this library's entry from PendingCtorsDtors
453 Device.PendingCtorsDtors.erase(desc);
454 // All constructors have been issued, wait for them now.
455 if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS)
456 DP("Failed synchronizing destructors kernels.\n");
457 }
458 Device.PendingGlobalsMtx.unlock();
459 }
460
461 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
462 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
463
464 break;
465 }
466
467 // if no RTL was found proceed to unregister the next image
468 if (!FoundRTL) {
469 DP("No RTLs in use support the image " DPxMOD "!\n",
470 DPxPTR(img->ImageStart));
471 }
472 }
473 PM->RTLsMtx.unlock();
474 DP("Done unregistering images!\n");
475
476 // Remove entries from PM->HostPtrToTableMap
477 PM->TblMapMtx.lock();
478 for (__tgt_offload_entry *cur = desc->HostEntriesBegin;
479 cur < desc->HostEntriesEnd; ++cur) {
480 PM->HostPtrToTableMap.erase(cur->addr);
481 }
482
483 // Remove translation table for this descriptor.
484 auto TransTable =
485 PM->HostEntriesBeginToTransTable.find(desc->HostEntriesBegin);
486 if (TransTable != PM->HostEntriesBeginToTransTable.end()) {
487 DP("Removing translation table for descriptor " DPxMOD "\n",
488 DPxPTR(desc->HostEntriesBegin));
489 PM->HostEntriesBeginToTransTable.erase(TransTable);
490 } else {
491 DP("Translation table for descriptor " DPxMOD " cannot be found, probably "
492 "it has been already removed.\n",
493 DPxPTR(desc->HostEntriesBegin));
494 }
495
496 PM->TblMapMtx.unlock();
497
498 // TODO: Remove RTL and the devices it manages if it's not used anymore?
499 // TODO: Write some RTL->unload_image(...) function?
500
501 DP("Done unregistering library!\n");
502 }
503