1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of OpenMP API interface functions.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include <omptarget.h>
14
15 #include "device.h"
16 #include "private.h"
17 #include "rtl.h"
18
19 #include <climits>
20 #include <cstring>
21 #include <cstdlib>
22
omp_get_num_devices(void)23 EXTERN int omp_get_num_devices(void) {
24 RTLsMtx->lock();
25 size_t Devices_size = Devices.size();
26 RTLsMtx->unlock();
27
28 DP("Call to omp_get_num_devices returning %zd\n", Devices_size);
29
30 return Devices_size;
31 }
32
omp_get_initial_device(void)33 EXTERN int omp_get_initial_device(void) {
34 DP("Call to omp_get_initial_device returning %d\n", HOST_DEVICE);
35 return HOST_DEVICE;
36 }
37
omp_target_alloc(size_t size,int device_num)38 EXTERN void *omp_target_alloc(size_t size, int device_num) {
39 DP("Call to omp_target_alloc for device %d requesting %zu bytes\n",
40 device_num, size);
41
42 if (size <= 0) {
43 DP("Call to omp_target_alloc with non-positive length\n");
44 return NULL;
45 }
46
47 void *rc = NULL;
48
49 if (device_num == omp_get_initial_device()) {
50 rc = malloc(size);
51 DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc));
52 return rc;
53 }
54
55 if (!device_is_ready(device_num)) {
56 DP("omp_target_alloc returns NULL ptr\n");
57 return NULL;
58 }
59
60 DeviceTy &Device = Devices[device_num];
61 rc = Device.RTL->data_alloc(Device.RTLDeviceID, size, NULL);
62 DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
63 return rc;
64 }
65
omp_target_free(void * device_ptr,int device_num)66 EXTERN void omp_target_free(void *device_ptr, int device_num) {
67 DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
68 device_num, DPxPTR(device_ptr));
69
70 if (!device_ptr) {
71 DP("Call to omp_target_free with NULL ptr\n");
72 return;
73 }
74
75 if (device_num == omp_get_initial_device()) {
76 free(device_ptr);
77 DP("omp_target_free deallocated host ptr\n");
78 return;
79 }
80
81 if (!device_is_ready(device_num)) {
82 DP("omp_target_free returns, nothing to do\n");
83 return;
84 }
85
86 DeviceTy &Device = Devices[device_num];
87 Device.RTL->data_delete(Device.RTLDeviceID, (void *)device_ptr);
88 DP("omp_target_free deallocated device ptr\n");
89 }
90
omp_target_is_present(void * ptr,int device_num)91 EXTERN int omp_target_is_present(void *ptr, int device_num) {
92 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
93 device_num, DPxPTR(ptr));
94
95 if (!ptr) {
96 DP("Call to omp_target_is_present with NULL ptr, returning false\n");
97 return false;
98 }
99
100 if (device_num == omp_get_initial_device()) {
101 DP("Call to omp_target_is_present on host, returning true\n");
102 return true;
103 }
104
105 RTLsMtx->lock();
106 size_t Devices_size = Devices.size();
107 RTLsMtx->unlock();
108 if (Devices_size <= (size_t)device_num) {
109 DP("Call to omp_target_is_present with invalid device ID, returning "
110 "false\n");
111 return false;
112 }
113
114 DeviceTy& Device = Devices[device_num];
115 bool IsLast; // not used
116 bool IsHostPtr;
117 void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr);
118 int rc = (TgtPtr != NULL);
119 // Under unified memory the host pointer can be returned by the
120 // getTgtPtrBegin() function which means that there is no device
121 // corresponding point for ptr. This function should return false
122 // in that situation.
123 if (RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
124 rc = !IsHostPtr;
125 DP("Call to omp_target_is_present returns %d\n", rc);
126 return rc;
127 }
128
omp_target_memcpy(void * dst,void * src,size_t length,size_t dst_offset,size_t src_offset,int dst_device,int src_device)129 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
130 size_t dst_offset, size_t src_offset, int dst_device, int src_device) {
131 DP("Call to omp_target_memcpy, dst device %d, src device %d, "
132 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
133 "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst),
134 DPxPTR(src), dst_offset, src_offset, length);
135
136 if (!dst || !src || length <= 0) {
137 DP("Call to omp_target_memcpy with invalid arguments\n");
138 return OFFLOAD_FAIL;
139 }
140
141 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
142 DP("omp_target_memcpy returns OFFLOAD_FAIL\n");
143 return OFFLOAD_FAIL;
144 }
145
146 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
147 DP("omp_target_memcpy returns OFFLOAD_FAIL\n");
148 return OFFLOAD_FAIL;
149 }
150
151 int rc = OFFLOAD_SUCCESS;
152 void *srcAddr = (char *)src + src_offset;
153 void *dstAddr = (char *)dst + dst_offset;
154
155 if (src_device == omp_get_initial_device() &&
156 dst_device == omp_get_initial_device()) {
157 DP("copy from host to host\n");
158 const void *p = memcpy(dstAddr, srcAddr, length);
159 if (p == NULL)
160 rc = OFFLOAD_FAIL;
161 } else if (src_device == omp_get_initial_device()) {
162 DP("copy from host to device\n");
163 DeviceTy& DstDev = Devices[dst_device];
164 rc = DstDev.data_submit(dstAddr, srcAddr, length, nullptr);
165 } else if (dst_device == omp_get_initial_device()) {
166 DP("copy from device to host\n");
167 DeviceTy& SrcDev = Devices[src_device];
168 rc = SrcDev.data_retrieve(dstAddr, srcAddr, length, nullptr);
169 } else {
170 DP("copy from device to device\n");
171 DeviceTy &SrcDev = Devices[src_device];
172 DeviceTy &DstDev = Devices[dst_device];
173 // First try to use D2D memcpy which is more efficient. If fails, fall back
174 // to unefficient way.
175 if (SrcDev.isDataExchangable(DstDev)) {
176 rc = SrcDev.data_exchange(srcAddr, DstDev, dstAddr, length, nullptr);
177 if (rc == OFFLOAD_SUCCESS)
178 return OFFLOAD_SUCCESS;
179 }
180
181 void *buffer = malloc(length);
182 rc = SrcDev.data_retrieve(buffer, srcAddr, length, nullptr);
183 if (rc == OFFLOAD_SUCCESS)
184 rc = DstDev.data_submit(dstAddr, buffer, length, nullptr);
185 free(buffer);
186 }
187
188 DP("omp_target_memcpy returns %d\n", rc);
189 return rc;
190 }
191
omp_target_memcpy_rect(void * dst,void * src,size_t element_size,int num_dims,const size_t * volume,const size_t * dst_offsets,const size_t * src_offsets,const size_t * dst_dimensions,const size_t * src_dimensions,int dst_device,int src_device)192 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
193 int num_dims, const size_t *volume, const size_t *dst_offsets,
194 const size_t *src_offsets, const size_t *dst_dimensions,
195 const size_t *src_dimensions, int dst_device, int src_device) {
196 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
197 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
198 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
199 "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device,
200 src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
201 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
202 DPxPTR(volume), element_size, num_dims);
203
204 if (!(dst || src)) {
205 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
206 INT_MAX);
207 return INT_MAX;
208 }
209
210 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
211 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
212 DP("Call to omp_target_memcpy_rect with invalid arguments\n");
213 return OFFLOAD_FAIL;
214 }
215
216 int rc;
217 if (num_dims == 1) {
218 rc = omp_target_memcpy(dst, src, element_size * volume[0],
219 element_size * dst_offsets[0], element_size * src_offsets[0],
220 dst_device, src_device);
221 } else {
222 size_t dst_slice_size = element_size;
223 size_t src_slice_size = element_size;
224 for (int i=1; i<num_dims; ++i) {
225 dst_slice_size *= dst_dimensions[i];
226 src_slice_size *= src_dimensions[i];
227 }
228
229 size_t dst_off = dst_offsets[0] * dst_slice_size;
230 size_t src_off = src_offsets[0] * src_slice_size;
231 for (size_t i=0; i<volume[0]; ++i) {
232 rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i,
233 (char *) src + src_off + src_slice_size * i, element_size,
234 num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1,
235 dst_dimensions + 1, src_dimensions + 1, dst_device, src_device);
236
237 if (rc) {
238 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
239 return rc;
240 }
241 }
242 }
243
244 DP("omp_target_memcpy_rect returns %d\n", rc);
245 return rc;
246 }
247
omp_target_associate_ptr(void * host_ptr,void * device_ptr,size_t size,size_t device_offset,int device_num)248 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
249 size_t size, size_t device_offset, int device_num) {
250 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
251 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
252 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
253
254 if (!host_ptr || !device_ptr || size <= 0) {
255 DP("Call to omp_target_associate_ptr with invalid arguments\n");
256 return OFFLOAD_FAIL;
257 }
258
259 if (device_num == omp_get_initial_device()) {
260 DP("omp_target_associate_ptr: no association possible on the host\n");
261 return OFFLOAD_FAIL;
262 }
263
264 if (!device_is_ready(device_num)) {
265 DP("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
266 return OFFLOAD_FAIL;
267 }
268
269 DeviceTy& Device = Devices[device_num];
270 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
271 int rc = Device.associatePtr(host_ptr, device_addr, size);
272 DP("omp_target_associate_ptr returns %d\n", rc);
273 return rc;
274 }
275
omp_target_disassociate_ptr(void * host_ptr,int device_num)276 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
277 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
278 "device_num %d\n", DPxPTR(host_ptr), device_num);
279
280 if (!host_ptr) {
281 DP("Call to omp_target_associate_ptr with invalid host_ptr\n");
282 return OFFLOAD_FAIL;
283 }
284
285 if (device_num == omp_get_initial_device()) {
286 DP("omp_target_disassociate_ptr: no association possible on the host\n");
287 return OFFLOAD_FAIL;
288 }
289
290 if (!device_is_ready(device_num)) {
291 DP("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
292 return OFFLOAD_FAIL;
293 }
294
295 DeviceTy& Device = Devices[device_num];
296 int rc = Device.disassociatePtr(host_ptr);
297 DP("omp_target_disassociate_ptr returns %d\n", rc);
298 return rc;
299 }
300