1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of OpenMP API interface functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "omptarget.h" 15 #include "private.h" 16 #include "rtl.h" 17 18 #include <climits> 19 #include <cstdlib> 20 #include <cstring> 21 22 EXTERN int omp_get_num_devices(void) { 23 TIMESCOPE(); 24 PM->RTLsMtx.lock(); 25 size_t DevicesSize = PM->Devices.size(); 26 PM->RTLsMtx.unlock(); 27 28 DP("Call to omp_get_num_devices returning %zd\n", DevicesSize); 29 30 return DevicesSize; 31 } 32 33 EXTERN int omp_get_initial_device(void) { 34 TIMESCOPE(); 35 int hostDevice = omp_get_num_devices(); 36 DP("Call to omp_get_initial_device returning %d\n", hostDevice); 37 return hostDevice; 38 } 39 40 EXTERN void *omp_target_alloc(size_t size, int device_num) { 41 return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEFAULT, __func__); 42 } 43 44 EXTERN void *llvm_omp_target_alloc_device(size_t size, int device_num) { 45 return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEVICE, __func__); 46 } 47 48 EXTERN void *llvm_omp_target_alloc_host(size_t size, int device_num) { 49 return targetAllocExplicit(size, device_num, TARGET_ALLOC_HOST, __func__); 50 } 51 52 EXTERN void *llvm_omp_target_alloc_shared(size_t size, int device_num) { 53 return targetAllocExplicit(size, device_num, TARGET_ALLOC_SHARED, __func__); 54 } 55 56 EXTERN void omp_target_free(void *device_ptr, int device_num) { 57 TIMESCOPE(); 58 DP("Call to omp_target_free for device %d and address " DPxMOD "\n", 59 device_num, DPxPTR(device_ptr)); 60 61 if (!device_ptr) { 62 DP("Call to omp_target_free with NULL ptr\n"); 63 return; 64 } 65 66 if (device_num == omp_get_initial_device()) { 67 free(device_ptr); 68 DP("omp_target_free deallocated host ptr\n"); 69 return; 70 } 71 72 if (!device_is_ready(device_num)) { 73 DP("omp_target_free returns, nothing to do\n"); 74 return; 75 } 76 77 PM->Devices[device_num].deleteData(device_ptr); 78 DP("omp_target_free deallocated device ptr\n"); 79 } 80 81 EXTERN int omp_target_is_present(const void *ptr, int device_num) { 82 TIMESCOPE(); 83 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n", 84 device_num, DPxPTR(ptr)); 85 86 if (!ptr) { 87 DP("Call to omp_target_is_present with NULL ptr, returning false\n"); 88 return false; 89 } 90 91 if (device_num == omp_get_initial_device()) { 92 DP("Call to omp_target_is_present on host, returning true\n"); 93 return true; 94 } 95 96 PM->RTLsMtx.lock(); 97 size_t DevicesSize = PM->Devices.size(); 98 PM->RTLsMtx.unlock(); 99 if (DevicesSize <= (size_t)device_num) { 100 DP("Call to omp_target_is_present with invalid device ID, returning " 101 "false\n"); 102 return false; 103 } 104 105 DeviceTy &Device = PM->Devices[device_num]; 106 bool IsLast; // not used 107 bool IsHostPtr; 108 void *TgtPtr = Device.getTgtPtrBegin(const_cast<void *>(ptr), 0, IsLast, 109 false, IsHostPtr); 110 int rc = (TgtPtr != NULL); 111 // Under unified memory the host pointer can be returned by the 112 // getTgtPtrBegin() function which means that there is no device 113 // corresponding point for ptr. This function should return false 114 // in that situation. 115 if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) 116 rc = !IsHostPtr; 117 DP("Call to omp_target_is_present returns %d\n", rc); 118 return rc; 119 } 120 121 EXTERN int omp_target_memcpy(void *dst, const void *src, size_t length, 122 size_t dst_offset, size_t src_offset, 123 int dst_device, int src_device) { 124 TIMESCOPE(); 125 DP("Call to omp_target_memcpy, dst device %d, src device %d, " 126 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, " 127 "src offset %zu, length %zu\n", 128 dst_device, src_device, DPxPTR(dst), DPxPTR(src), dst_offset, src_offset, 129 length); 130 131 if (!dst || !src || length <= 0) { 132 if (length == 0) { 133 DP("Call to omp_target_memcpy with zero length, nothing to do\n"); 134 return OFFLOAD_SUCCESS; 135 } 136 137 REPORT("Call to omp_target_memcpy with invalid arguments\n"); parse<T: ParseMacroInput>(token_stream: TokenStream) -> Result<T>138 return OFFLOAD_FAIL; 139 } 140 141 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) { 142 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 143 return OFFLOAD_FAIL; 144 } parse(input: ParseStream) -> Result<Self>145 146 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) { 147 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 148 return OFFLOAD_FAIL; 149 } 150 151 int rc = OFFLOAD_SUCCESS; 152 void *srcAddr = (char *)const_cast<void *>(src) + src_offset; 153 void *dstAddr = (char *)dst + dst_offset; 154 155 if (src_device == omp_get_initial_device() && 156 dst_device == omp_get_initial_device()) { 157 DP("copy from host to host\n"); 158 const void *p = memcpy(dstAddr, srcAddr, length); 159 if (p == NULL) 160 rc = OFFLOAD_FAIL; 161 } else if (src_device == omp_get_initial_device()) { parse(input: ParseStream) -> Result<Self>162 DP("copy from host to device\n"); 163 DeviceTy &DstDev = PM->Devices[dst_device]; 164 AsyncInfoTy AsyncInfo(DstDev); 165 rc = DstDev.submitData(dstAddr, srcAddr, length, AsyncInfo); 166 } else if (dst_device == omp_get_initial_device()) { 167 DP("copy from device to host\n"); 168 DeviceTy &SrcDev = PM->Devices[src_device]; 169 AsyncInfoTy AsyncInfo(SrcDev); 170 rc = SrcDev.retrieveData(dstAddr, srcAddr, length, AsyncInfo); 171 } else { 172 DP("copy from device to device\n"); 173 DeviceTy &SrcDev = PM->Devices[src_device]; 174 DeviceTy &DstDev = PM->Devices[dst_device]; 175 // First try to use D2D memcpy which is more efficient. If fails, fall back 176 // to unefficient way. 177 if (SrcDev.isDataExchangable(DstDev)) { 178 AsyncInfoTy AsyncInfo(SrcDev); 179 rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, AsyncInfo); 180 if (rc == OFFLOAD_SUCCESS) 181 return OFFLOAD_SUCCESS; 182 } 183 184 void *buffer = malloc(length); 185 { 186 AsyncInfoTy AsyncInfo(SrcDev); 187 rc = SrcDev.retrieveData(buffer, srcAddr, length, AsyncInfo); 188 } 189 if (rc == OFFLOAD_SUCCESS) { 190 AsyncInfoTy AsyncInfo(SrcDev); 191 rc = DstDev.submitData(dstAddr, buffer, length, AsyncInfo); 192 } 193 free(buffer); 194 } 195 196 DP("omp_target_memcpy returns %d\n", rc); 197 return rc; 198 } 199 200 EXTERN int omp_target_memcpy_rect( 201 void *dst, const void *src, size_t element_size, int num_dims, 202 const size_t *volume, const size_t *dst_offsets, const size_t *src_offsets, 203 const size_t *dst_dimensions, const size_t *src_dimensions, int dst_device, 204 int src_device) { 205 TIMESCOPE(); 206 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, " 207 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", " 208 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", " 209 "volume " DPxMOD ", element size %zu, num_dims %d\n", 210 dst_device, src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets), 211 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions), 212 DPxPTR(volume), element_size, num_dims); 213 214 if (!(dst || src)) { 215 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n", 216 INT_MAX); 217 return INT_MAX; 218 } 219 220 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume || 221 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) { 222 REPORT("Call to omp_target_memcpy_rect with invalid arguments\n"); 223 return OFFLOAD_FAIL; 224 } 225 226 int rc; 227 if (num_dims == 1) { 228 rc = omp_target_memcpy( 229 dst, src, element_size * volume[0], element_size * dst_offsets[0], 230 element_size * src_offsets[0], dst_device, src_device); 231 } else { 232 size_t dst_slice_size = element_size; 233 size_t src_slice_size = element_size; 234 for (int i = 1; i < num_dims; ++i) { 235 dst_slice_size *= dst_dimensions[i]; 236 src_slice_size *= src_dimensions[i]; 237 } 238 239 size_t dst_off = dst_offsets[0] * dst_slice_size; 240 size_t src_off = src_offsets[0] * src_slice_size; 241 for (size_t i = 0; i < volume[0]; ++i) { 242 rc = omp_target_memcpy_rect( 243 (char *)dst + dst_off + dst_slice_size * i, 244 (char *)const_cast<void *>(src) + src_off + src_slice_size * i, 245 element_size, num_dims - 1, volume + 1, dst_offsets + 1, 246 src_offsets + 1, dst_dimensions + 1, src_dimensions + 1, dst_device, 247 src_device); 248 249 if (rc) { 250 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n"); 251 return rc; 252 } 253 } 254 } 255 256 DP("omp_target_memcpy_rect returns %d\n", rc); 257 return rc; 258 } 259 260 EXTERN int omp_target_associate_ptr(const void *host_ptr, 261 const void *device_ptr, size_t size, 262 size_t device_offset, int device_num) { 263 TIMESCOPE(); 264 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", " 265 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n", 266 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num); 267 268 if (!host_ptr || !device_ptr || size <= 0) { 269 REPORT("Call to omp_target_associate_ptr with invalid arguments\n"); 270 return OFFLOAD_FAIL; 271 } 272 273 if (device_num == omp_get_initial_device()) { 274 REPORT("omp_target_associate_ptr: no association possible on the host\n"); 275 return OFFLOAD_FAIL; 276 } 277 278 if (!device_is_ready(device_num)) { 279 REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n"); 280 return OFFLOAD_FAIL; 281 } 282 283 DeviceTy &Device = PM->Devices[device_num]; 284 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset); 285 int rc = Device.associatePtr(const_cast<void *>(host_ptr), 286 const_cast<void *>(device_addr), size); 287 DP("omp_target_associate_ptr returns %d\n", rc); 288 return rc; 289 } 290 291 EXTERN int omp_target_disassociate_ptr(const void *host_ptr, int device_num) { 292 TIMESCOPE(); 293 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", " 294 "device_num %d\n", 295 DPxPTR(host_ptr), device_num); 296 297 if (!host_ptr) { 298 REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n"); 299 return OFFLOAD_FAIL; 300 } 301 302 if (device_num == omp_get_initial_device()) { 303 REPORT( 304 "omp_target_disassociate_ptr: no association possible on the host\n"); 305 return OFFLOAD_FAIL; 306 } 307 308 if (!device_is_ready(device_num)) { 309 REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n"); 310 return OFFLOAD_FAIL; 311 } 312 313 DeviceTy &Device = PM->Devices[device_num]; 314 int rc = Device.disassociatePtr(const_cast<void *>(host_ptr)); 315 DP("omp_target_disassociate_ptr returns %d\n", rc); 316 return rc; 317 } 318