1 /* Copyright (C) 2013-2018 Free Software Foundation, Inc. 2 3 Contributed by Mentor Embedded. 4 5 This file is part of the GNU Offloading and Multi Processing Library 6 (libgomp). 7 8 Libgomp is free software; you can redistribute it and/or modify it 9 under the terms of the GNU General Public License as published by 10 the Free Software Foundation; either version 3, or (at your option) 11 any later version. 12 13 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 16 more details. 17 18 Under Section 7 of GPL version 3, you are granted additional 19 permissions described in the GCC Runtime Library Exception, version 20 3.1, as published by the Free Software Foundation. 21 22 You should have received a copy of the GNU General Public License and 23 a copy of the GCC Runtime Library Exception along with this program; 24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 25 <http://www.gnu.org/licenses/>. */ 26 27 /* This file handles OpenACC constructs. */ 28 29 #include "openacc.h" 30 #include "libgomp.h" 31 #include "libgomp_g.h" 32 #include "gomp-constants.h" 33 #include "oacc-int.h" 34 #ifdef HAVE_INTTYPES_H 35 # include <inttypes.h> /* For PRIu64. */ 36 #endif 37 #include <string.h> 38 #include <stdarg.h> 39 #include <assert.h> 40 41 static int 42 find_pset (int pos, size_t mapnum, unsigned short *kinds) 43 { 44 if (pos + 1 >= mapnum) 45 return 0; 46 47 unsigned char kind = kinds[pos+1] & 0xff; 48 49 return kind == GOMP_MAP_TO_PSET; 50 } 51 52 static void goacc_wait (int async, int num_waits, va_list *ap); 53 54 55 /* Launch a possibly offloaded function on DEVICE. FN is the host fn 56 address. MAPNUM, HOSTADDRS, SIZES & KINDS describe the memory 57 blocks to be copied to/from the device. Varadic arguments are 58 keyed optional parameters terminated with a zero. */ 59 60 void 61 GOACC_parallel_keyed (int device, void (*fn) (void *), 62 size_t mapnum, void **hostaddrs, size_t *sizes, 63 unsigned short *kinds, ...) 64 { 65 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 66 va_list ap; 67 struct goacc_thread *thr; 68 struct gomp_device_descr *acc_dev; 69 struct target_mem_desc *tgt; 70 void **devaddrs; 71 unsigned int i; 72 struct splay_tree_key_s k; 73 splay_tree_key tgt_fn_key; 74 void (*tgt_fn); 75 int async = GOMP_ASYNC_SYNC; 76 unsigned dims[GOMP_DIM_MAX]; 77 unsigned tag; 78 79 #ifdef HAVE_INTTYPES_H 80 gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n", 81 __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds); 82 #else 83 gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n", 84 __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds); 85 #endif 86 goacc_lazy_initialize (); 87 88 thr = goacc_thread (); 89 acc_dev = thr->dev; 90 91 /* Host fallback if "if" clause is false or if the current device is set to 92 the host. */ 93 if (host_fallback) 94 { 95 goacc_save_and_set_bind (acc_device_host); 96 fn (hostaddrs); 97 goacc_restore_bind (); 98 return; 99 } 100 else if (acc_device_type (acc_dev->type) == acc_device_host) 101 { 102 fn (hostaddrs); 103 return; 104 } 105 106 /* Default: let the runtime choose. */ 107 for (i = 0; i != GOMP_DIM_MAX; i++) 108 dims[i] = 0; 109 110 va_start (ap, kinds); 111 /* TODO: This will need amending when device_type is implemented. */ 112 while ((tag = va_arg (ap, unsigned)) != 0) 113 { 114 if (GOMP_LAUNCH_DEVICE (tag)) 115 gomp_fatal ("device_type '%d' offload parameters, libgomp is too old", 116 GOMP_LAUNCH_DEVICE (tag)); 117 118 switch (GOMP_LAUNCH_CODE (tag)) 119 { 120 case GOMP_LAUNCH_DIM: 121 { 122 unsigned mask = GOMP_LAUNCH_OP (tag); 123 124 for (i = 0; i != GOMP_DIM_MAX; i++) 125 if (mask & GOMP_DIM_MASK (i)) 126 dims[i] = va_arg (ap, unsigned); 127 } 128 break; 129 130 case GOMP_LAUNCH_ASYNC: 131 { 132 /* Small constant values are encoded in the operand. */ 133 async = GOMP_LAUNCH_OP (tag); 134 135 if (async == GOMP_LAUNCH_OP_MAX) 136 async = va_arg (ap, unsigned); 137 break; 138 } 139 140 case GOMP_LAUNCH_WAIT: 141 { 142 unsigned num_waits = GOMP_LAUNCH_OP (tag); 143 144 if (num_waits) 145 goacc_wait (async, num_waits, &ap); 146 break; 147 } 148 149 default: 150 gomp_fatal ("unrecognized offload code '%d'," 151 " libgomp is too old", GOMP_LAUNCH_CODE (tag)); 152 } 153 } 154 va_end (ap); 155 156 acc_dev->openacc.async_set_async_func (async); 157 158 if (!(acc_dev->capabilities & GOMP_OFFLOAD_CAP_NATIVE_EXEC)) 159 { 160 k.host_start = (uintptr_t) fn; 161 k.host_end = k.host_start + 1; 162 gomp_mutex_lock (&acc_dev->lock); 163 tgt_fn_key = splay_tree_lookup (&acc_dev->mem_map, &k); 164 gomp_mutex_unlock (&acc_dev->lock); 165 166 if (tgt_fn_key == NULL) 167 gomp_fatal ("target function wasn't mapped"); 168 169 tgt_fn = (void (*)) tgt_fn_key->tgt_offset; 170 } 171 else 172 tgt_fn = (void (*)) fn; 173 174 tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true, 175 GOMP_MAP_VARS_OPENACC); 176 177 devaddrs = gomp_alloca (sizeof (void *) * mapnum); 178 for (i = 0; i < mapnum; i++) 179 devaddrs[i] = (void *) (tgt->list[i].key->tgt->tgt_start 180 + tgt->list[i].key->tgt_offset); 181 182 acc_dev->openacc.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs, 183 async, dims, tgt); 184 185 /* If running synchronously, unmap immediately. */ 186 if (async < acc_async_noval) 187 gomp_unmap_vars (tgt, true); 188 else 189 tgt->device_descr->openacc.register_async_cleanup_func (tgt, async); 190 191 acc_dev->openacc.async_set_async_func (acc_async_sync); 192 } 193 194 /* Legacy entry point, only provide host execution. */ 195 196 void 197 GOACC_parallel (int device, void (*fn) (void *), 198 size_t mapnum, void **hostaddrs, size_t *sizes, 199 unsigned short *kinds, 200 int num_gangs, int num_workers, int vector_length, 201 int async, int num_waits, ...) 202 { 203 goacc_save_and_set_bind (acc_device_host); 204 fn (hostaddrs); 205 goacc_restore_bind (); 206 } 207 208 void 209 GOACC_data_start (int device, size_t mapnum, 210 void **hostaddrs, size_t *sizes, unsigned short *kinds) 211 { 212 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 213 struct target_mem_desc *tgt; 214 215 #ifdef HAVE_INTTYPES_H 216 gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n", 217 __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds); 218 #else 219 gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n", 220 __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds); 221 #endif 222 223 goacc_lazy_initialize (); 224 225 struct goacc_thread *thr = goacc_thread (); 226 struct gomp_device_descr *acc_dev = thr->dev; 227 228 /* Host fallback or 'do nothing'. */ 229 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) 230 || host_fallback) 231 { 232 tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true, 233 GOMP_MAP_VARS_OPENACC); 234 tgt->prev = thr->mapped_data; 235 thr->mapped_data = tgt; 236 237 return; 238 } 239 240 gomp_debug (0, " %s: prepare mappings\n", __FUNCTION__); 241 tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true, 242 GOMP_MAP_VARS_OPENACC); 243 gomp_debug (0, " %s: mappings prepared\n", __FUNCTION__); 244 tgt->prev = thr->mapped_data; 245 thr->mapped_data = tgt; 246 } 247 248 void 249 GOACC_data_end (void) 250 { 251 struct goacc_thread *thr = goacc_thread (); 252 struct target_mem_desc *tgt = thr->mapped_data; 253 254 gomp_debug (0, " %s: restore mappings\n", __FUNCTION__); 255 thr->mapped_data = tgt->prev; 256 gomp_unmap_vars (tgt, true); 257 gomp_debug (0, " %s: mappings restored\n", __FUNCTION__); 258 } 259 260 void 261 GOACC_enter_exit_data (int device, size_t mapnum, 262 void **hostaddrs, size_t *sizes, unsigned short *kinds, 263 int async, int num_waits, ...) 264 { 265 struct goacc_thread *thr; 266 struct gomp_device_descr *acc_dev; 267 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 268 bool data_enter = false; 269 size_t i; 270 271 goacc_lazy_initialize (); 272 273 thr = goacc_thread (); 274 acc_dev = thr->dev; 275 276 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) 277 || host_fallback) 278 return; 279 280 if (num_waits) 281 { 282 va_list ap; 283 284 va_start (ap, num_waits); 285 goacc_wait (async, num_waits, &ap); 286 va_end (ap); 287 } 288 289 acc_dev->openacc.async_set_async_func (async); 290 291 /* Determine if this is an "acc enter data". */ 292 for (i = 0; i < mapnum; ++i) 293 { 294 unsigned char kind = kinds[i] & 0xff; 295 296 if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET) 297 continue; 298 299 if (kind == GOMP_MAP_FORCE_ALLOC 300 || kind == GOMP_MAP_FORCE_PRESENT 301 || kind == GOMP_MAP_FORCE_TO) 302 { 303 data_enter = true; 304 break; 305 } 306 307 if (kind == GOMP_MAP_DELETE 308 || kind == GOMP_MAP_FORCE_FROM) 309 break; 310 311 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x", 312 kind); 313 } 314 315 if (data_enter) 316 { 317 for (i = 0; i < mapnum; i++) 318 { 319 unsigned char kind = kinds[i] & 0xff; 320 321 /* Scan for PSETs. */ 322 int psets = find_pset (i, mapnum, kinds); 323 324 if (!psets) 325 { 326 switch (kind) 327 { 328 case GOMP_MAP_POINTER: 329 gomp_acc_insert_pointer (1, &hostaddrs[i], &sizes[i], 330 &kinds[i]); 331 break; 332 case GOMP_MAP_FORCE_ALLOC: 333 acc_create (hostaddrs[i], sizes[i]); 334 break; 335 case GOMP_MAP_FORCE_PRESENT: 336 acc_present_or_copyin (hostaddrs[i], sizes[i]); 337 break; 338 case GOMP_MAP_FORCE_TO: 339 acc_present_or_copyin (hostaddrs[i], sizes[i]); 340 break; 341 default: 342 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x", 343 kind); 344 break; 345 } 346 } 347 else 348 { 349 gomp_acc_insert_pointer (3, &hostaddrs[i], &sizes[i], &kinds[i]); 350 /* Increment 'i' by two because OpenACC requires fortran 351 arrays to be contiguous, so each PSET is associated with 352 one of MAP_FORCE_ALLOC/MAP_FORCE_PRESET/MAP_FORCE_TO, and 353 one MAP_POINTER. */ 354 i += 2; 355 } 356 } 357 } 358 else 359 for (i = 0; i < mapnum; ++i) 360 { 361 unsigned char kind = kinds[i] & 0xff; 362 363 int psets = find_pset (i, mapnum, kinds); 364 365 if (!psets) 366 { 367 switch (kind) 368 { 369 case GOMP_MAP_POINTER: 370 gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff) 371 == GOMP_MAP_FORCE_FROM, 372 async, 1); 373 break; 374 case GOMP_MAP_DELETE: 375 acc_delete (hostaddrs[i], sizes[i]); 376 break; 377 case GOMP_MAP_FORCE_FROM: 378 acc_copyout (hostaddrs[i], sizes[i]); 379 break; 380 default: 381 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x", 382 kind); 383 break; 384 } 385 } 386 else 387 { 388 gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff) 389 == GOMP_MAP_FORCE_FROM, async, 3); 390 /* See the above comment. */ 391 i += 2; 392 } 393 } 394 395 acc_dev->openacc.async_set_async_func (acc_async_sync); 396 } 397 398 static void 399 goacc_wait (int async, int num_waits, va_list *ap) 400 { 401 struct goacc_thread *thr = goacc_thread (); 402 struct gomp_device_descr *acc_dev = thr->dev; 403 404 while (num_waits--) 405 { 406 int qid = va_arg (*ap, int); 407 408 if (acc_async_test (qid)) 409 continue; 410 411 if (async == acc_async_sync) 412 acc_wait (qid); 413 else if (qid == async) 414 ;/* If we're waiting on the same asynchronous queue as we're 415 launching on, the queue itself will order work as 416 required, so there's no need to wait explicitly. */ 417 else 418 acc_dev->openacc.async_wait_async_func (qid, async); 419 } 420 } 421 422 void 423 GOACC_update (int device, size_t mapnum, 424 void **hostaddrs, size_t *sizes, unsigned short *kinds, 425 int async, int num_waits, ...) 426 { 427 bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK; 428 size_t i; 429 430 goacc_lazy_initialize (); 431 432 struct goacc_thread *thr = goacc_thread (); 433 struct gomp_device_descr *acc_dev = thr->dev; 434 435 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) 436 || host_fallback) 437 return; 438 439 if (num_waits) 440 { 441 va_list ap; 442 443 va_start (ap, num_waits); 444 goacc_wait (async, num_waits, &ap); 445 va_end (ap); 446 } 447 448 acc_dev->openacc.async_set_async_func (async); 449 450 for (i = 0; i < mapnum; ++i) 451 { 452 unsigned char kind = kinds[i] & 0xff; 453 454 switch (kind) 455 { 456 case GOMP_MAP_POINTER: 457 case GOMP_MAP_TO_PSET: 458 break; 459 460 case GOMP_MAP_FORCE_TO: 461 acc_update_device (hostaddrs[i], sizes[i]); 462 break; 463 464 case GOMP_MAP_FORCE_FROM: 465 acc_update_self (hostaddrs[i], sizes[i]); 466 break; 467 468 default: 469 gomp_fatal (">>>> GOACC_update UNHANDLED kind 0x%.2x", kind); 470 break; 471 } 472 } 473 474 acc_dev->openacc.async_set_async_func (acc_async_sync); 475 } 476 477 void 478 GOACC_wait (int async, int num_waits, ...) 479 { 480 if (num_waits) 481 { 482 va_list ap; 483 484 va_start (ap, num_waits); 485 goacc_wait (async, num_waits, &ap); 486 va_end (ap); 487 } 488 else if (async == acc_async_sync) 489 acc_wait_all (); 490 else if (async == acc_async_noval) 491 goacc_thread ()->dev->openacc.async_wait_all_async_func (acc_async_noval); 492 } 493 494 int 495 GOACC_get_num_threads (void) 496 { 497 return 1; 498 } 499 500 int 501 GOACC_get_thread_num (void) 502 { 503 return 0; 504 } 505 506 void 507 GOACC_declare (int device, size_t mapnum, 508 void **hostaddrs, size_t *sizes, unsigned short *kinds) 509 { 510 int i; 511 512 for (i = 0; i < mapnum; i++) 513 { 514 unsigned char kind = kinds[i] & 0xff; 515 516 if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET) 517 continue; 518 519 switch (kind) 520 { 521 case GOMP_MAP_FORCE_ALLOC: 522 case GOMP_MAP_FORCE_FROM: 523 case GOMP_MAP_FORCE_TO: 524 case GOMP_MAP_POINTER: 525 case GOMP_MAP_DELETE: 526 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 527 &kinds[i], 0, 0); 528 break; 529 530 case GOMP_MAP_FORCE_DEVICEPTR: 531 break; 532 533 case GOMP_MAP_ALLOC: 534 if (!acc_is_present (hostaddrs[i], sizes[i])) 535 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 536 &kinds[i], 0, 0); 537 break; 538 539 case GOMP_MAP_TO: 540 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 541 &kinds[i], 0, 0); 542 543 break; 544 545 case GOMP_MAP_FROM: 546 kinds[i] = GOMP_MAP_FORCE_FROM; 547 GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i], 548 &kinds[i], 0, 0); 549 break; 550 551 case GOMP_MAP_FORCE_PRESENT: 552 if (!acc_is_present (hostaddrs[i], sizes[i])) 553 gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i], 554 (unsigned long) sizes[i]); 555 break; 556 557 default: 558 assert (0); 559 break; 560 } 561 } 562 } 563