1 /* Copyright (C) 2005-2018 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Offloading and Multi Processing Library 5 (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This file contains data types and function declarations that are not 27 part of the official OpenACC or OpenMP user interfaces. There are 28 declarations in here that are part of the GNU Offloading and Multi 29 Processing ABI, in that the compiler is required to know about them 30 and use them. 31 32 The convention is that the all caps prefix "GOMP" is used group items 33 that are part of the external ABI, and the lower case prefix "gomp" 34 is used group items that are completely private to the library. */ 35 36 #ifndef LIBGOMP_H 37 #define LIBGOMP_H 1 38 39 #ifndef _LIBGOMP_CHECKING_ 40 /* Define to 1 to perform internal sanity checks. */ 41 #define _LIBGOMP_CHECKING_ 0 42 #endif 43 44 #include "config.h" 45 #include "gstdint.h" 46 #include "libgomp-plugin.h" 47 48 #ifdef HAVE_PTHREAD_H 49 #include <pthread.h> 50 #endif 51 #include <stdbool.h> 52 #include <stdlib.h> 53 #include <stdarg.h> 54 55 /* Needed for memset in priority_queue.c. */ 56 #if _LIBGOMP_CHECKING_ 57 # ifdef STRING_WITH_STRINGS 58 # include <string.h> 59 # include <strings.h> 60 # else 61 # ifdef HAVE_STRING_H 62 # include <string.h> 63 # else 64 # ifdef HAVE_STRINGS_H 65 # include <strings.h> 66 # endif 67 # endif 68 # endif 69 #endif 70 71 #ifdef HAVE_ATTRIBUTE_VISIBILITY 72 # pragma GCC visibility push(hidden) 73 #endif 74 75 /* If we were a C++ library, we'd get this from <std/atomic>. */ 76 enum memmodel 77 { 78 MEMMODEL_RELAXED = 0, 79 MEMMODEL_CONSUME = 1, 80 MEMMODEL_ACQUIRE = 2, 81 MEMMODEL_RELEASE = 3, 82 MEMMODEL_ACQ_REL = 4, 83 MEMMODEL_SEQ_CST = 5 84 }; 85 86 /* alloc.c */ 87 88 extern void *gomp_malloc (size_t) __attribute__((malloc)); 89 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc)); 90 extern void *gomp_realloc (void *, size_t); 91 92 /* Avoid conflicting prototypes of alloca() in system headers by using 93 GCC's builtin alloca(). */ 94 #define gomp_alloca(x) __builtin_alloca(x) 95 96 /* error.c */ 97 98 extern void gomp_vdebug (int, const char *, va_list); 99 extern void gomp_debug (int, const char *, ...) 100 __attribute__ ((format (printf, 2, 3))); 101 #define gomp_vdebug(KIND, FMT, VALIST) \ 102 do { \ 103 if (__builtin_expect (gomp_debug_var, 0)) \ 104 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \ 105 } while (0) 106 #define gomp_debug(KIND, ...) \ 107 do { \ 108 if (__builtin_expect (gomp_debug_var, 0)) \ 109 (gomp_debug) ((KIND), __VA_ARGS__); \ 110 } while (0) 111 extern void gomp_verror (const char *, va_list); 112 extern void gomp_error (const char *, ...) 113 __attribute__ ((format (printf, 1, 2))); 114 extern void gomp_vfatal (const char *, va_list) 115 __attribute__ ((noreturn)); 116 extern void gomp_fatal (const char *, ...) 117 __attribute__ ((noreturn, format (printf, 1, 2))); 118 119 struct gomp_task; 120 struct gomp_taskgroup; 121 struct htab; 122 123 #include "priority_queue.h" 124 #include "sem.h" 125 #include "mutex.h" 126 #include "bar.h" 127 #include "simple-bar.h" 128 #include "ptrlock.h" 129 130 131 /* This structure contains the data to control one work-sharing construct, 132 either a LOOP (FOR/DO) or a SECTIONS. */ 133 134 enum gomp_schedule_type 135 { 136 GFS_RUNTIME, 137 GFS_STATIC, 138 GFS_DYNAMIC, 139 GFS_GUIDED, 140 GFS_AUTO 141 }; 142 143 struct gomp_doacross_work_share 144 { 145 union { 146 /* chunk_size copy, as ws->chunk_size is multiplied by incr for 147 GFS_DYNAMIC. */ 148 long chunk_size; 149 /* Likewise, but for ull implementation. */ 150 unsigned long long chunk_size_ull; 151 /* For schedule(static,0) this is the number 152 of iterations assigned to the last thread, i.e. number of 153 iterations / number of threads. */ 154 long q; 155 /* Likewise, but for ull implementation. */ 156 unsigned long long q_ull; 157 }; 158 /* Size of each array entry (padded to cache line size). */ 159 unsigned long elt_sz; 160 /* Number of dimensions in sink vectors. */ 161 unsigned int ncounts; 162 /* True if the iterations can be flattened. */ 163 bool flattened; 164 /* Actual array (of elt_sz sized units), aligned to cache line size. 165 This is indexed by team_id for GFS_STATIC and outermost iteration 166 / chunk_size for other schedules. */ 167 unsigned char *array; 168 /* These two are only used for schedule(static,0). */ 169 /* This one is number of iterations % number of threads. */ 170 long t; 171 union { 172 /* And this one is cached t * (q + 1). */ 173 long boundary; 174 /* Likewise, but for the ull implementation. */ 175 unsigned long long boundary_ull; 176 }; 177 /* Array of shift counts for each dimension if they can be flattened. */ 178 unsigned int shift_counts[]; 179 }; 180 181 struct gomp_work_share 182 { 183 /* This member records the SCHEDULE clause to be used for this construct. 184 The user specification of "runtime" will already have been resolved. 185 If this is a SECTIONS construct, this value will always be DYNAMIC. */ 186 enum gomp_schedule_type sched; 187 188 int mode; 189 190 union { 191 struct { 192 /* This is the chunk_size argument to the SCHEDULE clause. */ 193 long chunk_size; 194 195 /* This is the iteration end point. If this is a SECTIONS construct, 196 this is the number of contained sections. */ 197 long end; 198 199 /* This is the iteration step. If this is a SECTIONS construct, this 200 is always 1. */ 201 long incr; 202 }; 203 204 struct { 205 /* The same as above, but for the unsigned long long loop variants. */ 206 unsigned long long chunk_size_ull; 207 unsigned long long end_ull; 208 unsigned long long incr_ull; 209 }; 210 }; 211 212 union { 213 /* This is a circular queue that details which threads will be allowed 214 into the ordered region and in which order. When a thread allocates 215 iterations on which it is going to work, it also registers itself at 216 the end of the array. When a thread reaches the ordered region, it 217 checks to see if it is the one at the head of the queue. If not, it 218 blocks on its RELEASE semaphore. */ 219 unsigned *ordered_team_ids; 220 221 /* This is a pointer to DOACROSS work share data. */ 222 struct gomp_doacross_work_share *doacross; 223 }; 224 225 /* This is the number of threads that have registered themselves in 226 the circular queue ordered_team_ids. */ 227 unsigned ordered_num_used; 228 229 /* This is the team_id of the currently acknowledged owner of the ordered 230 section, or -1u if the ordered section has not been acknowledged by 231 any thread. This is distinguished from the thread that is *allowed* 232 to take the section next. */ 233 unsigned ordered_owner; 234 235 /* This is the index into the circular queue ordered_team_ids of the 236 current thread that's allowed into the ordered reason. */ 237 unsigned ordered_cur; 238 239 /* This is a chain of allocated gomp_work_share blocks, valid only 240 in the first gomp_work_share struct in the block. */ 241 struct gomp_work_share *next_alloc; 242 243 /* The above fields are written once during workshare initialization, 244 or related to ordered worksharing. Make sure the following fields 245 are in a different cache line. */ 246 247 /* This lock protects the update of the following members. */ 248 gomp_mutex_t lock __attribute__((aligned (64))); 249 250 /* This is the count of the number of threads that have exited the work 251 share construct. If the construct was marked nowait, they have moved on 252 to other work; otherwise they're blocked on a barrier. The last member 253 of the team to exit the work share construct must deallocate it. */ 254 unsigned threads_completed; 255 256 union { 257 /* This is the next iteration value to be allocated. In the case of 258 GFS_STATIC loops, this the iteration start point and never changes. */ 259 long next; 260 261 /* The same, but with unsigned long long type. */ 262 unsigned long long next_ull; 263 264 /* This is the returned data structure for SINGLE COPYPRIVATE. */ 265 void *copyprivate; 266 }; 267 268 union { 269 /* Link to gomp_work_share struct for next work sharing construct 270 encountered after this one. */ 271 gomp_ptrlock_t next_ws; 272 273 /* gomp_work_share structs are chained in the free work share cache 274 through this. */ 275 struct gomp_work_share *next_free; 276 }; 277 278 /* If only few threads are in the team, ordered_team_ids can point 279 to this array which fills the padding at the end of this struct. */ 280 unsigned inline_ordered_team_ids[0]; 281 }; 282 283 /* This structure contains all of the thread-local data associated with 284 a thread team. This is the data that must be saved when a thread 285 encounters a nested PARALLEL construct. */ 286 287 struct gomp_team_state 288 { 289 /* This is the team of which the thread is currently a member. */ 290 struct gomp_team *team; 291 292 /* This is the work share construct which this thread is currently 293 processing. Recall that with NOWAIT, not all threads may be 294 processing the same construct. */ 295 struct gomp_work_share *work_share; 296 297 /* This is the previous work share construct or NULL if there wasn't any. 298 When all threads are done with the current work sharing construct, 299 the previous one can be freed. The current one can't, as its 300 next_ws field is used. */ 301 struct gomp_work_share *last_work_share; 302 303 /* This is the ID of this thread within the team. This value is 304 guaranteed to be between 0 and N-1, where N is the number of 305 threads in the team. */ 306 unsigned team_id; 307 308 /* Nesting level. */ 309 unsigned level; 310 311 /* Active nesting level. Only active parallel regions are counted. */ 312 unsigned active_level; 313 314 /* Place-partition-var, offset and length into gomp_places_list array. */ 315 unsigned place_partition_off; 316 unsigned place_partition_len; 317 318 #ifdef HAVE_SYNC_BUILTINS 319 /* Number of single stmts encountered. */ 320 unsigned long single_count; 321 #endif 322 323 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the 324 trip number through the loop. So first time a particular loop 325 is encountered this number is 0, the second time through the loop 326 is 1, etc. This is unused when the compiler knows in advance that 327 the loop is statically scheduled. */ 328 unsigned long static_trip; 329 }; 330 331 struct target_mem_desc; 332 333 /* These are the OpenMP 4.0 Internal Control Variables described in 334 section 2.3.1. Those described as having one copy per task are 335 stored within the structure; those described as having one copy 336 for the whole program are (naturally) global variables. */ 337 338 struct gomp_task_icv 339 { 340 unsigned long nthreads_var; 341 enum gomp_schedule_type run_sched_var; 342 int run_sched_chunk_size; 343 int default_device_var; 344 unsigned int thread_limit_var; 345 bool dyn_var; 346 bool nest_var; 347 char bind_var; 348 /* Internal ICV. */ 349 struct target_mem_desc *target_data; 350 }; 351 352 extern struct gomp_task_icv gomp_global_icv; 353 #ifndef HAVE_SYNC_BUILTINS 354 extern gomp_mutex_t gomp_managed_threads_lock; 355 #endif 356 extern unsigned long gomp_max_active_levels_var; 357 extern bool gomp_cancel_var; 358 extern int gomp_max_task_priority_var; 359 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var; 360 extern unsigned long gomp_available_cpus, gomp_managed_threads; 361 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len; 362 extern char *gomp_bind_var_list; 363 extern unsigned long gomp_bind_var_list_len; 364 extern void **gomp_places_list; 365 extern unsigned long gomp_places_list_len; 366 extern unsigned int gomp_num_teams_var; 367 extern int gomp_debug_var; 368 extern int goacc_device_num; 369 extern char *goacc_device_type; 370 371 enum gomp_task_kind 372 { 373 /* Implicit task. */ 374 GOMP_TASK_IMPLICIT, 375 /* Undeferred task. */ 376 GOMP_TASK_UNDEFERRED, 377 /* Task created by GOMP_task and waiting to be run. */ 378 GOMP_TASK_WAITING, 379 /* Task currently executing or scheduled and about to execute. */ 380 GOMP_TASK_TIED, 381 /* Used for target tasks that have vars mapped and async run started, 382 but not yet completed. Once that completes, they will be readded 383 into the queues as GOMP_TASK_WAITING in order to perform the var 384 unmapping. */ 385 GOMP_TASK_ASYNC_RUNNING 386 }; 387 388 struct gomp_task_depend_entry 389 { 390 /* Address of dependency. */ 391 void *addr; 392 struct gomp_task_depend_entry *next; 393 struct gomp_task_depend_entry *prev; 394 /* Task that provides the dependency in ADDR. */ 395 struct gomp_task *task; 396 /* Depend entry is of type "IN". */ 397 bool is_in; 398 bool redundant; 399 bool redundant_out; 400 }; 401 402 struct gomp_dependers_vec 403 { 404 size_t n_elem; 405 size_t allocated; 406 struct gomp_task *elem[]; 407 }; 408 409 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */ 410 411 struct gomp_taskwait 412 { 413 bool in_taskwait; 414 bool in_depend_wait; 415 /* Number of tasks we are waiting for. */ 416 size_t n_depend; 417 gomp_sem_t taskwait_sem; 418 }; 419 420 /* This structure describes a "task" to be run by a thread. */ 421 422 struct gomp_task 423 { 424 /* Parent of this task. */ 425 struct gomp_task *parent; 426 /* Children of this task. */ 427 struct priority_queue children_queue; 428 /* Taskgroup this task belongs in. */ 429 struct gomp_taskgroup *taskgroup; 430 /* Tasks that depend on this task. */ 431 struct gomp_dependers_vec *dependers; 432 struct htab *depend_hash; 433 struct gomp_taskwait *taskwait; 434 /* Number of items in DEPEND. */ 435 size_t depend_count; 436 /* Number of tasks this task depends on. Once this counter reaches 437 0, we have no unsatisfied dependencies, and this task can be put 438 into the various queues to be scheduled. */ 439 size_t num_dependees; 440 441 /* Priority of this task. */ 442 int priority; 443 /* The priority node for this task in each of the different queues. 444 We put this here to avoid allocating space for each priority 445 node. Then we play offsetof() games to convert between pnode[] 446 entries and the gomp_task in which they reside. */ 447 struct priority_node pnode[3]; 448 449 struct gomp_task_icv icv; 450 void (*fn) (void *); 451 void *fn_data; 452 enum gomp_task_kind kind; 453 bool in_tied_task; 454 bool final_task; 455 bool copy_ctors_done; 456 /* Set for undeferred tasks with unsatisfied dependencies which 457 block further execution of their parent until the dependencies 458 are satisfied. */ 459 bool parent_depends_on; 460 /* Dependencies provided and/or needed for this task. DEPEND_COUNT 461 is the number of items available. */ 462 struct gomp_task_depend_entry depend[]; 463 }; 464 465 /* This structure describes a single #pragma omp taskgroup. */ 466 467 struct gomp_taskgroup 468 { 469 struct gomp_taskgroup *prev; 470 /* Queue of tasks that belong in this taskgroup. */ 471 struct priority_queue taskgroup_queue; 472 bool in_taskgroup_wait; 473 bool cancelled; 474 gomp_sem_t taskgroup_sem; 475 size_t num_children; 476 }; 477 478 /* Various state of OpenMP async offloading tasks. */ 479 enum gomp_target_task_state 480 { 481 GOMP_TARGET_TASK_DATA, 482 GOMP_TARGET_TASK_BEFORE_MAP, 483 GOMP_TARGET_TASK_FALLBACK, 484 GOMP_TARGET_TASK_READY_TO_RUN, 485 GOMP_TARGET_TASK_RUNNING, 486 GOMP_TARGET_TASK_FINISHED 487 }; 488 489 /* This structure describes a target task. */ 490 491 struct gomp_target_task 492 { 493 struct gomp_device_descr *devicep; 494 void (*fn) (void *); 495 size_t mapnum; 496 size_t *sizes; 497 unsigned short *kinds; 498 unsigned int flags; 499 enum gomp_target_task_state state; 500 struct target_mem_desc *tgt; 501 struct gomp_task *task; 502 struct gomp_team *team; 503 /* Device-specific target arguments. */ 504 void **args; 505 void *hostaddrs[]; 506 }; 507 508 /* This structure describes a "team" of threads. These are the threads 509 that are spawned by a PARALLEL constructs, as well as the work sharing 510 constructs that the team encounters. */ 511 512 struct gomp_team 513 { 514 /* This is the number of threads in the current team. */ 515 unsigned nthreads; 516 517 /* This is number of gomp_work_share structs that have been allocated 518 as a block last time. */ 519 unsigned work_share_chunk; 520 521 /* This is the saved team state that applied to a master thread before 522 the current thread was created. */ 523 struct gomp_team_state prev_ts; 524 525 /* This semaphore should be used by the master thread instead of its 526 "native" semaphore in the thread structure. Required for nested 527 parallels, as the master is a member of two teams. */ 528 gomp_sem_t master_release; 529 530 /* This points to an array with pointers to the release semaphore 531 of the threads in the team. */ 532 gomp_sem_t **ordered_release; 533 534 /* List of work shares on which gomp_fini_work_share hasn't been 535 called yet. If the team hasn't been cancelled, this should be 536 equal to each thr->ts.work_share, but otherwise it can be a possibly 537 long list of workshares. */ 538 struct gomp_work_share *work_shares_to_free; 539 540 /* List of gomp_work_share structs chained through next_free fields. 541 This is populated and taken off only by the first thread in the 542 team encountering a new work sharing construct, in a critical 543 section. */ 544 struct gomp_work_share *work_share_list_alloc; 545 546 /* List of gomp_work_share structs freed by free_work_share. New 547 entries are atomically added to the start of the list, and 548 alloc_work_share can safely only move all but the first entry 549 to work_share_list alloc, as free_work_share can happen concurrently 550 with alloc_work_share. */ 551 struct gomp_work_share *work_share_list_free; 552 553 #ifdef HAVE_SYNC_BUILTINS 554 /* Number of simple single regions encountered by threads in this 555 team. */ 556 unsigned long single_count; 557 #else 558 /* Mutex protecting addition of workshares to work_share_list_free. */ 559 gomp_mutex_t work_share_list_free_lock; 560 #endif 561 562 /* This barrier is used for most synchronization of the team. */ 563 gomp_barrier_t barrier; 564 565 /* Initial work shares, to avoid allocating any gomp_work_share 566 structs in the common case. */ 567 struct gomp_work_share work_shares[8]; 568 569 gomp_mutex_t task_lock; 570 /* Scheduled tasks. */ 571 struct priority_queue task_queue; 572 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */ 573 unsigned int task_count; 574 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */ 575 unsigned int task_queued_count; 576 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running 577 directly in gomp_barrier_handle_tasks; tasks spawned 578 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when 579 that is called from a task run from gomp_barrier_handle_tasks. 580 task_running_count should be always <= team->nthreads, 581 and if current task isn't in_tied_task, then it will be 582 even < team->nthreads. */ 583 unsigned int task_running_count; 584 int work_share_cancelled; 585 int team_cancelled; 586 587 /* This array contains structures for implicit tasks. */ 588 struct gomp_task implicit_task[]; 589 }; 590 591 /* This structure contains all data that is private to libgomp and is 592 allocated per thread. */ 593 594 struct gomp_thread 595 { 596 /* This is the function that the thread should run upon launch. */ 597 void (*fn) (void *data); 598 void *data; 599 600 /* This is the current team state for this thread. The ts.team member 601 is NULL only if the thread is idle. */ 602 struct gomp_team_state ts; 603 604 /* This is the task that the thread is currently executing. */ 605 struct gomp_task *task; 606 607 /* This semaphore is used for ordered loops. */ 608 gomp_sem_t release; 609 610 /* Place this thread is bound to plus one, or zero if not bound 611 to any place. */ 612 unsigned int place; 613 614 /* User pthread thread pool */ 615 struct gomp_thread_pool *thread_pool; 616 }; 617 618 619 struct gomp_thread_pool 620 { 621 /* This array manages threads spawned from the top level, which will 622 return to the idle loop once the current PARALLEL construct ends. */ 623 struct gomp_thread **threads; 624 unsigned threads_size; 625 unsigned threads_used; 626 /* The last team is used for non-nested teams to delay their destruction to 627 make sure all the threads in the team move on to the pool's barrier before 628 the team's barrier is destroyed. */ 629 struct gomp_team *last_team; 630 /* Number of threads running in this contention group. */ 631 unsigned long threads_busy; 632 633 /* This barrier holds and releases threads waiting in thread pools. */ 634 gomp_simple_barrier_t threads_dock; 635 }; 636 637 enum gomp_cancel_kind 638 { 639 GOMP_CANCEL_PARALLEL = 1, 640 GOMP_CANCEL_LOOP = 2, 641 GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP, 642 GOMP_CANCEL_DO = GOMP_CANCEL_LOOP, 643 GOMP_CANCEL_SECTIONS = 4, 644 GOMP_CANCEL_TASKGROUP = 8 645 }; 646 647 /* ... and here is that TLS data. */ 648 649 #if defined __nvptx__ 650 extern struct gomp_thread *nvptx_thrs __attribute__((shared)); 651 static inline struct gomp_thread *gomp_thread (void) 652 { 653 int tid; 654 asm ("mov.u32 %0, %%tid.y;" : "=r" (tid)); 655 return nvptx_thrs + tid; 656 } 657 #elif defined HAVE_TLS || defined USE_EMUTLS 658 extern __thread struct gomp_thread gomp_tls_data; 659 static inline struct gomp_thread *gomp_thread (void) 660 { 661 return &gomp_tls_data; 662 } 663 #else 664 extern pthread_key_t gomp_tls_key; 665 static inline struct gomp_thread *gomp_thread (void) 666 { 667 return pthread_getspecific (gomp_tls_key); 668 } 669 #endif 670 671 extern struct gomp_task_icv *gomp_new_icv (void); 672 673 /* Here's how to access the current copy of the ICVs. */ 674 675 static inline struct gomp_task_icv *gomp_icv (bool write) 676 { 677 struct gomp_task *task = gomp_thread ()->task; 678 if (task) 679 return &task->icv; 680 else if (write) 681 return gomp_new_icv (); 682 else 683 return &gomp_global_icv; 684 } 685 686 #ifdef LIBGOMP_USE_PTHREADS 687 /* The attributes to be used during thread creation. */ 688 extern pthread_attr_t gomp_thread_attr; 689 690 extern pthread_key_t gomp_thread_destructor; 691 #endif 692 693 /* Function prototypes. */ 694 695 /* affinity.c */ 696 697 extern void gomp_init_affinity (void); 698 #ifdef LIBGOMP_USE_PTHREADS 699 extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int); 700 #endif 701 extern void **gomp_affinity_alloc (unsigned long, bool); 702 extern void gomp_affinity_init_place (void *); 703 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long, 704 long, bool); 705 extern bool gomp_affinity_remove_cpu (void *, unsigned long); 706 extern bool gomp_affinity_copy_place (void *, void *, long); 707 extern bool gomp_affinity_same_place (void *, void *); 708 extern bool gomp_affinity_finalize_place_list (bool); 709 extern bool gomp_affinity_init_level (int, unsigned long, bool); 710 extern void gomp_affinity_print_place (void *); 711 extern void gomp_get_place_proc_ids_8 (int, int64_t *); 712 713 /* iter.c */ 714 715 extern int gomp_iter_static_next (long *, long *); 716 extern bool gomp_iter_dynamic_next_locked (long *, long *); 717 extern bool gomp_iter_guided_next_locked (long *, long *); 718 719 #ifdef HAVE_SYNC_BUILTINS 720 extern bool gomp_iter_dynamic_next (long *, long *); 721 extern bool gomp_iter_guided_next (long *, long *); 722 #endif 723 724 /* iter_ull.c */ 725 726 extern int gomp_iter_ull_static_next (unsigned long long *, 727 unsigned long long *); 728 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *, 729 unsigned long long *); 730 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *, 731 unsigned long long *); 732 733 #if defined HAVE_SYNC_BUILTINS && defined __LP64__ 734 extern bool gomp_iter_ull_dynamic_next (unsigned long long *, 735 unsigned long long *); 736 extern bool gomp_iter_ull_guided_next (unsigned long long *, 737 unsigned long long *); 738 #endif 739 740 /* ordered.c */ 741 742 extern void gomp_ordered_first (void); 743 extern void gomp_ordered_last (void); 744 extern void gomp_ordered_next (void); 745 extern void gomp_ordered_static_init (void); 746 extern void gomp_ordered_static_next (void); 747 extern void gomp_ordered_sync (void); 748 extern void gomp_doacross_init (unsigned, long *, long); 749 extern void gomp_doacross_ull_init (unsigned, unsigned long long *, 750 unsigned long long); 751 752 /* parallel.c */ 753 754 extern unsigned gomp_resolve_num_threads (unsigned, unsigned); 755 756 /* proc.c (in config/) */ 757 758 extern void gomp_init_num_threads (void); 759 extern unsigned gomp_dynamic_max_threads (void); 760 761 /* task.c */ 762 763 extern void gomp_init_task (struct gomp_task *, struct gomp_task *, 764 struct gomp_task_icv *); 765 extern void gomp_end_task (void); 766 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t); 767 extern void gomp_task_maybe_wait_for_dependencies (void **); 768 extern bool gomp_create_target_task (struct gomp_device_descr *, 769 void (*) (void *), size_t, void **, 770 size_t *, unsigned short *, unsigned int, 771 void **, void **, 772 enum gomp_target_task_state); 773 774 static void inline 775 gomp_finish_task (struct gomp_task *task) 776 { 777 if (__builtin_expect (task->depend_hash != NULL, 0)) 778 free (task->depend_hash); 779 } 780 781 /* team.c */ 782 783 extern struct gomp_team *gomp_new_team (unsigned); 784 extern void gomp_team_start (void (*) (void *), void *, unsigned, 785 unsigned, struct gomp_team *); 786 extern void gomp_team_end (void); 787 extern void gomp_free_thread (void *); 788 789 /* target.c */ 790 791 extern void gomp_init_targets_once (void); 792 extern int gomp_get_num_devices (void); 793 extern bool gomp_target_task_fn (void *); 794 795 /* Splay tree definitions. */ 796 typedef struct splay_tree_node_s *splay_tree_node; 797 typedef struct splay_tree_s *splay_tree; 798 typedef struct splay_tree_key_s *splay_tree_key; 799 800 struct target_var_desc { 801 /* Splay key. */ 802 splay_tree_key key; 803 /* True if data should be copied from device to host at the end. */ 804 bool copy_from; 805 /* True if data always should be copied from device to host at the end. */ 806 bool always_copy_from; 807 /* Relative offset against key host_start. */ 808 uintptr_t offset; 809 /* Actual length. */ 810 uintptr_t length; 811 }; 812 813 struct target_mem_desc { 814 /* Reference count. */ 815 uintptr_t refcount; 816 /* All the splay nodes allocated together. */ 817 splay_tree_node array; 818 /* Start of the target region. */ 819 uintptr_t tgt_start; 820 /* End of the targer region. */ 821 uintptr_t tgt_end; 822 /* Handle to free. */ 823 void *to_free; 824 /* Previous target_mem_desc. */ 825 struct target_mem_desc *prev; 826 /* Number of items in following list. */ 827 size_t list_count; 828 829 /* Corresponding target device descriptor. */ 830 struct gomp_device_descr *device_descr; 831 832 /* List of target items to remove (or decrease refcount) 833 at the end of region. */ 834 struct target_var_desc list[]; 835 }; 836 837 /* Special value for refcount - infinity. */ 838 #define REFCOUNT_INFINITY (~(uintptr_t) 0) 839 /* Special value for refcount - tgt_offset contains target address of the 840 artificial pointer to "omp declare target link" object. */ 841 #define REFCOUNT_LINK (~(uintptr_t) 1) 842 843 struct splay_tree_key_s { 844 /* Address of the host object. */ 845 uintptr_t host_start; 846 /* Address immediately after the host object. */ 847 uintptr_t host_end; 848 /* Descriptor of the target memory. */ 849 struct target_mem_desc *tgt; 850 /* Offset from tgt->tgt_start to the start of the target object. */ 851 uintptr_t tgt_offset; 852 /* Reference count. */ 853 uintptr_t refcount; 854 /* Pointer to the original mapping of "omp declare target link" object. */ 855 splay_tree_key link_key; 856 }; 857 858 /* The comparison function. */ 859 860 static inline int 861 splay_compare (splay_tree_key x, splay_tree_key y) 862 { 863 if (x->host_start == x->host_end 864 && y->host_start == y->host_end) 865 return 0; 866 if (x->host_end <= y->host_start) 867 return -1; 868 if (x->host_start >= y->host_end) 869 return 1; 870 return 0; 871 } 872 873 #include "splay-tree.h" 874 875 typedef struct acc_dispatch_t 876 { 877 /* This is a linked list of data mapped using the 878 acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas. 879 Unlike mapped_data in the goacc_thread struct, unmapping can 880 happen out-of-order with respect to mapping. */ 881 /* This is guarded by the lock in the "outer" struct gomp_device_descr. */ 882 struct target_mem_desc *data_environ; 883 884 /* Execute. */ 885 __typeof (GOMP_OFFLOAD_openacc_exec) *exec_func; 886 887 /* Async cleanup callback registration. */ 888 __typeof (GOMP_OFFLOAD_openacc_register_async_cleanup) 889 *register_async_cleanup_func; 890 891 /* Asynchronous routines. */ 892 __typeof (GOMP_OFFLOAD_openacc_async_test) *async_test_func; 893 __typeof (GOMP_OFFLOAD_openacc_async_test_all) *async_test_all_func; 894 __typeof (GOMP_OFFLOAD_openacc_async_wait) *async_wait_func; 895 __typeof (GOMP_OFFLOAD_openacc_async_wait_async) *async_wait_async_func; 896 __typeof (GOMP_OFFLOAD_openacc_async_wait_all) *async_wait_all_func; 897 __typeof (GOMP_OFFLOAD_openacc_async_wait_all_async) 898 *async_wait_all_async_func; 899 __typeof (GOMP_OFFLOAD_openacc_async_set_async) *async_set_async_func; 900 901 /* Create/destroy TLS data. */ 902 __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func; 903 __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data) 904 *destroy_thread_data_func; 905 906 /* NVIDIA target specific routines. */ 907 struct { 908 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device) 909 *get_current_device_func; 910 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context) 911 *get_current_context_func; 912 __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream) *get_stream_func; 913 __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream) *set_stream_func; 914 } cuda; 915 } acc_dispatch_t; 916 917 /* Various state of the accelerator device. */ 918 enum gomp_device_state 919 { 920 GOMP_DEVICE_UNINITIALIZED, 921 GOMP_DEVICE_INITIALIZED, 922 GOMP_DEVICE_FINALIZED 923 }; 924 925 /* This structure describes accelerator device. 926 It contains name of the corresponding libgomp plugin, function handlers for 927 interaction with the device, ID-number of the device, and information about 928 mapped memory. */ 929 struct gomp_device_descr 930 { 931 /* Immutable data, which is only set during initialization, and which is not 932 guarded by the lock. */ 933 934 /* The name of the device. */ 935 const char *name; 936 937 /* Capabilities of device (supports OpenACC, OpenMP). */ 938 unsigned int capabilities; 939 940 /* This is the ID number of device among devices of the same type. */ 941 int target_id; 942 943 /* This is the TYPE of device. */ 944 enum offload_target_type type; 945 946 /* Function handlers. */ 947 __typeof (GOMP_OFFLOAD_get_name) *get_name_func; 948 __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func; 949 __typeof (GOMP_OFFLOAD_get_type) *get_type_func; 950 __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func; 951 __typeof (GOMP_OFFLOAD_init_device) *init_device_func; 952 __typeof (GOMP_OFFLOAD_fini_device) *fini_device_func; 953 __typeof (GOMP_OFFLOAD_version) *version_func; 954 __typeof (GOMP_OFFLOAD_load_image) *load_image_func; 955 __typeof (GOMP_OFFLOAD_unload_image) *unload_image_func; 956 __typeof (GOMP_OFFLOAD_alloc) *alloc_func; 957 __typeof (GOMP_OFFLOAD_free) *free_func; 958 __typeof (GOMP_OFFLOAD_dev2host) *dev2host_func; 959 __typeof (GOMP_OFFLOAD_host2dev) *host2dev_func; 960 __typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func; 961 __typeof (GOMP_OFFLOAD_can_run) *can_run_func; 962 __typeof (GOMP_OFFLOAD_run) *run_func; 963 __typeof (GOMP_OFFLOAD_async_run) *async_run_func; 964 965 /* Splay tree containing information about mapped memory regions. */ 966 struct splay_tree_s mem_map; 967 968 /* Mutex for the mutable data. */ 969 gomp_mutex_t lock; 970 971 /* Current state of the device. OpenACC allows to move from INITIALIZED state 972 back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED 973 to FINALIZED state (at program shutdown). */ 974 enum gomp_device_state state; 975 976 /* OpenACC-specific data and functions. */ 977 /* This is mutable because of its mutable data_environ and target_data 978 members. */ 979 acc_dispatch_t openacc; 980 }; 981 982 /* Kind of the pragma, for which gomp_map_vars () is called. */ 983 enum gomp_map_vars_kind 984 { 985 GOMP_MAP_VARS_OPENACC, 986 GOMP_MAP_VARS_TARGET, 987 GOMP_MAP_VARS_DATA, 988 GOMP_MAP_VARS_ENTER_DATA 989 }; 990 991 extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *); 992 extern void gomp_acc_remove_pointer (void *, bool, int, int); 993 994 extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *, 995 size_t, void **, void **, 996 size_t *, void *, bool, 997 enum gomp_map_vars_kind); 998 extern void gomp_unmap_vars (struct target_mem_desc *, bool); 999 extern void gomp_init_device (struct gomp_device_descr *); 1000 extern void gomp_free_memmap (struct splay_tree_s *); 1001 extern void gomp_unload_device (struct gomp_device_descr *); 1002 1003 /* work.c */ 1004 1005 extern void gomp_init_work_share (struct gomp_work_share *, bool, unsigned); 1006 extern void gomp_fini_work_share (struct gomp_work_share *); 1007 extern bool gomp_work_share_start (bool); 1008 extern void gomp_work_share_end (void); 1009 extern bool gomp_work_share_end_cancel (void); 1010 extern void gomp_work_share_end_nowait (void); 1011 1012 static inline void 1013 gomp_work_share_init_done (void) 1014 { 1015 struct gomp_thread *thr = gomp_thread (); 1016 if (__builtin_expect (thr->ts.last_work_share != NULL, 1)) 1017 gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share); 1018 } 1019 1020 #ifdef HAVE_ATTRIBUTE_VISIBILITY 1021 # pragma GCC visibility pop 1022 #endif 1023 1024 /* Now that we're back to default visibility, include the globals. */ 1025 #include "libgomp_g.h" 1026 1027 /* Include omp.h by parts. */ 1028 #include "omp-lock.h" 1029 #define _LIBGOMP_OMP_LOCK_DEFINED 1 1030 #include "omp.h.in" 1031 1032 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \ 1033 || !defined (HAVE_ATTRIBUTE_ALIAS) \ 1034 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \ 1035 || !defined (PIC) \ 1036 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT) 1037 # undef LIBGOMP_GNU_SYMBOL_VERSIONING 1038 #endif 1039 1040 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING 1041 extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 1042 extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 1043 extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 1044 extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 1045 extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW; 1046 extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 1047 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 1048 extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 1049 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 1050 extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW; 1051 1052 extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 1053 extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 1054 extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 1055 extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 1056 extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW; 1057 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 1058 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 1059 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 1060 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 1061 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW; 1062 1063 # define omp_lock_symver(fn) \ 1064 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \ 1065 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0"); 1066 #else 1067 # define gomp_init_lock_30 omp_init_lock 1068 # define gomp_destroy_lock_30 omp_destroy_lock 1069 # define gomp_set_lock_30 omp_set_lock 1070 # define gomp_unset_lock_30 omp_unset_lock 1071 # define gomp_test_lock_30 omp_test_lock 1072 # define gomp_init_nest_lock_30 omp_init_nest_lock 1073 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock 1074 # define gomp_set_nest_lock_30 omp_set_nest_lock 1075 # define gomp_unset_nest_lock_30 omp_unset_nest_lock 1076 # define gomp_test_nest_lock_30 omp_test_nest_lock 1077 #endif 1078 1079 #ifdef HAVE_ATTRIBUTE_VISIBILITY 1080 # define attribute_hidden __attribute__ ((visibility ("hidden"))) 1081 #else 1082 # define attribute_hidden 1083 #endif 1084 1085 #ifdef HAVE_ATTRIBUTE_ALIAS 1086 # define strong_alias(fn, al) \ 1087 extern __typeof (fn) al __attribute__ ((alias (#fn))); 1088 1089 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__) 1090 # define ialias_str1(x) ialias_str2(x) 1091 # define ialias_str2(x) #x 1092 # define ialias(fn) \ 1093 extern __typeof (fn) gomp_ialias_##fn \ 1094 __attribute__ ((alias (#fn))) attribute_hidden; 1095 # define ialias_redirect(fn) \ 1096 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden; 1097 # define ialias_call(fn) gomp_ialias_ ## fn 1098 #else 1099 # define ialias(fn) 1100 # define ialias_redirect(fn) 1101 # define ialias_call(fn) fn 1102 #endif 1103 1104 /* Helper function for priority_node_to_task() and 1105 task_to_priority_node(). 1106 1107 Return the offset from a task to its priority_node entry. The 1108 priority_node entry is has a type of TYPE. */ 1109 1110 static inline size_t 1111 priority_queue_offset (enum priority_queue_type type) 1112 { 1113 return offsetof (struct gomp_task, pnode[(int) type]); 1114 } 1115 1116 /* Return the task associated with a priority NODE of type TYPE. */ 1117 1118 static inline struct gomp_task * 1119 priority_node_to_task (enum priority_queue_type type, 1120 struct priority_node *node) 1121 { 1122 return (struct gomp_task *) ((char *) node - priority_queue_offset (type)); 1123 } 1124 1125 /* Return the priority node of type TYPE for a given TASK. */ 1126 1127 static inline struct priority_node * 1128 task_to_priority_node (enum priority_queue_type type, 1129 struct gomp_task *task) 1130 { 1131 return (struct priority_node *) ((char *) task 1132 + priority_queue_offset (type)); 1133 } 1134 #endif /* LIBGOMP_H */ 1135