1 /* Copyright (C) 2005-2021 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
3
4 This file is part of the GNU Offloading and Multi Processing Library
5 (libgomp).
6
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 more details.
16
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
20
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
25
26 /* This file contains data types and function declarations that are not
27 part of the official OpenACC or OpenMP user interfaces. There are
28 declarations in here that are part of the GNU Offloading and Multi
29 Processing ABI, in that the compiler is required to know about them
30 and use them.
31
32 The convention is that the all caps prefix "GOMP" is used group items
33 that are part of the external ABI, and the lower case prefix "gomp"
34 is used group items that are completely private to the library. */
35
36 #ifndef LIBGOMP_H
37 #define LIBGOMP_H 1
38
39 #ifndef _LIBGOMP_CHECKING_
40 /* Define to 1 to perform internal sanity checks. */
41 #define _LIBGOMP_CHECKING_ 0
42 #endif
43
44 #include "config.h"
45 #include <stdint.h>
46 #include "libgomp-plugin.h"
47 #include "gomp-constants.h"
48
49 #ifdef HAVE_PTHREAD_H
50 #include <pthread.h>
51 #endif
52 #include <stdbool.h>
53 #include <stdlib.h>
54 #include <stdarg.h>
55
56 /* Needed for memset in priority_queue.c. */
57 #if _LIBGOMP_CHECKING_
58 # ifdef STRING_WITH_STRINGS
59 # include <string.h>
60 # include <strings.h>
61 # else
62 # ifdef HAVE_STRING_H
63 # include <string.h>
64 # else
65 # ifdef HAVE_STRINGS_H
66 # include <strings.h>
67 # endif
68 # endif
69 # endif
70 #endif
71
72 #ifdef HAVE_ATTRIBUTE_VISIBILITY
73 # pragma GCC visibility push(hidden)
74 #endif
75
76 /* If we were a C++ library, we'd get this from <std/atomic>. */
77 enum memmodel
78 {
79 MEMMODEL_RELAXED = 0,
80 MEMMODEL_CONSUME = 1,
81 MEMMODEL_ACQUIRE = 2,
82 MEMMODEL_RELEASE = 3,
83 MEMMODEL_ACQ_REL = 4,
84 MEMMODEL_SEQ_CST = 5
85 };
86
87 /* alloc.c */
88
89 #if defined(HAVE_ALIGNED_ALLOC) \
90 || defined(HAVE__ALIGNED_MALLOC) \
91 || defined(HAVE_POSIX_MEMALIGN) \
92 || defined(HAVE_MEMALIGN)
93 /* Defined if gomp_aligned_alloc doesn't use fallback version
94 and free can be used instead of gomp_aligned_free. */
95 #define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
96 #endif
97
98 extern void *gomp_malloc (size_t) __attribute__((malloc));
99 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
100 extern void *gomp_realloc (void *, size_t);
101 extern void *gomp_aligned_alloc (size_t, size_t)
102 __attribute__((malloc, alloc_size (2)));
103 extern void gomp_aligned_free (void *);
104
105 /* Avoid conflicting prototypes of alloca() in system headers by using
106 GCC's builtin alloca(). */
107 #define gomp_alloca(x) __builtin_alloca(x)
108
109 /* Optimized allocators for team-specific data that will die with the team. */
110
111 #ifdef __AMDGCN__
112 /* The arena is initialized in config/gcn/team.c. */
113 #define TEAM_ARENA_SIZE 64*1024 /* Must match the value in plugin-gcn.c. */
114 #define TEAM_ARENA_START 16 /* LDS offset of free pointer. */
115 #define TEAM_ARENA_FREE 24 /* LDS offset of free pointer. */
116 #define TEAM_ARENA_END 32 /* LDS offset of end pointer. */
117
118 static inline void * __attribute__((malloc))
team_malloc(size_t size)119 team_malloc (size_t size)
120 {
121 /* 4-byte align the size. */
122 size = (size + 3) & ~3;
123
124 /* Allocate directly from the arena.
125 The compiler does not support DS atomics, yet. */
126 void *result;
127 asm ("ds_add_rtn_u64 %0, %1, %2\n\ts_waitcnt 0"
128 : "=v"(result) : "v"(TEAM_ARENA_FREE), "v"(size), "e"(1L) : "memory");
129
130 /* Handle OOM. */
131 if (result + size > *(void * __lds *)TEAM_ARENA_END)
132 {
133 /* While this is experimental, let's make sure we know when OOM
134 happens. */
135 const char msg[] = "GCN team arena exhausted\n";
136 write (2, msg, sizeof(msg)-1);
137
138 /* Fall back to using the heap (slowly). */
139 result = gomp_malloc (size);
140 }
141 return result;
142 }
143
144 static inline void * __attribute__((malloc))
team_malloc_cleared(size_t size)145 team_malloc_cleared (size_t size)
146 {
147 char *result = team_malloc (size);
148
149 /* Clear the allocated memory. */
150 __builtin_memset (result, 0, size);
151
152 return result;
153 }
154
155 static inline void
team_free(void * ptr)156 team_free (void *ptr)
157 {
158 /* The whole arena is freed when the kernel exits.
159 However, if we fell back to using heap then we should free it.
160 It would be better if this function could be a no-op, but at least
161 LDS loads are cheap. */
162 if (ptr < *(void * __lds *)TEAM_ARENA_START
163 || ptr >= *(void * __lds *)TEAM_ARENA_END)
164 free (ptr);
165 }
166 #else
167 #define team_malloc(...) gomp_malloc (__VA_ARGS__)
168 #define team_malloc_cleared(...) gomp_malloc_cleared (__VA_ARGS__)
169 #define team_free(...) free (__VA_ARGS__)
170 #endif
171
172 /* error.c */
173
174 extern void gomp_vdebug (int, const char *, va_list);
175 extern void gomp_debug (int, const char *, ...)
176 __attribute__ ((format (printf, 2, 3)));
177 #define gomp_vdebug(KIND, FMT, VALIST) \
178 do { \
179 if (__builtin_expect (gomp_debug_var, 0)) \
180 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
181 } while (0)
182 #define gomp_debug(KIND, ...) \
183 do { \
184 if (__builtin_expect (gomp_debug_var, 0)) \
185 (gomp_debug) ((KIND), __VA_ARGS__); \
186 } while (0)
187 extern void gomp_verror (const char *, va_list);
188 extern void gomp_error (const char *, ...)
189 __attribute__ ((format (printf, 1, 2)));
190 extern void gomp_vfatal (const char *, va_list)
191 __attribute__ ((noreturn));
192 extern void gomp_fatal (const char *, ...)
193 __attribute__ ((noreturn, format (printf, 1, 2)));
194
195 struct gomp_task;
196 struct gomp_taskgroup;
197 struct htab;
198
199 #include "priority_queue.h"
200 #include "sem.h"
201 #include "mutex.h"
202 #include "bar.h"
203 #include "simple-bar.h"
204 #include "ptrlock.h"
205
206
207 /* This structure contains the data to control one work-sharing construct,
208 either a LOOP (FOR/DO) or a SECTIONS. */
209
210 enum gomp_schedule_type
211 {
212 GFS_RUNTIME,
213 GFS_STATIC,
214 GFS_DYNAMIC,
215 GFS_GUIDED,
216 GFS_AUTO,
217 GFS_MONOTONIC = 0x80000000U
218 };
219
220 struct gomp_doacross_work_share
221 {
222 union {
223 /* chunk_size copy, as ws->chunk_size is multiplied by incr for
224 GFS_DYNAMIC. */
225 long chunk_size;
226 /* Likewise, but for ull implementation. */
227 unsigned long long chunk_size_ull;
228 /* For schedule(static,0) this is the number
229 of iterations assigned to the last thread, i.e. number of
230 iterations / number of threads. */
231 long q;
232 /* Likewise, but for ull implementation. */
233 unsigned long long q_ull;
234 };
235 /* Size of each array entry (padded to cache line size). */
236 unsigned long elt_sz;
237 /* Number of dimensions in sink vectors. */
238 unsigned int ncounts;
239 /* True if the iterations can be flattened. */
240 bool flattened;
241 /* Actual array (of elt_sz sized units), aligned to cache line size.
242 This is indexed by team_id for GFS_STATIC and outermost iteration
243 / chunk_size for other schedules. */
244 unsigned char *array;
245 /* These two are only used for schedule(static,0). */
246 /* This one is number of iterations % number of threads. */
247 long t;
248 union {
249 /* And this one is cached t * (q + 1). */
250 long boundary;
251 /* Likewise, but for the ull implementation. */
252 unsigned long long boundary_ull;
253 };
254 /* Pointer to extra memory if needed for lastprivate(conditional). */
255 void *extra;
256 /* Array of shift counts for each dimension if they can be flattened. */
257 unsigned int shift_counts[];
258 };
259
260 struct gomp_work_share
261 {
262 /* This member records the SCHEDULE clause to be used for this construct.
263 The user specification of "runtime" will already have been resolved.
264 If this is a SECTIONS construct, this value will always be DYNAMIC. */
265 enum gomp_schedule_type sched;
266
267 int mode;
268
269 union {
270 struct {
271 /* This is the chunk_size argument to the SCHEDULE clause. */
272 long chunk_size;
273
274 /* This is the iteration end point. If this is a SECTIONS construct,
275 this is the number of contained sections. */
276 long end;
277
278 /* This is the iteration step. If this is a SECTIONS construct, this
279 is always 1. */
280 long incr;
281 };
282
283 struct {
284 /* The same as above, but for the unsigned long long loop variants. */
285 unsigned long long chunk_size_ull;
286 unsigned long long end_ull;
287 unsigned long long incr_ull;
288 };
289 };
290
291 union {
292 /* This is a circular queue that details which threads will be allowed
293 into the ordered region and in which order. When a thread allocates
294 iterations on which it is going to work, it also registers itself at
295 the end of the array. When a thread reaches the ordered region, it
296 checks to see if it is the one at the head of the queue. If not, it
297 blocks on its RELEASE semaphore. */
298 unsigned *ordered_team_ids;
299
300 /* This is a pointer to DOACROSS work share data. */
301 struct gomp_doacross_work_share *doacross;
302 };
303
304 /* This is the number of threads that have registered themselves in
305 the circular queue ordered_team_ids. */
306 unsigned ordered_num_used;
307
308 /* This is the team_id of the currently acknowledged owner of the ordered
309 section, or -1u if the ordered section has not been acknowledged by
310 any thread. This is distinguished from the thread that is *allowed*
311 to take the section next. */
312 unsigned ordered_owner;
313
314 /* This is the index into the circular queue ordered_team_ids of the
315 current thread that's allowed into the ordered reason. */
316 unsigned ordered_cur;
317
318 /* This is a chain of allocated gomp_work_share blocks, valid only
319 in the first gomp_work_share struct in the block. */
320 struct gomp_work_share *next_alloc;
321
322 /* The above fields are written once during workshare initialization,
323 or related to ordered worksharing. Make sure the following fields
324 are in a different cache line. */
325
326 /* This lock protects the update of the following members. */
327 gomp_mutex_t lock __attribute__((aligned (64)));
328
329 /* This is the count of the number of threads that have exited the work
330 share construct. If the construct was marked nowait, they have moved on
331 to other work; otherwise they're blocked on a barrier. The last member
332 of the team to exit the work share construct must deallocate it. */
333 unsigned threads_completed;
334
335 union {
336 /* This is the next iteration value to be allocated. In the case of
337 GFS_STATIC loops, this the iteration start point and never changes. */
338 long next;
339
340 /* The same, but with unsigned long long type. */
341 unsigned long long next_ull;
342
343 /* This is the returned data structure for SINGLE COPYPRIVATE. */
344 void *copyprivate;
345 };
346
347 union {
348 /* Link to gomp_work_share struct for next work sharing construct
349 encountered after this one. */
350 gomp_ptrlock_t next_ws;
351
352 /* gomp_work_share structs are chained in the free work share cache
353 through this. */
354 struct gomp_work_share *next_free;
355 };
356
357 /* Task reductions for this work-sharing construct. */
358 uintptr_t *task_reductions;
359
360 /* If only few threads are in the team, ordered_team_ids can point
361 to this array which fills the padding at the end of this struct. */
362 unsigned inline_ordered_team_ids[0];
363 };
364
365 /* This structure contains all of the thread-local data associated with
366 a thread team. This is the data that must be saved when a thread
367 encounters a nested PARALLEL construct. */
368
369 struct gomp_team_state
370 {
371 /* This is the team of which the thread is currently a member. */
372 struct gomp_team *team;
373
374 /* This is the work share construct which this thread is currently
375 processing. Recall that with NOWAIT, not all threads may be
376 processing the same construct. */
377 struct gomp_work_share *work_share;
378
379 /* This is the previous work share construct or NULL if there wasn't any.
380 When all threads are done with the current work sharing construct,
381 the previous one can be freed. The current one can't, as its
382 next_ws field is used. */
383 struct gomp_work_share *last_work_share;
384
385 /* This is the ID of this thread within the team. This value is
386 guaranteed to be between 0 and N-1, where N is the number of
387 threads in the team. */
388 unsigned team_id;
389
390 /* Nesting level. */
391 unsigned level;
392
393 /* Active nesting level. Only active parallel regions are counted. */
394 unsigned active_level;
395
396 /* Place-partition-var, offset and length into gomp_places_list array. */
397 unsigned place_partition_off;
398 unsigned place_partition_len;
399
400 /* Def-allocator-var ICV. */
401 uintptr_t def_allocator;
402
403 #ifdef HAVE_SYNC_BUILTINS
404 /* Number of single stmts encountered. */
405 unsigned long single_count;
406 #endif
407
408 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
409 trip number through the loop. So first time a particular loop
410 is encountered this number is 0, the second time through the loop
411 is 1, etc. This is unused when the compiler knows in advance that
412 the loop is statically scheduled. */
413 unsigned long static_trip;
414 };
415
416 struct target_mem_desc;
417
418 /* These are the OpenMP 4.0 Internal Control Variables described in
419 section 2.3.1. Those described as having one copy per task are
420 stored within the structure; those described as having one copy
421 for the whole program are (naturally) global variables. */
422
423 struct gomp_task_icv
424 {
425 unsigned long nthreads_var;
426 enum gomp_schedule_type run_sched_var;
427 int run_sched_chunk_size;
428 int default_device_var;
429 unsigned int thread_limit_var;
430 bool dyn_var;
431 unsigned char max_active_levels_var;
432 char bind_var;
433 /* Internal ICV. */
434 struct target_mem_desc *target_data;
435 };
436
437 enum gomp_target_offload_t
438 {
439 GOMP_TARGET_OFFLOAD_DEFAULT,
440 GOMP_TARGET_OFFLOAD_MANDATORY,
441 GOMP_TARGET_OFFLOAD_DISABLED
442 };
443
444 #define gomp_supported_active_levels UCHAR_MAX
445
446 extern struct gomp_task_icv gomp_global_icv;
447 #ifndef HAVE_SYNC_BUILTINS
448 extern gomp_mutex_t gomp_managed_threads_lock;
449 #endif
450 extern bool gomp_cancel_var;
451 extern enum gomp_target_offload_t gomp_target_offload_var;
452 extern int gomp_max_task_priority_var;
453 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
454 extern unsigned long gomp_available_cpus, gomp_managed_threads;
455 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;
456 extern char *gomp_bind_var_list;
457 extern unsigned long gomp_bind_var_list_len;
458 extern void **gomp_places_list;
459 extern unsigned long gomp_places_list_len;
460 extern unsigned int gomp_num_teams_var;
461 extern int gomp_debug_var;
462 extern bool gomp_display_affinity_var;
463 extern char *gomp_affinity_format_var;
464 extern size_t gomp_affinity_format_len;
465 extern uintptr_t gomp_def_allocator;
466 extern int goacc_device_num;
467 extern char *goacc_device_type;
468 extern int goacc_default_dims[GOMP_DIM_MAX];
469
470 enum gomp_task_kind
471 {
472 /* Implicit task. */
473 GOMP_TASK_IMPLICIT,
474 /* Undeferred task. */
475 GOMP_TASK_UNDEFERRED,
476 /* Task created by GOMP_task and waiting to be run. */
477 GOMP_TASK_WAITING,
478 /* Task currently executing or scheduled and about to execute. */
479 GOMP_TASK_TIED,
480 /* Used for target tasks that have vars mapped and async run started,
481 but not yet completed. Once that completes, they will be readded
482 into the queues as GOMP_TASK_WAITING in order to perform the var
483 unmapping. */
484 GOMP_TASK_ASYNC_RUNNING,
485 /* Task that has finished executing but is waiting for its
486 completion event to be fulfilled. */
487 GOMP_TASK_DETACHED
488 };
489
490 struct gomp_task_depend_entry
491 {
492 /* Address of dependency. */
493 void *addr;
494 struct gomp_task_depend_entry *next;
495 struct gomp_task_depend_entry *prev;
496 /* Task that provides the dependency in ADDR. */
497 struct gomp_task *task;
498 /* Depend entry is of type "IN". */
499 bool is_in;
500 bool redundant;
501 bool redundant_out;
502 };
503
504 struct gomp_dependers_vec
505 {
506 size_t n_elem;
507 size_t allocated;
508 struct gomp_task *elem[];
509 };
510
511 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */
512
513 struct gomp_taskwait
514 {
515 bool in_taskwait;
516 bool in_depend_wait;
517 /* Number of tasks we are waiting for. */
518 size_t n_depend;
519 gomp_sem_t taskwait_sem;
520 };
521
522 /* This structure describes a "task" to be run by a thread. */
523
524 struct gomp_task
525 {
526 /* Parent of this task. */
527 struct gomp_task *parent;
528 /* Children of this task. */
529 struct priority_queue children_queue;
530 /* Taskgroup this task belongs in. */
531 struct gomp_taskgroup *taskgroup;
532 /* Tasks that depend on this task. */
533 struct gomp_dependers_vec *dependers;
534 struct htab *depend_hash;
535 struct gomp_taskwait *taskwait;
536 /* Number of items in DEPEND. */
537 size_t depend_count;
538 /* Number of tasks this task depends on. Once this counter reaches
539 0, we have no unsatisfied dependencies, and this task can be put
540 into the various queues to be scheduled. */
541 size_t num_dependees;
542
543 union {
544 /* Valid only if deferred_p is false. */
545 gomp_sem_t *completion_sem;
546 /* Valid only if deferred_p is true. Set to the team that executes the
547 task if the task is detached and the completion event has yet to be
548 fulfilled. */
549 struct gomp_team *detach_team;
550 };
551 bool deferred_p;
552
553 /* Priority of this task. */
554 int priority;
555 /* The priority node for this task in each of the different queues.
556 We put this here to avoid allocating space for each priority
557 node. Then we play offsetof() games to convert between pnode[]
558 entries and the gomp_task in which they reside. */
559 struct priority_node pnode[3];
560
561 struct gomp_task_icv icv;
562 void (*fn) (void *);
563 void *fn_data;
564 enum gomp_task_kind kind;
565 bool in_tied_task;
566 bool final_task;
567 bool copy_ctors_done;
568 /* Set for undeferred tasks with unsatisfied dependencies which
569 block further execution of their parent until the dependencies
570 are satisfied. */
571 bool parent_depends_on;
572 /* Dependencies provided and/or needed for this task. DEPEND_COUNT
573 is the number of items available. */
574 struct gomp_task_depend_entry depend[];
575 };
576
577 /* This structure describes a single #pragma omp taskgroup. */
578
579 struct gomp_taskgroup
580 {
581 struct gomp_taskgroup *prev;
582 /* Queue of tasks that belong in this taskgroup. */
583 struct priority_queue taskgroup_queue;
584 uintptr_t *reductions;
585 bool in_taskgroup_wait;
586 bool cancelled;
587 bool workshare;
588 gomp_sem_t taskgroup_sem;
589 size_t num_children;
590 };
591
592 /* Various state of OpenMP async offloading tasks. */
593 enum gomp_target_task_state
594 {
595 GOMP_TARGET_TASK_DATA,
596 GOMP_TARGET_TASK_BEFORE_MAP,
597 GOMP_TARGET_TASK_FALLBACK,
598 GOMP_TARGET_TASK_READY_TO_RUN,
599 GOMP_TARGET_TASK_RUNNING,
600 GOMP_TARGET_TASK_FINISHED
601 };
602
603 /* This structure describes a target task. */
604
605 struct gomp_target_task
606 {
607 struct gomp_device_descr *devicep;
608 void (*fn) (void *);
609 size_t mapnum;
610 size_t *sizes;
611 unsigned short *kinds;
612 unsigned int flags;
613 enum gomp_target_task_state state;
614 struct target_mem_desc *tgt;
615 struct gomp_task *task;
616 struct gomp_team *team;
617 /* Device-specific target arguments. */
618 void **args;
619 void *hostaddrs[];
620 };
621
622 /* This structure describes a "team" of threads. These are the threads
623 that are spawned by a PARALLEL constructs, as well as the work sharing
624 constructs that the team encounters. */
625
626 struct gomp_team
627 {
628 /* This is the number of threads in the current team. */
629 unsigned nthreads;
630
631 /* This is number of gomp_work_share structs that have been allocated
632 as a block last time. */
633 unsigned work_share_chunk;
634
635 /* This is the saved team state that applied to a master thread before
636 the current thread was created. */
637 struct gomp_team_state prev_ts;
638
639 /* This semaphore should be used by the master thread instead of its
640 "native" semaphore in the thread structure. Required for nested
641 parallels, as the master is a member of two teams. */
642 gomp_sem_t master_release;
643
644 /* This points to an array with pointers to the release semaphore
645 of the threads in the team. */
646 gomp_sem_t **ordered_release;
647
648 /* List of work shares on which gomp_fini_work_share hasn't been
649 called yet. If the team hasn't been cancelled, this should be
650 equal to each thr->ts.work_share, but otherwise it can be a possibly
651 long list of workshares. */
652 struct gomp_work_share *work_shares_to_free;
653
654 /* List of gomp_work_share structs chained through next_free fields.
655 This is populated and taken off only by the first thread in the
656 team encountering a new work sharing construct, in a critical
657 section. */
658 struct gomp_work_share *work_share_list_alloc;
659
660 /* List of gomp_work_share structs freed by free_work_share. New
661 entries are atomically added to the start of the list, and
662 alloc_work_share can safely only move all but the first entry
663 to work_share_list alloc, as free_work_share can happen concurrently
664 with alloc_work_share. */
665 struct gomp_work_share *work_share_list_free;
666
667 #ifdef HAVE_SYNC_BUILTINS
668 /* Number of simple single regions encountered by threads in this
669 team. */
670 unsigned long single_count;
671 #else
672 /* Mutex protecting addition of workshares to work_share_list_free. */
673 gomp_mutex_t work_share_list_free_lock;
674 #endif
675
676 /* This barrier is used for most synchronization of the team. */
677 gomp_barrier_t barrier;
678
679 /* Initial work shares, to avoid allocating any gomp_work_share
680 structs in the common case. */
681 struct gomp_work_share work_shares[8];
682
683 gomp_mutex_t task_lock;
684 /* Scheduled tasks. */
685 struct priority_queue task_queue;
686 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */
687 unsigned int task_count;
688 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */
689 unsigned int task_queued_count;
690 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
691 directly in gomp_barrier_handle_tasks; tasks spawned
692 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
693 that is called from a task run from gomp_barrier_handle_tasks.
694 task_running_count should be always <= team->nthreads,
695 and if current task isn't in_tied_task, then it will be
696 even < team->nthreads. */
697 unsigned int task_running_count;
698 int work_share_cancelled;
699 int team_cancelled;
700
701 /* Number of tasks waiting for their completion event to be fulfilled. */
702 unsigned int task_detach_count;
703
704 /* This array contains structures for implicit tasks. */
705 struct gomp_task implicit_task[];
706 };
707
708 /* This structure contains all data that is private to libgomp and is
709 allocated per thread. */
710
711 struct gomp_thread
712 {
713 /* This is the function that the thread should run upon launch. */
714 void (*fn) (void *data);
715 void *data;
716
717 /* This is the current team state for this thread. The ts.team member
718 is NULL only if the thread is idle. */
719 struct gomp_team_state ts;
720
721 /* This is the task that the thread is currently executing. */
722 struct gomp_task *task;
723
724 /* This semaphore is used for ordered loops. */
725 gomp_sem_t release;
726
727 /* Place this thread is bound to plus one, or zero if not bound
728 to any place. */
729 unsigned int place;
730
731 /* User pthread thread pool */
732 struct gomp_thread_pool *thread_pool;
733
734 #if defined(LIBGOMP_USE_PTHREADS) \
735 && (!defined(HAVE_TLS) \
736 || !defined(__GLIBC__) \
737 || !defined(USING_INITIAL_EXEC_TLS))
738 /* pthread_t of the thread containing this gomp_thread.
739 On Linux when using initial-exec TLS,
740 (typeof (pthread_t)) gomp_thread () - pthread_self ()
741 is constant in all threads, so we can optimize and not
742 store it. */
743 #define GOMP_NEEDS_THREAD_HANDLE 1
744 pthread_t handle;
745 #endif
746 };
747
748
749 struct gomp_thread_pool
750 {
751 /* This array manages threads spawned from the top level, which will
752 return to the idle loop once the current PARALLEL construct ends. */
753 struct gomp_thread **threads;
754 unsigned threads_size;
755 unsigned threads_used;
756 /* The last team is used for non-nested teams to delay their destruction to
757 make sure all the threads in the team move on to the pool's barrier before
758 the team's barrier is destroyed. */
759 struct gomp_team *last_team;
760 /* Number of threads running in this contention group. */
761 unsigned long threads_busy;
762
763 /* This barrier holds and releases threads waiting in thread pools. */
764 gomp_simple_barrier_t threads_dock;
765 };
766
767 enum gomp_cancel_kind
768 {
769 GOMP_CANCEL_PARALLEL = 1,
770 GOMP_CANCEL_LOOP = 2,
771 GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP,
772 GOMP_CANCEL_DO = GOMP_CANCEL_LOOP,
773 GOMP_CANCEL_SECTIONS = 4,
774 GOMP_CANCEL_TASKGROUP = 8
775 };
776
777 /* ... and here is that TLS data. */
778
779 #if defined __nvptx__
780 extern struct gomp_thread *nvptx_thrs __attribute__((shared));
gomp_thread(void)781 static inline struct gomp_thread *gomp_thread (void)
782 {
783 int tid;
784 asm ("mov.u32 %0, %%tid.y;" : "=r" (tid));
785 return nvptx_thrs + tid;
786 }
787 #elif defined __AMDGCN__
gcn_thrs(void)788 static inline struct gomp_thread *gcn_thrs (void)
789 {
790 /* The value is at the bottom of LDS. */
791 struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
792 return *thrs;
793 }
set_gcn_thrs(struct gomp_thread * val)794 static inline void set_gcn_thrs (struct gomp_thread *val)
795 {
796 /* The value is at the bottom of LDS. */
797 struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
798 *thrs = val;
799 }
gomp_thread(void)800 static inline struct gomp_thread *gomp_thread (void)
801 {
802 int tid = __builtin_gcn_dim_pos(1);
803 return gcn_thrs () + tid;
804 }
805 #elif defined HAVE_TLS || defined USE_EMUTLS
806 extern __thread struct gomp_thread gomp_tls_data;
gomp_thread(void)807 static inline struct gomp_thread *gomp_thread (void)
808 {
809 return &gomp_tls_data;
810 }
811 #else
812 extern pthread_key_t gomp_tls_key;
gomp_thread(void)813 static inline struct gomp_thread *gomp_thread (void)
814 {
815 return pthread_getspecific (gomp_tls_key);
816 }
817 #endif
818
819 extern struct gomp_task_icv *gomp_new_icv (void);
820
821 /* Here's how to access the current copy of the ICVs. */
822
gomp_icv(bool write)823 static inline struct gomp_task_icv *gomp_icv (bool write)
824 {
825 struct gomp_task *task = gomp_thread ()->task;
826 if (task)
827 return &task->icv;
828 else if (write)
829 return gomp_new_icv ();
830 else
831 return &gomp_global_icv;
832 }
833
834 #ifdef LIBGOMP_USE_PTHREADS
835 /* The attributes to be used during thread creation. */
836 extern pthread_attr_t gomp_thread_attr;
837
838 extern pthread_key_t gomp_thread_destructor;
839 #endif
840
841 /* Function prototypes. */
842
843 /* affinity.c */
844
845 extern void gomp_init_affinity (void);
846 #ifdef LIBGOMP_USE_PTHREADS
847 extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int);
848 #endif
849 extern void **gomp_affinity_alloc (unsigned long, bool);
850 extern void gomp_affinity_init_place (void *);
851 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
852 long, bool);
853 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
854 extern bool gomp_affinity_copy_place (void *, void *, long);
855 extern bool gomp_affinity_same_place (void *, void *);
856 extern bool gomp_affinity_finalize_place_list (bool);
857 extern bool gomp_affinity_init_level (int, unsigned long, bool);
858 extern void gomp_affinity_print_place (void *);
859 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
860 extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
861
862 /* affinity-fmt.c */
863
864 extern bool gomp_print_string (const char *str, size_t len);
865 extern void gomp_set_affinity_format (const char *, size_t);
866 extern void gomp_display_string (char *, size_t, size_t *, const char *,
867 size_t);
868 #ifdef LIBGOMP_USE_PTHREADS
869 typedef pthread_t gomp_thread_handle;
870 #else
871 typedef struct {} gomp_thread_handle;
872 #endif
873 extern size_t gomp_display_affinity (char *, size_t, const char *,
874 gomp_thread_handle,
875 struct gomp_team_state *, unsigned int);
876 extern void gomp_display_affinity_thread (gomp_thread_handle,
877 struct gomp_team_state *,
878 unsigned int) __attribute__((cold));
879
880 /* iter.c */
881
882 extern int gomp_iter_static_next (long *, long *);
883 extern bool gomp_iter_dynamic_next_locked (long *, long *);
884 extern bool gomp_iter_guided_next_locked (long *, long *);
885
886 #ifdef HAVE_SYNC_BUILTINS
887 extern bool gomp_iter_dynamic_next (long *, long *);
888 extern bool gomp_iter_guided_next (long *, long *);
889 #endif
890
891 /* iter_ull.c */
892
893 extern int gomp_iter_ull_static_next (unsigned long long *,
894 unsigned long long *);
895 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
896 unsigned long long *);
897 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
898 unsigned long long *);
899
900 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
901 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
902 unsigned long long *);
903 extern bool gomp_iter_ull_guided_next (unsigned long long *,
904 unsigned long long *);
905 #endif
906
907 /* ordered.c */
908
909 extern void gomp_ordered_first (void);
910 extern void gomp_ordered_last (void);
911 extern void gomp_ordered_next (void);
912 extern void gomp_ordered_static_init (void);
913 extern void gomp_ordered_static_next (void);
914 extern void gomp_ordered_sync (void);
915 extern void gomp_doacross_init (unsigned, long *, long, size_t);
916 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
917 unsigned long long, size_t);
918
919 /* parallel.c */
920
921 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
922
923 /* proc.c (in config/) */
924
925 extern void gomp_init_num_threads (void);
926 extern unsigned gomp_dynamic_max_threads (void);
927
928 /* task.c */
929
930 extern void gomp_init_task (struct gomp_task *, struct gomp_task *,
931 struct gomp_task_icv *);
932 extern void gomp_end_task (void);
933 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);
934 extern void gomp_task_maybe_wait_for_dependencies (void **);
935 extern bool gomp_create_target_task (struct gomp_device_descr *,
936 void (*) (void *), size_t, void **,
937 size_t *, unsigned short *, unsigned int,
938 void **, void **,
939 enum gomp_target_task_state);
940 extern struct gomp_taskgroup *gomp_parallel_reduction_register (uintptr_t *,
941 unsigned);
942 extern void gomp_workshare_taskgroup_start (void);
943 extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);
944
945 static void inline
gomp_finish_task(struct gomp_task * task)946 gomp_finish_task (struct gomp_task *task)
947 {
948 if (__builtin_expect (task->depend_hash != NULL, 0))
949 free (task->depend_hash);
950 }
951
952 /* team.c */
953
954 extern struct gomp_team *gomp_new_team (unsigned);
955 extern void gomp_team_start (void (*) (void *), void *, unsigned,
956 unsigned, struct gomp_team *,
957 struct gomp_taskgroup *);
958 extern void gomp_team_end (void);
959 extern void gomp_free_thread (void *);
960 extern int gomp_pause_host (void);
961
962 /* target.c */
963
964 extern void gomp_init_targets_once (void);
965 extern int gomp_get_num_devices (void);
966 extern bool gomp_target_task_fn (void *);
967
968 /* Splay tree definitions. */
969 typedef struct splay_tree_node_s *splay_tree_node;
970 typedef struct splay_tree_s *splay_tree;
971 typedef struct splay_tree_key_s *splay_tree_key;
972
973 struct target_var_desc {
974 /* Splay key. */
975 splay_tree_key key;
976 /* True if data should be copied from device to host at the end. */
977 bool copy_from;
978 /* True if data always should be copied from device to host at the end. */
979 bool always_copy_from;
980 /* True if this is for OpenACC 'attach'. */
981 bool is_attach;
982 /* If GOMP_MAP_TO_PSET had a NULL pointer; used for Fortran descriptors,
983 which were initially unallocated. */
984 bool has_null_ptr_assoc;
985 /* Relative offset against key host_start. */
986 uintptr_t offset;
987 /* Actual length. */
988 uintptr_t length;
989 };
990
991 struct target_mem_desc {
992 /* Reference count. */
993 uintptr_t refcount;
994 /* All the splay nodes allocated together. */
995 splay_tree_node array;
996 /* Start of the target region. */
997 uintptr_t tgt_start;
998 /* End of the targer region. */
999 uintptr_t tgt_end;
1000 /* Handle to free. */
1001 void *to_free;
1002 /* Previous target_mem_desc. */
1003 struct target_mem_desc *prev;
1004 /* Number of items in following list. */
1005 size_t list_count;
1006
1007 /* Corresponding target device descriptor. */
1008 struct gomp_device_descr *device_descr;
1009
1010 /* List of target items to remove (or decrease refcount)
1011 at the end of region. */
1012 struct target_var_desc list[];
1013 };
1014
1015 /* Special value for refcount - infinity. */
1016 #define REFCOUNT_INFINITY (~(uintptr_t) 0)
1017 /* Special value for refcount - tgt_offset contains target address of the
1018 artificial pointer to "omp declare target link" object. */
1019 #define REFCOUNT_LINK (~(uintptr_t) 1)
1020
1021 /* Special offset values. */
1022 #define OFFSET_INLINED (~(uintptr_t) 0)
1023 #define OFFSET_POINTER (~(uintptr_t) 1)
1024 #define OFFSET_STRUCT (~(uintptr_t) 2)
1025
1026 /* Auxiliary structure for infrequently-used or API-specific data. */
1027
1028 struct splay_tree_aux {
1029 /* Pointer to the original mapping of "omp declare target link" object. */
1030 splay_tree_key link_key;
1031 /* For a block with attached pointers, the attachment counters for each.
1032 Only used for OpenACC. */
1033 uintptr_t *attach_count;
1034 };
1035
1036 struct splay_tree_key_s {
1037 /* Address of the host object. */
1038 uintptr_t host_start;
1039 /* Address immediately after the host object. */
1040 uintptr_t host_end;
1041 /* Descriptor of the target memory. */
1042 struct target_mem_desc *tgt;
1043 /* Offset from tgt->tgt_start to the start of the target object. */
1044 uintptr_t tgt_offset;
1045 /* Reference count. */
1046 uintptr_t refcount;
1047 /* Dynamic reference count. */
1048 uintptr_t dynamic_refcount;
1049 struct splay_tree_aux *aux;
1050 };
1051
1052 /* The comparison function. */
1053
1054 static inline int
splay_compare(splay_tree_key x,splay_tree_key y)1055 splay_compare (splay_tree_key x, splay_tree_key y)
1056 {
1057 if (x->host_start == x->host_end
1058 && y->host_start == y->host_end)
1059 return 0;
1060 if (x->host_end <= y->host_start)
1061 return -1;
1062 if (x->host_start >= y->host_end)
1063 return 1;
1064 return 0;
1065 }
1066
1067 #include "splay-tree.h"
1068
1069 typedef struct acc_dispatch_t
1070 {
1071 /* Execute. */
1072 __typeof (GOMP_OFFLOAD_openacc_exec) *exec_func;
1073
1074 /* Create/destroy TLS data. */
1075 __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func;
1076 __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data)
1077 *destroy_thread_data_func;
1078
1079 struct {
1080 /* Once created and put into the "active" list, asyncqueues are then never
1081 destructed and removed from the "active" list, other than if the TODO
1082 device is shut down. */
1083 gomp_mutex_t lock;
1084 int nasyncqueue;
1085 struct goacc_asyncqueue **asyncqueue;
1086 struct goacc_asyncqueue_list *active;
1087
1088 __typeof (GOMP_OFFLOAD_openacc_async_construct) *construct_func;
1089 __typeof (GOMP_OFFLOAD_openacc_async_destruct) *destruct_func;
1090 __typeof (GOMP_OFFLOAD_openacc_async_test) *test_func;
1091 __typeof (GOMP_OFFLOAD_openacc_async_synchronize) *synchronize_func;
1092 __typeof (GOMP_OFFLOAD_openacc_async_serialize) *serialize_func;
1093 __typeof (GOMP_OFFLOAD_openacc_async_queue_callback) *queue_callback_func;
1094
1095 __typeof (GOMP_OFFLOAD_openacc_async_exec) *exec_func;
1096 __typeof (GOMP_OFFLOAD_openacc_async_dev2host) *dev2host_func;
1097 __typeof (GOMP_OFFLOAD_openacc_async_host2dev) *host2dev_func;
1098 } async;
1099
1100 __typeof (GOMP_OFFLOAD_openacc_get_property) *get_property_func;
1101
1102 /* NVIDIA target specific routines. */
1103 struct {
1104 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device)
1105 *get_current_device_func;
1106 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context)
1107 *get_current_context_func;
1108 __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream) *get_stream_func;
1109 __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream) *set_stream_func;
1110 } cuda;
1111 } acc_dispatch_t;
1112
1113 /* Various state of the accelerator device. */
1114 enum gomp_device_state
1115 {
1116 GOMP_DEVICE_UNINITIALIZED,
1117 GOMP_DEVICE_INITIALIZED,
1118 GOMP_DEVICE_FINALIZED
1119 };
1120
1121 /* This structure describes accelerator device.
1122 It contains name of the corresponding libgomp plugin, function handlers for
1123 interaction with the device, ID-number of the device, and information about
1124 mapped memory. */
1125 struct gomp_device_descr
1126 {
1127 /* Immutable data, which is only set during initialization, and which is not
1128 guarded by the lock. */
1129
1130 /* The name of the device. */
1131 const char *name;
1132
1133 /* Capabilities of device (supports OpenACC, OpenMP). */
1134 unsigned int capabilities;
1135
1136 /* This is the ID number of device among devices of the same type. */
1137 int target_id;
1138
1139 /* This is the TYPE of device. */
1140 enum offload_target_type type;
1141
1142 /* Function handlers. */
1143 __typeof (GOMP_OFFLOAD_get_name) *get_name_func;
1144 __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func;
1145 __typeof (GOMP_OFFLOAD_get_type) *get_type_func;
1146 __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func;
1147 __typeof (GOMP_OFFLOAD_init_device) *init_device_func;
1148 __typeof (GOMP_OFFLOAD_fini_device) *fini_device_func;
1149 __typeof (GOMP_OFFLOAD_version) *version_func;
1150 __typeof (GOMP_OFFLOAD_load_image) *load_image_func;
1151 __typeof (GOMP_OFFLOAD_unload_image) *unload_image_func;
1152 __typeof (GOMP_OFFLOAD_alloc) *alloc_func;
1153 __typeof (GOMP_OFFLOAD_free) *free_func;
1154 __typeof (GOMP_OFFLOAD_dev2host) *dev2host_func;
1155 __typeof (GOMP_OFFLOAD_host2dev) *host2dev_func;
1156 __typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func;
1157 __typeof (GOMP_OFFLOAD_can_run) *can_run_func;
1158 __typeof (GOMP_OFFLOAD_run) *run_func;
1159 __typeof (GOMP_OFFLOAD_async_run) *async_run_func;
1160
1161 /* Splay tree containing information about mapped memory regions. */
1162 struct splay_tree_s mem_map;
1163
1164 /* Mutex for the mutable data. */
1165 gomp_mutex_t lock;
1166
1167 /* Current state of the device. OpenACC allows to move from INITIALIZED state
1168 back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED
1169 to FINALIZED state (at program shutdown). */
1170 enum gomp_device_state state;
1171
1172 /* OpenACC-specific data and functions. */
1173 /* This is mutable because of its mutable target_data member. */
1174 acc_dispatch_t openacc;
1175 };
1176
1177 /* Kind of the pragma, for which gomp_map_vars () is called. */
1178 enum gomp_map_vars_kind
1179 {
1180 GOMP_MAP_VARS_OPENACC = 1,
1181 GOMP_MAP_VARS_TARGET = 2,
1182 GOMP_MAP_VARS_DATA = 4,
1183 GOMP_MAP_VARS_ENTER_DATA = 8
1184 };
1185
1186 extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
1187 unsigned short *);
1188 struct gomp_coalesce_buf;
1189 extern void gomp_copy_host2dev (struct gomp_device_descr *,
1190 struct goacc_asyncqueue *, void *, const void *,
1191 size_t, struct gomp_coalesce_buf *);
1192 extern void gomp_copy_dev2host (struct gomp_device_descr *,
1193 struct goacc_asyncqueue *, void *, const void *,
1194 size_t);
1195 extern uintptr_t gomp_map_val (struct target_mem_desc *, void **, size_t);
1196 extern void gomp_attach_pointer (struct gomp_device_descr *,
1197 struct goacc_asyncqueue *, splay_tree,
1198 splay_tree_key, uintptr_t, size_t,
1199 struct gomp_coalesce_buf *);
1200 extern void gomp_detach_pointer (struct gomp_device_descr *,
1201 struct goacc_asyncqueue *, splay_tree_key,
1202 uintptr_t, bool, struct gomp_coalesce_buf *);
1203
1204 extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *,
1205 size_t, void **, void **,
1206 size_t *, void *, bool,
1207 enum gomp_map_vars_kind);
1208 extern struct target_mem_desc *gomp_map_vars_async (struct gomp_device_descr *,
1209 struct goacc_asyncqueue *,
1210 size_t, void **, void **,
1211 size_t *, void *, bool,
1212 enum gomp_map_vars_kind);
1213 extern void gomp_unmap_vars (struct target_mem_desc *, bool);
1214 extern void gomp_unmap_vars_async (struct target_mem_desc *, bool,
1215 struct goacc_asyncqueue *);
1216 extern void gomp_init_device (struct gomp_device_descr *);
1217 extern bool gomp_fini_device (struct gomp_device_descr *);
1218 extern void gomp_unload_device (struct gomp_device_descr *);
1219 extern bool gomp_remove_var (struct gomp_device_descr *, splay_tree_key);
1220 extern void gomp_remove_var_async (struct gomp_device_descr *, splay_tree_key,
1221 struct goacc_asyncqueue *);
1222
1223 /* work.c */
1224
1225 extern void gomp_init_work_share (struct gomp_work_share *, size_t, unsigned);
1226 extern void gomp_fini_work_share (struct gomp_work_share *);
1227 extern bool gomp_work_share_start (size_t);
1228 extern void gomp_work_share_end (void);
1229 extern bool gomp_work_share_end_cancel (void);
1230 extern void gomp_work_share_end_nowait (void);
1231
1232 static inline void
gomp_work_share_init_done(void)1233 gomp_work_share_init_done (void)
1234 {
1235 struct gomp_thread *thr = gomp_thread ();
1236 if (__builtin_expect (thr->ts.last_work_share != NULL, 1))
1237 gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share);
1238 }
1239
1240 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1241 # pragma GCC visibility pop
1242 #endif
1243
1244 /* Now that we're back to default visibility, include the globals. */
1245 #include "libgomp_g.h"
1246
1247 /* Include omp.h by parts. */
1248 #include "omp-lock.h"
1249 #define _LIBGOMP_OMP_LOCK_DEFINED 1
1250 #include "omp.h.in"
1251
1252 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
1253 || !defined (HAVE_ATTRIBUTE_ALIAS) \
1254 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
1255 || !defined (PIC) \
1256 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
1257 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
1258 #endif
1259
1260 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
1261 extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1262 extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1263 extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1264 extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1265 extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1266 extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1267 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1268 extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1269 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1270 extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1271
1272 extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1273 extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1274 extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1275 extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1276 extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1277 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1278 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1279 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1280 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1281 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1282
1283 # define omp_lock_symver(fn) \
1284 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
1285 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
1286 #else
1287 # define gomp_init_lock_30 omp_init_lock
1288 # define gomp_destroy_lock_30 omp_destroy_lock
1289 # define gomp_set_lock_30 omp_set_lock
1290 # define gomp_unset_lock_30 omp_unset_lock
1291 # define gomp_test_lock_30 omp_test_lock
1292 # define gomp_init_nest_lock_30 omp_init_nest_lock
1293 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
1294 # define gomp_set_nest_lock_30 omp_set_nest_lock
1295 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
1296 # define gomp_test_nest_lock_30 omp_test_nest_lock
1297 #endif
1298
1299 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1300 # define attribute_hidden __attribute__ ((visibility ("hidden")))
1301 #else
1302 # define attribute_hidden
1303 #endif
1304
1305 #if __GNUC__ >= 9
1306 # define HAVE_ATTRIBUTE_COPY
1307 #endif
1308
1309 #ifdef HAVE_ATTRIBUTE_COPY
1310 # define attribute_copy(arg) __attribute__ ((copy (arg)))
1311 #else
1312 # define attribute_copy(arg)
1313 #endif
1314
1315 #ifdef HAVE_ATTRIBUTE_ALIAS
1316 # define strong_alias(fn, al) \
1317 extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
1318
1319 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__)
1320 # define ialias_str1(x) ialias_str2(x)
1321 # define ialias_str2(x) #x
1322 # define ialias(fn) \
1323 extern __typeof (fn) gomp_ialias_##fn \
1324 __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
1325 # define ialias_redirect(fn) \
1326 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1327 # define ialias_call(fn) gomp_ialias_ ## fn
1328 #else
1329 # define ialias(fn)
1330 # define ialias_redirect(fn)
1331 # define ialias_call(fn) fn
1332 #endif
1333
1334 /* Helper function for priority_node_to_task() and
1335 task_to_priority_node().
1336
1337 Return the offset from a task to its priority_node entry. The
1338 priority_node entry is has a type of TYPE. */
1339
1340 static inline size_t
priority_queue_offset(enum priority_queue_type type)1341 priority_queue_offset (enum priority_queue_type type)
1342 {
1343 return offsetof (struct gomp_task, pnode[(int) type]);
1344 }
1345
1346 /* Return the task associated with a priority NODE of type TYPE. */
1347
1348 static inline struct gomp_task *
priority_node_to_task(enum priority_queue_type type,struct priority_node * node)1349 priority_node_to_task (enum priority_queue_type type,
1350 struct priority_node *node)
1351 {
1352 return (struct gomp_task *) ((char *) node - priority_queue_offset (type));
1353 }
1354
1355 /* Return the priority node of type TYPE for a given TASK. */
1356
1357 static inline struct priority_node *
task_to_priority_node(enum priority_queue_type type,struct gomp_task * task)1358 task_to_priority_node (enum priority_queue_type type,
1359 struct gomp_task *task)
1360 {
1361 return (struct priority_node *) ((char *) task
1362 + priority_queue_offset (type));
1363 }
1364
1365 #ifdef LIBGOMP_USE_PTHREADS
1366 static inline gomp_thread_handle
gomp_thread_self(void)1367 gomp_thread_self (void)
1368 {
1369 return pthread_self ();
1370 }
1371
1372 static inline gomp_thread_handle
gomp_thread_to_pthread_t(struct gomp_thread * thr)1373 gomp_thread_to_pthread_t (struct gomp_thread *thr)
1374 {
1375 struct gomp_thread *this_thr = gomp_thread ();
1376 if (thr == this_thr)
1377 return pthread_self ();
1378 #ifdef GOMP_NEEDS_THREAD_HANDLE
1379 return thr->handle;
1380 #else
1381 /* On Linux with initial-exec TLS, the pthread_t of the thread containing
1382 thr can be computed from thr, this_thr and pthread_self (),
1383 as the distance between this_thr and pthread_self () is constant. */
1384 return pthread_self () + ((uintptr_t) thr - (uintptr_t) this_thr);
1385 #endif
1386 }
1387 #else
1388 static inline gomp_thread_handle
gomp_thread_self(void)1389 gomp_thread_self (void)
1390 {
1391 return (gomp_thread_handle) {};
1392 }
1393
1394 static inline gomp_thread_handle
gomp_thread_to_pthread_t(struct gomp_thread * thr)1395 gomp_thread_to_pthread_t (struct gomp_thread *thr)
1396 {
1397 (void) thr;
1398 return gomp_thread_self ();
1399 }
1400 #endif
1401
1402 #endif /* LIBGOMP_H */
1403