1 /* Copyright (C) 2005-2018 Free Software Foundation, Inc.
2    Contributed by Richard Henderson <rth@redhat.com>.
3 
4    This file is part of the GNU Offloading and Multi Processing Library
5    (libgomp).
6 
7    Libgomp is free software; you can redistribute it and/or modify it
8    under the terms of the GNU General Public License as published by
9    the Free Software Foundation; either version 3, or (at your option)
10    any later version.
11 
12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15    more details.
16 
17    Under Section 7 of GPL version 3, you are granted additional
18    permissions described in the GCC Runtime Library Exception, version
19    3.1, as published by the Free Software Foundation.
20 
21    You should have received a copy of the GNU General Public License and
22    a copy of the GCC Runtime Library Exception along with this program;
23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
24    <http://www.gnu.org/licenses/>.  */
25 
26 /* This is the default implementation of a barrier synchronization mechanism
27    for libgomp.  This type is private to the library.  Note that we rely on
28    being able to adjust the barrier count while threads are blocked, so the
29    POSIX pthread_barrier_t won't work.  */
30 
31 #include "libgomp.h"
32 
33 
34 void
35 gomp_barrier_init (gomp_barrier_t *bar, unsigned count)
36 {
37   gomp_mutex_init (&bar->mutex1);
38 #ifndef HAVE_SYNC_BUILTINS
39   gomp_mutex_init (&bar->mutex2);
40 #endif
41   gomp_sem_init (&bar->sem1, 0);
42   gomp_sem_init (&bar->sem2, 0);
43   bar->total = count;
44   bar->arrived = 0;
45   bar->generation = 0;
46   bar->cancellable = false;
47 }
48 
49 void
50 gomp_barrier_destroy (gomp_barrier_t *bar)
51 {
52   /* Before destroying, make sure all threads have left the barrier.  */
53   gomp_mutex_lock (&bar->mutex1);
54   gomp_mutex_unlock (&bar->mutex1);
55 
56   gomp_mutex_destroy (&bar->mutex1);
57 #ifndef HAVE_SYNC_BUILTINS
58   gomp_mutex_destroy (&bar->mutex2);
59 #endif
60   gomp_sem_destroy (&bar->sem1);
61   gomp_sem_destroy (&bar->sem2);
62 }
63 
64 void
65 gomp_barrier_reinit (gomp_barrier_t *bar, unsigned count)
66 {
67   gomp_mutex_lock (&bar->mutex1);
68   bar->total = count;
69   gomp_mutex_unlock (&bar->mutex1);
70 }
71 
72 void
73 gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
74 {
75   unsigned int n;
76 
77   if (state & BAR_WAS_LAST)
78     {
79       n = --bar->arrived;
80       if (n > 0)
81 	{
82 	  do
83 	    gomp_sem_post (&bar->sem1);
84 	  while (--n != 0);
85 	  gomp_sem_wait (&bar->sem2);
86 	}
87       gomp_mutex_unlock (&bar->mutex1);
88     }
89   else
90     {
91       gomp_mutex_unlock (&bar->mutex1);
92       gomp_sem_wait (&bar->sem1);
93 
94 #ifdef HAVE_SYNC_BUILTINS
95       n = __sync_add_and_fetch (&bar->arrived, -1);
96 #else
97       gomp_mutex_lock (&bar->mutex2);
98       n = --bar->arrived;
99       gomp_mutex_unlock (&bar->mutex2);
100 #endif
101 
102       if (n == 0)
103 	gomp_sem_post (&bar->sem2);
104     }
105 }
106 
107 void
108 gomp_barrier_wait (gomp_barrier_t *barrier)
109 {
110   gomp_barrier_wait_end (barrier, gomp_barrier_wait_start (barrier));
111 }
112 
113 void
114 gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
115 {
116   unsigned int n;
117 
118   state &= ~BAR_CANCELLED;
119   if (state & BAR_WAS_LAST)
120     {
121       n = --bar->arrived;
122       struct gomp_thread *thr = gomp_thread ();
123       struct gomp_team *team = thr->ts.team;
124 
125       team->work_share_cancelled = 0;
126       if (team->task_count)
127 	{
128 	  gomp_barrier_handle_tasks (state);
129 	  if (n > 0)
130 	    gomp_sem_wait (&bar->sem2);
131 	  gomp_mutex_unlock (&bar->mutex1);
132 	  return;
133 	}
134 
135       bar->generation = state + BAR_INCR - BAR_WAS_LAST;
136       if (n > 0)
137 	{
138 	  do
139 	    gomp_sem_post (&bar->sem1);
140 	  while (--n != 0);
141 	  gomp_sem_wait (&bar->sem2);
142 	}
143       gomp_mutex_unlock (&bar->mutex1);
144     }
145   else
146     {
147       gomp_mutex_unlock (&bar->mutex1);
148       int gen;
149       do
150 	{
151 	  gomp_sem_wait (&bar->sem1);
152 	  gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
153 	  if (gen & BAR_TASK_PENDING)
154 	    {
155 	      gomp_barrier_handle_tasks (state);
156 	      gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
157 	    }
158 	}
159       while (gen != state + BAR_INCR);
160 
161 #ifdef HAVE_SYNC_BUILTINS
162       n = __sync_add_and_fetch (&bar->arrived, -1);
163 #else
164       gomp_mutex_lock (&bar->mutex2);
165       n = --bar->arrived;
166       gomp_mutex_unlock (&bar->mutex2);
167 #endif
168 
169       if (n == 0)
170 	gomp_sem_post (&bar->sem2);
171     }
172 }
173 
174 bool
175 gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar,
176 				   gomp_barrier_state_t state)
177 {
178   unsigned int n;
179 
180   if (state & BAR_WAS_LAST)
181     {
182       bar->cancellable = false;
183       n = --bar->arrived;
184       struct gomp_thread *thr = gomp_thread ();
185       struct gomp_team *team = thr->ts.team;
186 
187       team->work_share_cancelled = 0;
188       if (team->task_count)
189 	{
190 	  gomp_barrier_handle_tasks (state);
191 	  if (n > 0)
192 	    gomp_sem_wait (&bar->sem2);
193 	  gomp_mutex_unlock (&bar->mutex1);
194 	  return false;
195 	}
196 
197       bar->generation = state + BAR_INCR - BAR_WAS_LAST;
198       if (n > 0)
199 	{
200 	  do
201 	    gomp_sem_post (&bar->sem1);
202 	  while (--n != 0);
203 	  gomp_sem_wait (&bar->sem2);
204 	}
205       gomp_mutex_unlock (&bar->mutex1);
206     }
207   else
208     {
209       if (state & BAR_CANCELLED)
210 	{
211 	  gomp_mutex_unlock (&bar->mutex1);
212 	  return true;
213 	}
214       bar->cancellable = true;
215       gomp_mutex_unlock (&bar->mutex1);
216       int gen;
217       do
218 	{
219 	  gomp_sem_wait (&bar->sem1);
220 	  gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
221 	  if (gen & BAR_CANCELLED)
222 	    break;
223 	  if (gen & BAR_TASK_PENDING)
224 	    {
225 	      gomp_barrier_handle_tasks (state);
226 	      gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
227 	      if (gen & BAR_CANCELLED)
228 		break;
229 	    }
230 	}
231       while (gen != state + BAR_INCR);
232 
233 #ifdef HAVE_SYNC_BUILTINS
234       n = __sync_add_and_fetch (&bar->arrived, -1);
235 #else
236       gomp_mutex_lock (&bar->mutex2);
237       n = --bar->arrived;
238       gomp_mutex_unlock (&bar->mutex2);
239 #endif
240 
241       if (n == 0)
242 	gomp_sem_post (&bar->sem2);
243       if (gen & BAR_CANCELLED)
244 	return true;
245     }
246   return false;
247 }
248 
249 void
250 gomp_team_barrier_wait (gomp_barrier_t *barrier)
251 {
252   gomp_team_barrier_wait_end (barrier, gomp_barrier_wait_start (barrier));
253 }
254 
255 void
256 gomp_team_barrier_wake (gomp_barrier_t *bar, int count)
257 {
258   if (count == 0)
259     count = bar->total - 1;
260   while (count-- > 0)
261     gomp_sem_post (&bar->sem1);
262 }
263 
264 bool
265 gomp_team_barrier_wait_cancel (gomp_barrier_t *bar)
266 {
267   gomp_barrier_state_t state = gomp_barrier_wait_cancel_start (bar);
268   return gomp_team_barrier_wait_cancel_end (bar, state);
269 }
270 
271 void
272 gomp_team_barrier_cancel (struct gomp_team *team)
273 {
274   if (team->barrier.generation & BAR_CANCELLED)
275     return;
276   gomp_mutex_lock (&team->barrier.mutex1);
277   gomp_mutex_lock (&team->task_lock);
278   if (team->barrier.generation & BAR_CANCELLED)
279     {
280       gomp_mutex_unlock (&team->task_lock);
281       gomp_mutex_unlock (&team->barrier.mutex1);
282       return;
283     }
284   team->barrier.generation |= BAR_CANCELLED;
285   gomp_mutex_unlock (&team->task_lock);
286   if (team->barrier.cancellable)
287     {
288       int n = team->barrier.arrived;
289       if (n > 0)
290 	{
291 	  do
292 	    gomp_sem_post (&team->barrier.sem1);
293 	  while (--n != 0);
294 	  gomp_sem_wait (&team->barrier.sem2);
295 	}
296       team->barrier.cancellable = false;
297     }
298   gomp_mutex_unlock (&team->barrier.mutex1);
299 }
300