1 /* Copyright (C) 2005-2018 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Offloading and Multi Processing Library 5 (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This is the default implementation of a barrier synchronization mechanism 27 for libgomp. This type is private to the library. Note that we rely on 28 being able to adjust the barrier count while threads are blocked, so the 29 POSIX pthread_barrier_t won't work. */ 30 31 #include "libgomp.h" 32 33 34 void 35 gomp_barrier_init (gomp_barrier_t *bar, unsigned count) 36 { 37 gomp_mutex_init (&bar->mutex1); 38 #ifndef HAVE_SYNC_BUILTINS 39 gomp_mutex_init (&bar->mutex2); 40 #endif 41 gomp_sem_init (&bar->sem1, 0); 42 gomp_sem_init (&bar->sem2, 0); 43 bar->total = count; 44 bar->arrived = 0; 45 bar->generation = 0; 46 bar->cancellable = false; 47 } 48 49 void 50 gomp_barrier_destroy (gomp_barrier_t *bar) 51 { 52 /* Before destroying, make sure all threads have left the barrier. */ 53 gomp_mutex_lock (&bar->mutex1); 54 gomp_mutex_unlock (&bar->mutex1); 55 56 gomp_mutex_destroy (&bar->mutex1); 57 #ifndef HAVE_SYNC_BUILTINS 58 gomp_mutex_destroy (&bar->mutex2); 59 #endif 60 gomp_sem_destroy (&bar->sem1); 61 gomp_sem_destroy (&bar->sem2); 62 } 63 64 void 65 gomp_barrier_reinit (gomp_barrier_t *bar, unsigned count) 66 { 67 gomp_mutex_lock (&bar->mutex1); 68 bar->total = count; 69 gomp_mutex_unlock (&bar->mutex1); 70 } 71 72 void 73 gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) 74 { 75 unsigned int n; 76 77 if (state & BAR_WAS_LAST) 78 { 79 n = --bar->arrived; 80 if (n > 0) 81 { 82 do 83 gomp_sem_post (&bar->sem1); 84 while (--n != 0); 85 gomp_sem_wait (&bar->sem2); 86 } 87 gomp_mutex_unlock (&bar->mutex1); 88 } 89 else 90 { 91 gomp_mutex_unlock (&bar->mutex1); 92 gomp_sem_wait (&bar->sem1); 93 94 #ifdef HAVE_SYNC_BUILTINS 95 n = __sync_add_and_fetch (&bar->arrived, -1); 96 #else 97 gomp_mutex_lock (&bar->mutex2); 98 n = --bar->arrived; 99 gomp_mutex_unlock (&bar->mutex2); 100 #endif 101 102 if (n == 0) 103 gomp_sem_post (&bar->sem2); 104 } 105 } 106 107 void 108 gomp_barrier_wait (gomp_barrier_t *barrier) 109 { 110 gomp_barrier_wait_end (barrier, gomp_barrier_wait_start (barrier)); 111 } 112 113 void 114 gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) 115 { 116 unsigned int n; 117 118 state &= ~BAR_CANCELLED; 119 if (state & BAR_WAS_LAST) 120 { 121 n = --bar->arrived; 122 struct gomp_thread *thr = gomp_thread (); 123 struct gomp_team *team = thr->ts.team; 124 125 team->work_share_cancelled = 0; 126 if (team->task_count) 127 { 128 gomp_barrier_handle_tasks (state); 129 if (n > 0) 130 gomp_sem_wait (&bar->sem2); 131 gomp_mutex_unlock (&bar->mutex1); 132 return; 133 } 134 135 bar->generation = state + BAR_INCR - BAR_WAS_LAST; 136 if (n > 0) 137 { 138 do 139 gomp_sem_post (&bar->sem1); 140 while (--n != 0); 141 gomp_sem_wait (&bar->sem2); 142 } 143 gomp_mutex_unlock (&bar->mutex1); 144 } 145 else 146 { 147 gomp_mutex_unlock (&bar->mutex1); 148 int gen; 149 do 150 { 151 gomp_sem_wait (&bar->sem1); 152 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 153 if (gen & BAR_TASK_PENDING) 154 { 155 gomp_barrier_handle_tasks (state); 156 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 157 } 158 } 159 while (gen != state + BAR_INCR); 160 161 #ifdef HAVE_SYNC_BUILTINS 162 n = __sync_add_and_fetch (&bar->arrived, -1); 163 #else 164 gomp_mutex_lock (&bar->mutex2); 165 n = --bar->arrived; 166 gomp_mutex_unlock (&bar->mutex2); 167 #endif 168 169 if (n == 0) 170 gomp_sem_post (&bar->sem2); 171 } 172 } 173 174 bool 175 gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar, 176 gomp_barrier_state_t state) 177 { 178 unsigned int n; 179 180 if (state & BAR_WAS_LAST) 181 { 182 bar->cancellable = false; 183 n = --bar->arrived; 184 struct gomp_thread *thr = gomp_thread (); 185 struct gomp_team *team = thr->ts.team; 186 187 team->work_share_cancelled = 0; 188 if (team->task_count) 189 { 190 gomp_barrier_handle_tasks (state); 191 if (n > 0) 192 gomp_sem_wait (&bar->sem2); 193 gomp_mutex_unlock (&bar->mutex1); 194 return false; 195 } 196 197 bar->generation = state + BAR_INCR - BAR_WAS_LAST; 198 if (n > 0) 199 { 200 do 201 gomp_sem_post (&bar->sem1); 202 while (--n != 0); 203 gomp_sem_wait (&bar->sem2); 204 } 205 gomp_mutex_unlock (&bar->mutex1); 206 } 207 else 208 { 209 if (state & BAR_CANCELLED) 210 { 211 gomp_mutex_unlock (&bar->mutex1); 212 return true; 213 } 214 bar->cancellable = true; 215 gomp_mutex_unlock (&bar->mutex1); 216 int gen; 217 do 218 { 219 gomp_sem_wait (&bar->sem1); 220 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 221 if (gen & BAR_CANCELLED) 222 break; 223 if (gen & BAR_TASK_PENDING) 224 { 225 gomp_barrier_handle_tasks (state); 226 gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 227 if (gen & BAR_CANCELLED) 228 break; 229 } 230 } 231 while (gen != state + BAR_INCR); 232 233 #ifdef HAVE_SYNC_BUILTINS 234 n = __sync_add_and_fetch (&bar->arrived, -1); 235 #else 236 gomp_mutex_lock (&bar->mutex2); 237 n = --bar->arrived; 238 gomp_mutex_unlock (&bar->mutex2); 239 #endif 240 241 if (n == 0) 242 gomp_sem_post (&bar->sem2); 243 if (gen & BAR_CANCELLED) 244 return true; 245 } 246 return false; 247 } 248 249 void 250 gomp_team_barrier_wait (gomp_barrier_t *barrier) 251 { 252 gomp_team_barrier_wait_end (barrier, gomp_barrier_wait_start (barrier)); 253 } 254 255 void 256 gomp_team_barrier_wake (gomp_barrier_t *bar, int count) 257 { 258 if (count == 0) 259 count = bar->total - 1; 260 while (count-- > 0) 261 gomp_sem_post (&bar->sem1); 262 } 263 264 bool 265 gomp_team_barrier_wait_cancel (gomp_barrier_t *bar) 266 { 267 gomp_barrier_state_t state = gomp_barrier_wait_cancel_start (bar); 268 return gomp_team_barrier_wait_cancel_end (bar, state); 269 } 270 271 void 272 gomp_team_barrier_cancel (struct gomp_team *team) 273 { 274 if (team->barrier.generation & BAR_CANCELLED) 275 return; 276 gomp_mutex_lock (&team->barrier.mutex1); 277 gomp_mutex_lock (&team->task_lock); 278 if (team->barrier.generation & BAR_CANCELLED) 279 { 280 gomp_mutex_unlock (&team->task_lock); 281 gomp_mutex_unlock (&team->barrier.mutex1); 282 return; 283 } 284 team->barrier.generation |= BAR_CANCELLED; 285 gomp_mutex_unlock (&team->task_lock); 286 if (team->barrier.cancellable) 287 { 288 int n = team->barrier.arrived; 289 if (n > 0) 290 { 291 do 292 gomp_sem_post (&team->barrier.sem1); 293 while (--n != 0); 294 gomp_sem_wait (&team->barrier.sem2); 295 } 296 team->barrier.cancellable = false; 297 } 298 gomp_mutex_unlock (&team->barrier.mutex1); 299 } 300