1 /* Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 2 Free Software Foundation, Inc. 3 Contributed by Richard Henderson <rth@redhat.com>. 4 5 This file is part of the GNU OpenMP Library (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This file handles the maintainence of threads in response to team 27 creation and termination. */ 28 29 #include "libgomp.h" 30 #include <stdlib.h> 31 #include <string.h> 32 33 /* This attribute contains PTHREAD_CREATE_DETACHED. */ 34 pthread_attr_t gomp_thread_attr; 35 36 /* This key is for the thread destructor. */ 37 pthread_key_t gomp_thread_destructor; 38 39 40 /* This is the libgomp per-thread data structure. */ 41 #ifdef HAVE_TLS 42 __thread struct gomp_thread gomp_tls_data; 43 #else 44 pthread_key_t gomp_tls_key; 45 #endif 46 47 48 /* This structure is used to communicate across pthread_create. */ 49 50 struct gomp_thread_start_data 51 { 52 void (*fn) (void *); 53 void *fn_data; 54 struct gomp_team_state ts; 55 struct gomp_task *task; 56 struct gomp_thread_pool *thread_pool; 57 bool nested; 58 }; 59 60 61 /* This function is a pthread_create entry point. This contains the idle 62 loop in which a thread waits to be called up to become part of a team. */ 63 64 static void * 65 gomp_thread_start (void *xdata) 66 { 67 struct gomp_thread_start_data *data = xdata; 68 struct gomp_thread *thr; 69 struct gomp_thread_pool *pool; 70 void (*local_fn) (void *); 71 void *local_data; 72 73 #ifdef HAVE_TLS 74 thr = &gomp_tls_data; 75 #else 76 struct gomp_thread local_thr; 77 thr = &local_thr; 78 pthread_setspecific (gomp_tls_key, thr); 79 #endif 80 gomp_sem_init (&thr->release, 0); 81 82 /* Extract what we need from data. */ 83 local_fn = data->fn; 84 local_data = data->fn_data; 85 thr->thread_pool = data->thread_pool; 86 thr->ts = data->ts; 87 thr->task = data->task; 88 89 thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release; 90 91 /* Make thread pool local. */ 92 pool = thr->thread_pool; 93 94 if (data->nested) 95 { 96 struct gomp_team *team = thr->ts.team; 97 struct gomp_task *task = thr->task; 98 99 gomp_barrier_wait (&team->barrier); 100 101 local_fn (local_data); 102 gomp_team_barrier_wait (&team->barrier); 103 gomp_finish_task (task); 104 gomp_barrier_wait_last (&team->barrier); 105 } 106 else 107 { 108 pool->threads[thr->ts.team_id] = thr; 109 110 gomp_barrier_wait (&pool->threads_dock); 111 do 112 { 113 struct gomp_team *team = thr->ts.team; 114 struct gomp_task *task = thr->task; 115 116 local_fn (local_data); 117 gomp_team_barrier_wait (&team->barrier); 118 gomp_finish_task (task); 119 120 gomp_barrier_wait (&pool->threads_dock); 121 122 local_fn = thr->fn; 123 local_data = thr->data; 124 thr->fn = NULL; 125 } 126 while (local_fn); 127 } 128 129 gomp_sem_destroy (&thr->release); 130 return NULL; 131 } 132 133 134 /* Create a new team data structure. */ 135 136 struct gomp_team * 137 gomp_new_team (unsigned nthreads) 138 { 139 struct gomp_team *team; 140 size_t size; 141 int i; 142 143 size = sizeof (*team) + nthreads * (sizeof (team->ordered_release[0]) 144 + sizeof (team->implicit_task[0])); 145 team = gomp_malloc (size); 146 147 team->work_share_chunk = 8; 148 #ifdef HAVE_SYNC_BUILTINS 149 team->single_count = 0; 150 #else 151 gomp_mutex_init (&team->work_share_list_free_lock); 152 #endif 153 gomp_init_work_share (&team->work_shares[0], false, nthreads); 154 team->work_shares[0].next_alloc = NULL; 155 team->work_share_list_free = NULL; 156 team->work_share_list_alloc = &team->work_shares[1]; 157 for (i = 1; i < 7; i++) 158 team->work_shares[i].next_free = &team->work_shares[i + 1]; 159 team->work_shares[i].next_free = NULL; 160 161 team->nthreads = nthreads; 162 gomp_barrier_init (&team->barrier, nthreads); 163 164 gomp_sem_init (&team->master_release, 0); 165 team->ordered_release = (void *) &team->implicit_task[nthreads]; 166 team->ordered_release[0] = &team->master_release; 167 168 gomp_mutex_init (&team->task_lock); 169 team->task_queue = NULL; 170 team->task_count = 0; 171 team->task_running_count = 0; 172 173 return team; 174 } 175 176 177 /* Free a team data structure. */ 178 179 static void 180 free_team (struct gomp_team *team) 181 { 182 gomp_barrier_destroy (&team->barrier); 183 gomp_mutex_destroy (&team->task_lock); 184 free (team); 185 } 186 187 /* Allocate and initialize a thread pool. */ 188 189 static struct gomp_thread_pool *gomp_new_thread_pool (void) 190 { 191 struct gomp_thread_pool *pool 192 = gomp_malloc (sizeof(struct gomp_thread_pool)); 193 pool->threads = NULL; 194 pool->threads_size = 0; 195 pool->threads_used = 0; 196 pool->last_team = NULL; 197 return pool; 198 } 199 200 static void 201 gomp_free_pool_helper (void *thread_pool) 202 { 203 struct gomp_thread_pool *pool 204 = (struct gomp_thread_pool *) thread_pool; 205 gomp_barrier_wait_last (&pool->threads_dock); 206 gomp_sem_destroy (&gomp_thread ()->release); 207 pthread_exit (NULL); 208 } 209 210 /* Free a thread pool and release its threads. */ 211 212 static void 213 gomp_free_thread (void *arg __attribute__((unused))) 214 { 215 struct gomp_thread *thr = gomp_thread (); 216 struct gomp_thread_pool *pool = thr->thread_pool; 217 if (pool) 218 { 219 if (pool->threads_used > 0) 220 { 221 int i; 222 for (i = 1; i < pool->threads_used; i++) 223 { 224 struct gomp_thread *nthr = pool->threads[i]; 225 nthr->fn = gomp_free_pool_helper; 226 nthr->data = pool; 227 } 228 /* This barrier undocks threads docked on pool->threads_dock. */ 229 gomp_barrier_wait (&pool->threads_dock); 230 /* And this waits till all threads have called gomp_barrier_wait_last 231 in gomp_free_pool_helper. */ 232 gomp_barrier_wait (&pool->threads_dock); 233 /* Now it is safe to destroy the barrier and free the pool. */ 234 gomp_barrier_destroy (&pool->threads_dock); 235 } 236 free (pool->threads); 237 if (pool->last_team) 238 free_team (pool->last_team); 239 free (pool); 240 thr->thread_pool = NULL; 241 } 242 if (thr->task != NULL) 243 { 244 struct gomp_task *task = thr->task; 245 gomp_end_task (); 246 free (task); 247 } 248 } 249 250 /* Launch a team. */ 251 252 void 253 gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads, 254 struct gomp_team *team) 255 { 256 struct gomp_thread_start_data *start_data; 257 struct gomp_thread *thr, *nthr; 258 struct gomp_task *task; 259 struct gomp_task_icv *icv; 260 bool nested; 261 struct gomp_thread_pool *pool; 262 unsigned i, n, old_threads_used = 0; 263 pthread_attr_t thread_attr, *attr; 264 unsigned long nthreads_var; 265 266 thr = gomp_thread (); 267 nested = thr->ts.team != NULL; 268 if (__builtin_expect (thr->thread_pool == NULL, 0)) 269 { 270 thr->thread_pool = gomp_new_thread_pool (); 271 pthread_setspecific (gomp_thread_destructor, thr); 272 } 273 pool = thr->thread_pool; 274 task = thr->task; 275 icv = task ? &task->icv : &gomp_global_icv; 276 277 /* Always save the previous state, even if this isn't a nested team. 278 In particular, we should save any work share state from an outer 279 orphaned work share construct. */ 280 team->prev_ts = thr->ts; 281 282 thr->ts.team = team; 283 thr->ts.team_id = 0; 284 ++thr->ts.level; 285 if (nthreads > 1) 286 ++thr->ts.active_level; 287 thr->ts.work_share = &team->work_shares[0]; 288 thr->ts.last_work_share = NULL; 289 #ifdef HAVE_SYNC_BUILTINS 290 thr->ts.single_count = 0; 291 #endif 292 thr->ts.static_trip = 0; 293 thr->task = &team->implicit_task[0]; 294 nthreads_var = icv->nthreads_var; 295 if (__builtin_expect (gomp_nthreads_var_list != NULL, 0) 296 && thr->ts.level < gomp_nthreads_var_list_len) 297 nthreads_var = gomp_nthreads_var_list[thr->ts.level]; 298 gomp_init_task (thr->task, task, icv); 299 team->implicit_task[0].icv.nthreads_var = nthreads_var; 300 301 if (nthreads == 1) 302 return; 303 304 i = 1; 305 306 /* We only allow the reuse of idle threads for non-nested PARALLEL 307 regions. This appears to be implied by the semantics of 308 threadprivate variables, but perhaps that's reading too much into 309 things. Certainly it does prevent any locking problems, since 310 only the initial program thread will modify gomp_threads. */ 311 if (!nested) 312 { 313 old_threads_used = pool->threads_used; 314 315 if (nthreads <= old_threads_used) 316 n = nthreads; 317 else if (old_threads_used == 0) 318 { 319 n = 0; 320 gomp_barrier_init (&pool->threads_dock, nthreads); 321 } 322 else 323 { 324 n = old_threads_used; 325 326 /* Increase the barrier threshold to make sure all new 327 threads arrive before the team is released. */ 328 gomp_barrier_reinit (&pool->threads_dock, nthreads); 329 } 330 331 /* Not true yet, but soon will be. We're going to release all 332 threads from the dock, and those that aren't part of the 333 team will exit. */ 334 pool->threads_used = nthreads; 335 336 /* Release existing idle threads. */ 337 for (; i < n; ++i) 338 { 339 nthr = pool->threads[i]; 340 nthr->ts.team = team; 341 nthr->ts.work_share = &team->work_shares[0]; 342 nthr->ts.last_work_share = NULL; 343 nthr->ts.team_id = i; 344 nthr->ts.level = team->prev_ts.level + 1; 345 nthr->ts.active_level = thr->ts.active_level; 346 #ifdef HAVE_SYNC_BUILTINS 347 nthr->ts.single_count = 0; 348 #endif 349 nthr->ts.static_trip = 0; 350 nthr->task = &team->implicit_task[i]; 351 gomp_init_task (nthr->task, task, icv); 352 team->implicit_task[i].icv.nthreads_var = nthreads_var; 353 nthr->fn = fn; 354 nthr->data = data; 355 team->ordered_release[i] = &nthr->release; 356 } 357 358 if (i == nthreads) 359 goto do_release; 360 361 /* If necessary, expand the size of the gomp_threads array. It is 362 expected that changes in the number of threads are rare, thus we 363 make no effort to expand gomp_threads_size geometrically. */ 364 if (nthreads >= pool->threads_size) 365 { 366 pool->threads_size = nthreads + 1; 367 pool->threads 368 = gomp_realloc (pool->threads, 369 pool->threads_size 370 * sizeof (struct gomp_thread_data *)); 371 } 372 } 373 374 if (__builtin_expect (nthreads > old_threads_used, 0)) 375 { 376 long diff = (long) nthreads - (long) old_threads_used; 377 378 if (old_threads_used == 0) 379 --diff; 380 381 #ifdef HAVE_SYNC_BUILTINS 382 __sync_fetch_and_add (&gomp_managed_threads, diff); 383 #else 384 gomp_mutex_lock (&gomp_remaining_threads_lock); 385 gomp_managed_threads += diff; 386 gomp_mutex_unlock (&gomp_remaining_threads_lock); 387 #endif 388 } 389 390 attr = &gomp_thread_attr; 391 if (__builtin_expect (gomp_cpu_affinity != NULL, 0)) 392 { 393 size_t stacksize; 394 pthread_attr_init (&thread_attr); 395 pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_DETACHED); 396 if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize)) 397 pthread_attr_setstacksize (&thread_attr, stacksize); 398 attr = &thread_attr; 399 } 400 401 start_data = gomp_alloca (sizeof (struct gomp_thread_start_data) 402 * (nthreads-i)); 403 404 /* Launch new threads. */ 405 for (; i < nthreads; ++i, ++start_data) 406 { 407 pthread_t pt; 408 int err; 409 410 start_data->fn = fn; 411 start_data->fn_data = data; 412 start_data->ts.team = team; 413 start_data->ts.work_share = &team->work_shares[0]; 414 start_data->ts.last_work_share = NULL; 415 start_data->ts.team_id = i; 416 start_data->ts.level = team->prev_ts.level + 1; 417 start_data->ts.active_level = thr->ts.active_level; 418 #ifdef HAVE_SYNC_BUILTINS 419 start_data->ts.single_count = 0; 420 #endif 421 start_data->ts.static_trip = 0; 422 start_data->task = &team->implicit_task[i]; 423 gomp_init_task (start_data->task, task, icv); 424 team->implicit_task[i].icv.nthreads_var = nthreads_var; 425 start_data->thread_pool = pool; 426 start_data->nested = nested; 427 428 if (gomp_cpu_affinity != NULL) 429 gomp_init_thread_affinity (attr); 430 431 err = pthread_create (&pt, attr, gomp_thread_start, start_data); 432 if (err != 0) 433 gomp_fatal ("Thread creation failed: %s", strerror (err)); 434 } 435 436 if (__builtin_expect (gomp_cpu_affinity != NULL, 0)) 437 pthread_attr_destroy (&thread_attr); 438 439 do_release: 440 gomp_barrier_wait (nested ? &team->barrier : &pool->threads_dock); 441 442 /* Decrease the barrier threshold to match the number of threads 443 that should arrive back at the end of this team. The extra 444 threads should be exiting. Note that we arrange for this test 445 to never be true for nested teams. */ 446 if (__builtin_expect (nthreads < old_threads_used, 0)) 447 { 448 long diff = (long) nthreads - (long) old_threads_used; 449 450 gomp_barrier_reinit (&pool->threads_dock, nthreads); 451 452 #ifdef HAVE_SYNC_BUILTINS 453 __sync_fetch_and_add (&gomp_managed_threads, diff); 454 #else 455 gomp_mutex_lock (&gomp_remaining_threads_lock); 456 gomp_managed_threads += diff; 457 gomp_mutex_unlock (&gomp_remaining_threads_lock); 458 #endif 459 } 460 } 461 462 463 /* Terminate the current team. This is only to be called by the master 464 thread. We assume that we must wait for the other threads. */ 465 466 void 467 gomp_team_end (void) 468 { 469 struct gomp_thread *thr = gomp_thread (); 470 struct gomp_team *team = thr->ts.team; 471 472 /* This barrier handles all pending explicit threads. */ 473 gomp_team_barrier_wait (&team->barrier); 474 gomp_fini_work_share (thr->ts.work_share); 475 476 gomp_end_task (); 477 thr->ts = team->prev_ts; 478 479 if (__builtin_expect (thr->ts.team != NULL, 0)) 480 { 481 #ifdef HAVE_SYNC_BUILTINS 482 __sync_fetch_and_add (&gomp_managed_threads, 1L - team->nthreads); 483 #else 484 gomp_mutex_lock (&gomp_remaining_threads_lock); 485 gomp_managed_threads -= team->nthreads - 1L; 486 gomp_mutex_unlock (&gomp_remaining_threads_lock); 487 #endif 488 /* This barrier has gomp_barrier_wait_last counterparts 489 and ensures the team can be safely destroyed. */ 490 gomp_barrier_wait (&team->barrier); 491 } 492 493 if (__builtin_expect (team->work_shares[0].next_alloc != NULL, 0)) 494 { 495 struct gomp_work_share *ws = team->work_shares[0].next_alloc; 496 do 497 { 498 struct gomp_work_share *next_ws = ws->next_alloc; 499 free (ws); 500 ws = next_ws; 501 } 502 while (ws != NULL); 503 } 504 gomp_sem_destroy (&team->master_release); 505 #ifndef HAVE_SYNC_BUILTINS 506 gomp_mutex_destroy (&team->work_share_list_free_lock); 507 #endif 508 509 if (__builtin_expect (thr->ts.team != NULL, 0) 510 || __builtin_expect (team->nthreads == 1, 0)) 511 free_team (team); 512 else 513 { 514 struct gomp_thread_pool *pool = thr->thread_pool; 515 if (pool->last_team) 516 free_team (pool->last_team); 517 pool->last_team = team; 518 } 519 } 520 521 522 /* Constructors for this file. */ 523 524 static void __attribute__((constructor)) 525 initialize_team (void) 526 { 527 struct gomp_thread *thr; 528 529 #ifndef HAVE_TLS 530 static struct gomp_thread initial_thread_tls_data; 531 532 pthread_key_create (&gomp_tls_key, NULL); 533 pthread_setspecific (gomp_tls_key, &initial_thread_tls_data); 534 #endif 535 536 if (pthread_key_create (&gomp_thread_destructor, gomp_free_thread) != 0) 537 gomp_fatal ("could not create thread pool destructor."); 538 539 #ifdef HAVE_TLS 540 thr = &gomp_tls_data; 541 #else 542 thr = &initial_thread_tls_data; 543 #endif 544 gomp_sem_init (&thr->release, 0); 545 } 546 547 static void __attribute__((destructor)) 548 team_destructor (void) 549 { 550 /* Without this dlclose on libgomp could lead to subsequent 551 crashes. */ 552 pthread_key_delete (gomp_thread_destructor); 553 } 554 555 struct gomp_task_icv * 556 gomp_new_icv (void) 557 { 558 struct gomp_thread *thr = gomp_thread (); 559 struct gomp_task *task = gomp_malloc (sizeof (struct gomp_task)); 560 gomp_init_task (task, NULL, &gomp_global_icv); 561 thr->task = task; 562 pthread_setspecific (gomp_thread_destructor, thr); 563 return &task->icv; 564 } 565