1 /* 2 * kmp_tasking.cpp -- OpenMP 3.0 tasking support. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_i18n.h" 15 #include "kmp_itt.h" 16 #include "kmp_stats.h" 17 #include "kmp_wait_release.h" 18 #include "kmp_taskdeps.h" 19 20 #if OMPT_SUPPORT 21 #include "ompt-specific.h" 22 #endif 23 24 #if ENABLE_LIBOMPTARGET 25 // Declaration of synchronization function from libomptarget. 26 extern "C" void __tgt_target_nowait_query(void **) KMP_WEAK_ATTRIBUTE_INTERNAL; 27 #endif 28 29 /* forward declaration */ 30 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 31 kmp_info_t *this_thr); 32 static void __kmp_alloc_task_deque(kmp_info_t *thread, 33 kmp_thread_data_t *thread_data); 34 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 35 kmp_task_team_t *task_team); 36 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask); 37 38 #ifdef BUILD_TIED_TASK_STACK 39 40 // __kmp_trace_task_stack: print the tied tasks from the task stack in order 41 // from top do bottom 42 // 43 // gtid: global thread identifier for thread containing stack 44 // thread_data: thread data for task team thread containing stack 45 // threshold: value above which the trace statement triggers 46 // location: string identifying call site of this function (for trace) 47 static void __kmp_trace_task_stack(kmp_int32 gtid, 48 kmp_thread_data_t *thread_data, 49 int threshold, char *location) { 50 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 51 kmp_taskdata_t **stack_top = task_stack->ts_top; 52 kmp_int32 entries = task_stack->ts_entries; 53 kmp_taskdata_t *tied_task; 54 55 KA_TRACE( 56 threshold, 57 ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, " 58 "first_block = %p, stack_top = %p \n", 59 location, gtid, entries, task_stack->ts_first_block, stack_top)); 60 61 KMP_DEBUG_ASSERT(stack_top != NULL); 62 KMP_DEBUG_ASSERT(entries > 0); 63 64 while (entries != 0) { 65 KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0]); 66 // fix up ts_top if we need to pop from previous block 67 if (entries & TASK_STACK_INDEX_MASK == 0) { 68 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top); 69 70 stack_block = stack_block->sb_prev; 71 stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 72 } 73 74 // finish bookkeeping 75 stack_top--; 76 entries--; 77 78 tied_task = *stack_top; 79 80 KMP_DEBUG_ASSERT(tied_task != NULL); 81 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 82 83 KA_TRACE(threshold, 84 ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, " 85 "stack_top=%p, tied_task=%p\n", 86 location, gtid, entries, stack_top, tied_task)); 87 } 88 KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0]); 89 90 KA_TRACE(threshold, 91 ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n", 92 location, gtid)); 93 } 94 95 // __kmp_init_task_stack: initialize the task stack for the first time 96 // after a thread_data structure is created. 97 // It should not be necessary to do this again (assuming the stack works). 98 // 99 // gtid: global thread identifier of calling thread 100 // thread_data: thread data for task team thread containing stack 101 static void __kmp_init_task_stack(kmp_int32 gtid, 102 kmp_thread_data_t *thread_data) { 103 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 104 kmp_stack_block_t *first_block; 105 106 // set up the first block of the stack 107 first_block = &task_stack->ts_first_block; 108 task_stack->ts_top = (kmp_taskdata_t **)first_block; 109 memset((void *)first_block, '\0', 110 TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *)); 111 112 // initialize the stack to be empty 113 task_stack->ts_entries = TASK_STACK_EMPTY; 114 first_block->sb_next = NULL; 115 first_block->sb_prev = NULL; 116 } 117 118 // __kmp_free_task_stack: free the task stack when thread_data is destroyed. 119 // 120 // gtid: global thread identifier for calling thread 121 // thread_data: thread info for thread containing stack 122 static void __kmp_free_task_stack(kmp_int32 gtid, 123 kmp_thread_data_t *thread_data) { 124 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 125 kmp_stack_block_t *stack_block = &task_stack->ts_first_block; 126 127 KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY); 128 // free from the second block of the stack 129 while (stack_block != NULL) { 130 kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL; 131 132 stack_block->sb_next = NULL; 133 stack_block->sb_prev = NULL; 134 if (stack_block != &task_stack->ts_first_block) { 135 __kmp_thread_free(thread, 136 stack_block); // free the block, if not the first 137 } 138 stack_block = next_block; 139 } 140 // initialize the stack to be empty 141 task_stack->ts_entries = 0; 142 task_stack->ts_top = NULL; 143 } 144 145 // __kmp_push_task_stack: Push the tied task onto the task stack. 146 // Grow the stack if necessary by allocating another block. 147 // 148 // gtid: global thread identifier for calling thread 149 // thread: thread info for thread containing stack 150 // tied_task: the task to push on the stack 151 static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread, 152 kmp_taskdata_t *tied_task) { 153 // GEH - need to consider what to do if tt_threads_data not allocated yet 154 kmp_thread_data_t *thread_data = 155 &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 156 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 157 158 if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) { 159 return; // Don't push anything on stack if team or team tasks are serialized 160 } 161 162 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 163 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 164 165 KA_TRACE(20, 166 ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n", 167 gtid, thread, tied_task)); 168 // Store entry 169 *(task_stack->ts_top) = tied_task; 170 171 // Do bookkeeping for next push 172 task_stack->ts_top++; 173 task_stack->ts_entries++; 174 175 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 176 // Find beginning of this task block 177 kmp_stack_block_t *stack_block = 178 (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE); 179 180 // Check if we already have a block 181 if (stack_block->sb_next != 182 NULL) { // reset ts_top to beginning of next block 183 task_stack->ts_top = &stack_block->sb_next->sb_block[0]; 184 } else { // Alloc new block and link it up 185 kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc( 186 thread, sizeof(kmp_stack_block_t)); 187 188 task_stack->ts_top = &new_block->sb_block[0]; 189 stack_block->sb_next = new_block; 190 new_block->sb_prev = stack_block; 191 new_block->sb_next = NULL; 192 193 KA_TRACE( 194 30, 195 ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n", 196 gtid, tied_task, new_block)); 197 } 198 } 199 KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 200 tied_task)); 201 } 202 203 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return 204 // the task, just check to make sure it matches the ending task passed in. 205 // 206 // gtid: global thread identifier for the calling thread 207 // thread: thread info structure containing stack 208 // tied_task: the task popped off the stack 209 // ending_task: the task that is ending (should match popped task) 210 static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread, 211 kmp_taskdata_t *ending_task) { 212 // GEH - need to consider what to do if tt_threads_data not allocated yet 213 kmp_thread_data_t *thread_data = 214 &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)]; 215 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 216 kmp_taskdata_t *tied_task; 217 218 if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) { 219 // Don't pop anything from stack if team or team tasks are serialized 220 return; 221 } 222 223 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 224 KMP_DEBUG_ASSERT(task_stack->ts_entries > 0); 225 226 KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid, 227 thread)); 228 229 // fix up ts_top if we need to pop from previous block 230 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 231 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top); 232 233 stack_block = stack_block->sb_prev; 234 task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 235 } 236 237 // finish bookkeeping 238 task_stack->ts_top--; 239 task_stack->ts_entries--; 240 241 tied_task = *(task_stack->ts_top); 242 243 KMP_DEBUG_ASSERT(tied_task != NULL); 244 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 245 KMP_DEBUG_ASSERT(tied_task == ending_task); // If we built the stack correctly 246 247 KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 248 tied_task)); 249 return; 250 } 251 #endif /* BUILD_TIED_TASK_STACK */ 252 253 // returns 1 if new task is allowed to execute, 0 otherwise 254 // checks Task Scheduling constraint (if requested) and 255 // mutexinoutset dependencies if any 256 static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained, 257 const kmp_taskdata_t *tasknew, 258 const kmp_taskdata_t *taskcurr) { 259 if (is_constrained && (tasknew->td_flags.tiedness == TASK_TIED)) { 260 // Check if the candidate obeys the Task Scheduling Constraints (TSC) 261 // only descendant of all deferred tied tasks can be scheduled, checking 262 // the last one is enough, as it in turn is the descendant of all others 263 kmp_taskdata_t *current = taskcurr->td_last_tied; 264 KMP_DEBUG_ASSERT(current != NULL); 265 // check if the task is not suspended on barrier 266 if (current->td_flags.tasktype == TASK_EXPLICIT || 267 current->td_taskwait_thread > 0) { // <= 0 on barrier 268 kmp_int32 level = current->td_level; 269 kmp_taskdata_t *parent = tasknew->td_parent; 270 while (parent != current && parent->td_level > level) { 271 // check generation up to the level of the current task 272 parent = parent->td_parent; 273 KMP_DEBUG_ASSERT(parent != NULL); 274 } 275 if (parent != current) 276 return false; 277 } 278 } 279 // Check mutexinoutset dependencies, acquire locks 280 kmp_depnode_t *node = tasknew->td_depnode; 281 if (UNLIKELY(node && (node->dn.mtx_num_locks > 0))) { 282 for (int i = 0; i < node->dn.mtx_num_locks; ++i) { 283 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL); 284 if (__kmp_test_lock(node->dn.mtx_locks[i], gtid)) 285 continue; 286 // could not get the lock, release previous locks 287 for (int j = i - 1; j >= 0; --j) 288 __kmp_release_lock(node->dn.mtx_locks[j], gtid); 289 return false; 290 } 291 // negative num_locks means all locks acquired successfully 292 node->dn.mtx_num_locks = -node->dn.mtx_num_locks; 293 } 294 return true; 295 } 296 297 // __kmp_realloc_task_deque: 298 // Re-allocates a task deque for a particular thread, copies the content from 299 // the old deque and adjusts the necessary data structures relating to the 300 // deque. This operation must be done with the deque_lock being held 301 static void __kmp_realloc_task_deque(kmp_info_t *thread, 302 kmp_thread_data_t *thread_data) { 303 kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td); 304 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == size); 305 kmp_int32 new_size = 2 * size; 306 307 KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to " 308 "%d] for thread_data %p\n", 309 __kmp_gtid_from_thread(thread), size, new_size, thread_data)); 310 311 kmp_taskdata_t **new_deque = 312 (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *)); 313 314 int i, j; 315 for (i = thread_data->td.td_deque_head, j = 0; j < size; 316 i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++) 317 new_deque[j] = thread_data->td.td_deque[i]; 318 319 __kmp_free(thread_data->td.td_deque); 320 321 thread_data->td.td_deque_head = 0; 322 thread_data->td.td_deque_tail = size; 323 thread_data->td.td_deque = new_deque; 324 thread_data->td.td_deque_size = new_size; 325 } 326 327 static kmp_task_pri_t *__kmp_alloc_task_pri_list() { 328 kmp_task_pri_t *l = (kmp_task_pri_t *)__kmp_allocate(sizeof(kmp_task_pri_t)); 329 kmp_thread_data_t *thread_data = &l->td; 330 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock); 331 thread_data->td.td_deque_last_stolen = -1; 332 KE_TRACE(20, ("__kmp_alloc_task_pri_list: T#%d allocating deque[%d] " 333 "for thread_data %p\n", 334 __kmp_get_gtid(), INITIAL_TASK_DEQUE_SIZE, thread_data)); 335 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate( 336 INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *)); 337 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE; 338 return l; 339 } 340 341 // The function finds the deque of priority tasks with given priority, or 342 // allocates a new deque and put it into sorted (high -> low) list of deques. 343 // Deques of non-default priority tasks are shared between all threads in team, 344 // as opposed to per-thread deques of tasks with default priority. 345 // The function is called under the lock task_team->tt.tt_task_pri_lock. 346 static kmp_thread_data_t * 347 __kmp_get_priority_deque_data(kmp_task_team_t *task_team, kmp_int32 pri) { 348 kmp_thread_data_t *thread_data; 349 kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list; 350 if (lst->priority == pri) { 351 // Found queue of tasks with given priority. 352 thread_data = &lst->td; 353 } else if (lst->priority < pri) { 354 // All current priority queues contain tasks with lower priority. 355 // Allocate new one for given priority tasks. 356 kmp_task_pri_t *list = __kmp_alloc_task_pri_list(); 357 thread_data = &list->td; 358 list->priority = pri; 359 list->next = lst; 360 task_team->tt.tt_task_pri_list = list; 361 } else { // task_team->tt.tt_task_pri_list->priority > pri 362 kmp_task_pri_t *next_queue = lst->next; 363 while (next_queue && next_queue->priority > pri) { 364 lst = next_queue; 365 next_queue = lst->next; 366 } 367 // lst->priority > pri && (next == NULL || pri >= next->priority) 368 if (next_queue == NULL) { 369 // No queue with pri priority, need to allocate new one. 370 kmp_task_pri_t *list = __kmp_alloc_task_pri_list(); 371 thread_data = &list->td; 372 list->priority = pri; 373 list->next = NULL; 374 lst->next = list; 375 } else if (next_queue->priority == pri) { 376 // Found queue of tasks with given priority. 377 thread_data = &next_queue->td; 378 } else { // lst->priority > pri > next->priority 379 // insert newly allocated between existed queues 380 kmp_task_pri_t *list = __kmp_alloc_task_pri_list(); 381 thread_data = &list->td; 382 list->priority = pri; 383 list->next = next_queue; 384 lst->next = list; 385 } 386 } 387 return thread_data; 388 } 389 390 // __kmp_push_priority_task: Add a task to the team's priority task deque 391 static kmp_int32 __kmp_push_priority_task(kmp_int32 gtid, kmp_info_t *thread, 392 kmp_taskdata_t *taskdata, 393 kmp_task_team_t *task_team, 394 kmp_int32 pri) { 395 kmp_thread_data_t *thread_data = NULL; 396 KA_TRACE(20, 397 ("__kmp_push_priority_task: T#%d trying to push task %p, pri %d.\n", 398 gtid, taskdata, pri)); 399 400 // Find task queue specific to priority value 401 kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list; 402 if (UNLIKELY(lst == NULL)) { 403 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock); 404 if (task_team->tt.tt_task_pri_list == NULL) { 405 // List of queues is still empty, allocate one. 406 kmp_task_pri_t *list = __kmp_alloc_task_pri_list(); 407 thread_data = &list->td; 408 list->priority = pri; 409 list->next = NULL; 410 task_team->tt.tt_task_pri_list = list; 411 } else { 412 // Other thread initialized a queue. Check if it fits and get thread_data. 413 thread_data = __kmp_get_priority_deque_data(task_team, pri); 414 } 415 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock); 416 } else { 417 if (lst->priority == pri) { 418 // Found queue of tasks with given priority. 419 thread_data = &lst->td; 420 } else { 421 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock); 422 thread_data = __kmp_get_priority_deque_data(task_team, pri); 423 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock); 424 } 425 } 426 KMP_DEBUG_ASSERT(thread_data); 427 428 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 429 // Check if deque is full 430 if (TCR_4(thread_data->td.td_deque_ntasks) >= 431 TASK_DEQUE_SIZE(thread_data->td)) { 432 if (__kmp_enable_task_throttling && 433 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 434 thread->th.th_current_task)) { 435 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 436 KA_TRACE(20, ("__kmp_push_priority_task: T#%d deque is full; returning " 437 "TASK_NOT_PUSHED for task %p\n", 438 gtid, taskdata)); 439 return TASK_NOT_PUSHED; 440 } else { 441 // expand deque to push the task which is not allowed to execute 442 __kmp_realloc_task_deque(thread, thread_data); 443 } 444 } 445 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) < 446 TASK_DEQUE_SIZE(thread_data->td)); 447 // Push taskdata. 448 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; 449 // Wrap index. 450 thread_data->td.td_deque_tail = 451 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 452 TCW_4(thread_data->td.td_deque_ntasks, 453 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count 454 KMP_FSYNC_RELEASING(thread->th.th_current_task); // releasing self 455 KMP_FSYNC_RELEASING(taskdata); // releasing child 456 KA_TRACE(20, ("__kmp_push_priority_task: T#%d returning " 457 "TASK_SUCCESSFULLY_PUSHED: task=%p ntasks=%d head=%u tail=%u\n", 458 gtid, taskdata, thread_data->td.td_deque_ntasks, 459 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 460 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 461 task_team->tt.tt_num_task_pri++; // atomic inc 462 return TASK_SUCCESSFULLY_PUSHED; 463 } 464 465 // __kmp_push_task: Add a task to the thread's deque 466 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) { 467 kmp_info_t *thread = __kmp_threads[gtid]; 468 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 469 470 // If we encounter a hidden helper task, and the current thread is not a 471 // hidden helper thread, we have to give the task to any hidden helper thread 472 // starting from its shadow one. 473 if (UNLIKELY(taskdata->td_flags.hidden_helper && 474 !KMP_HIDDEN_HELPER_THREAD(gtid))) { 475 kmp_int32 shadow_gtid = KMP_GTID_TO_SHADOW_GTID(gtid); 476 __kmpc_give_task(task, __kmp_tid_from_gtid(shadow_gtid)); 477 // Signal the hidden helper threads. 478 __kmp_hidden_helper_worker_thread_signal(); 479 return TASK_SUCCESSFULLY_PUSHED; 480 } 481 482 kmp_task_team_t *task_team = thread->th.th_task_team; 483 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 484 kmp_thread_data_t *thread_data; 485 486 KA_TRACE(20, 487 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata)); 488 489 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { 490 // untied task needs to increment counter so that the task structure is not 491 // freed prematurely 492 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 493 KMP_DEBUG_USE_VAR(counter); 494 KA_TRACE( 495 20, 496 ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n", 497 gtid, counter, taskdata)); 498 } 499 500 // The first check avoids building task_team thread data if serialized 501 if (UNLIKELY(taskdata->td_flags.task_serial)) { 502 KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning " 503 "TASK_NOT_PUSHED for task %p\n", 504 gtid, taskdata)); 505 return TASK_NOT_PUSHED; 506 } 507 508 // Now that serialized tasks have returned, we can assume that we are not in 509 // immediate exec mode 510 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 511 if (UNLIKELY(!KMP_TASKING_ENABLED(task_team))) { 512 __kmp_enable_tasking(task_team, thread); 513 } 514 KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE); 515 KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL); 516 517 if (taskdata->td_flags.priority_specified && task->data2.priority > 0 && 518 __kmp_max_task_priority > 0) { 519 int pri = KMP_MIN(task->data2.priority, __kmp_max_task_priority); 520 return __kmp_push_priority_task(gtid, thread, taskdata, task_team, pri); 521 } 522 523 // Find tasking deque specific to encountering thread 524 thread_data = &task_team->tt.tt_threads_data[tid]; 525 526 // No lock needed since only owner can allocate. If the task is hidden_helper, 527 // we don't need it either because we have initialized the dequeue for hidden 528 // helper thread data. 529 if (UNLIKELY(thread_data->td.td_deque == NULL)) { 530 __kmp_alloc_task_deque(thread, thread_data); 531 } 532 533 int locked = 0; 534 // Check if deque is full 535 if (TCR_4(thread_data->td.td_deque_ntasks) >= 536 TASK_DEQUE_SIZE(thread_data->td)) { 537 if (__kmp_enable_task_throttling && 538 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 539 thread->th.th_current_task)) { 540 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning " 541 "TASK_NOT_PUSHED for task %p\n", 542 gtid, taskdata)); 543 return TASK_NOT_PUSHED; 544 } else { 545 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 546 locked = 1; 547 if (TCR_4(thread_data->td.td_deque_ntasks) >= 548 TASK_DEQUE_SIZE(thread_data->td)) { 549 // expand deque to push the task which is not allowed to execute 550 __kmp_realloc_task_deque(thread, thread_data); 551 } 552 } 553 } 554 // Lock the deque for the task push operation 555 if (!locked) { 556 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 557 // Need to recheck as we can get a proxy task from thread outside of OpenMP 558 if (TCR_4(thread_data->td.td_deque_ntasks) >= 559 TASK_DEQUE_SIZE(thread_data->td)) { 560 if (__kmp_enable_task_throttling && 561 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 562 thread->th.th_current_task)) { 563 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 564 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; " 565 "returning TASK_NOT_PUSHED for task %p\n", 566 gtid, taskdata)); 567 return TASK_NOT_PUSHED; 568 } else { 569 // expand deque to push the task which is not allowed to execute 570 __kmp_realloc_task_deque(thread, thread_data); 571 } 572 } 573 } 574 // Must have room since no thread can add tasks but calling thread 575 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) < 576 TASK_DEQUE_SIZE(thread_data->td)); 577 578 thread_data->td.td_deque[thread_data->td.td_deque_tail] = 579 taskdata; // Push taskdata 580 // Wrap index. 581 thread_data->td.td_deque_tail = 582 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 583 TCW_4(thread_data->td.td_deque_ntasks, 584 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count 585 KMP_FSYNC_RELEASING(thread->th.th_current_task); // releasing self 586 KMP_FSYNC_RELEASING(taskdata); // releasing child 587 KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: " 588 "task=%p ntasks=%d head=%u tail=%u\n", 589 gtid, taskdata, thread_data->td.td_deque_ntasks, 590 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 591 592 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 593 594 return TASK_SUCCESSFULLY_PUSHED; 595 } 596 597 // __kmp_pop_current_task_from_thread: set up current task from called thread 598 // when team ends 599 // 600 // this_thr: thread structure to set current_task in. 601 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) { 602 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d " 603 "this_thread=%p, curtask=%p, " 604 "curtask_parent=%p\n", 605 0, this_thr, this_thr->th.th_current_task, 606 this_thr->th.th_current_task->td_parent)); 607 608 this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent; 609 610 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d " 611 "this_thread=%p, curtask=%p, " 612 "curtask_parent=%p\n", 613 0, this_thr, this_thr->th.th_current_task, 614 this_thr->th.th_current_task->td_parent)); 615 } 616 617 // __kmp_push_current_task_to_thread: set up current task in called thread for a 618 // new team 619 // 620 // this_thr: thread structure to set up 621 // team: team for implicit task data 622 // tid: thread within team to set up 623 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team, 624 int tid) { 625 // current task of the thread is a parent of the new just created implicit 626 // tasks of new team 627 KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p " 628 "curtask=%p " 629 "parent_task=%p\n", 630 tid, this_thr, this_thr->th.th_current_task, 631 team->t.t_implicit_task_taskdata[tid].td_parent)); 632 633 KMP_DEBUG_ASSERT(this_thr != NULL); 634 635 if (tid == 0) { 636 if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) { 637 team->t.t_implicit_task_taskdata[0].td_parent = 638 this_thr->th.th_current_task; 639 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0]; 640 } 641 } else { 642 team->t.t_implicit_task_taskdata[tid].td_parent = 643 team->t.t_implicit_task_taskdata[0].td_parent; 644 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid]; 645 } 646 647 KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p " 648 "curtask=%p " 649 "parent_task=%p\n", 650 tid, this_thr, this_thr->th.th_current_task, 651 team->t.t_implicit_task_taskdata[tid].td_parent)); 652 } 653 654 // __kmp_task_start: bookkeeping for a task starting execution 655 // 656 // GTID: global thread id of calling thread 657 // task: task starting execution 658 // current_task: task suspending 659 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task, 660 kmp_taskdata_t *current_task) { 661 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 662 kmp_info_t *thread = __kmp_threads[gtid]; 663 664 KA_TRACE(10, 665 ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n", 666 gtid, taskdata, current_task)); 667 668 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 669 670 // mark currently executing task as suspended 671 // TODO: GEH - make sure root team implicit task is initialized properly. 672 // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 ); 673 current_task->td_flags.executing = 0; 674 675 // Add task to stack if tied 676 #ifdef BUILD_TIED_TASK_STACK 677 if (taskdata->td_flags.tiedness == TASK_TIED) { 678 __kmp_push_task_stack(gtid, thread, taskdata); 679 } 680 #endif /* BUILD_TIED_TASK_STACK */ 681 682 // mark starting task as executing and as current task 683 thread->th.th_current_task = taskdata; 684 685 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 || 686 taskdata->td_flags.tiedness == TASK_UNTIED); 687 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 || 688 taskdata->td_flags.tiedness == TASK_UNTIED); 689 taskdata->td_flags.started = 1; 690 taskdata->td_flags.executing = 1; 691 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 692 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 693 694 // GEH TODO: shouldn't we pass some sort of location identifier here? 695 // APT: yes, we will pass location here. 696 // need to store current thread state (in a thread or taskdata structure) 697 // before setting work_state, otherwise wrong state is set after end of task 698 699 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata)); 700 701 return; 702 } 703 704 #if OMPT_SUPPORT 705 //------------------------------------------------------------------------------ 706 // __ompt_task_init: 707 // Initialize OMPT fields maintained by a task. This will only be called after 708 // ompt_start_tool, so we already know whether ompt is enabled or not. 709 710 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) { 711 // The calls to __ompt_task_init already have the ompt_enabled condition. 712 task->ompt_task_info.task_data.value = 0; 713 task->ompt_task_info.frame.exit_frame = ompt_data_none; 714 task->ompt_task_info.frame.enter_frame = ompt_data_none; 715 task->ompt_task_info.frame.exit_frame_flags = 716 ompt_frame_runtime | ompt_frame_framepointer; 717 task->ompt_task_info.frame.enter_frame_flags = 718 ompt_frame_runtime | ompt_frame_framepointer; 719 task->ompt_task_info.dispatch_chunk.start = 0; 720 task->ompt_task_info.dispatch_chunk.iterations = 0; 721 } 722 723 // __ompt_task_start: 724 // Build and trigger task-begin event 725 static inline void __ompt_task_start(kmp_task_t *task, 726 kmp_taskdata_t *current_task, 727 kmp_int32 gtid) { 728 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 729 ompt_task_status_t status = ompt_task_switch; 730 if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) { 731 status = ompt_task_yield; 732 __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0; 733 } 734 /* let OMPT know that we're about to run this task */ 735 if (ompt_enabled.ompt_callback_task_schedule) { 736 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 737 &(current_task->ompt_task_info.task_data), status, 738 &(taskdata->ompt_task_info.task_data)); 739 } 740 taskdata->ompt_task_info.scheduling_parent = current_task; 741 } 742 743 // __ompt_task_finish: 744 // Build and trigger final task-schedule event 745 static inline void __ompt_task_finish(kmp_task_t *task, 746 kmp_taskdata_t *resumed_task, 747 ompt_task_status_t status) { 748 if (ompt_enabled.ompt_callback_task_schedule) { 749 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 750 if (__kmp_omp_cancellation && taskdata->td_taskgroup && 751 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) { 752 status = ompt_task_cancel; 753 } 754 755 /* let OMPT know that we're returning to the callee task */ 756 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 757 &(taskdata->ompt_task_info.task_data), status, 758 (resumed_task ? &(resumed_task->ompt_task_info.task_data) : NULL)); 759 } 760 } 761 #endif 762 763 template <bool ompt> 764 static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid, 765 kmp_task_t *task, 766 void *frame_address, 767 void *return_address) { 768 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 769 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 770 771 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p " 772 "current_task=%p\n", 773 gtid, loc_ref, taskdata, current_task)); 774 775 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { 776 // untied task needs to increment counter so that the task structure is not 777 // freed prematurely 778 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 779 KMP_DEBUG_USE_VAR(counter); 780 KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) " 781 "incremented for task %p\n", 782 gtid, counter, taskdata)); 783 } 784 785 taskdata->td_flags.task_serial = 786 1; // Execute this task immediately, not deferred. 787 __kmp_task_start(gtid, task, current_task); 788 789 #if OMPT_SUPPORT 790 if (ompt) { 791 if (current_task->ompt_task_info.frame.enter_frame.ptr == NULL) { 792 current_task->ompt_task_info.frame.enter_frame.ptr = 793 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address; 794 current_task->ompt_task_info.frame.enter_frame_flags = 795 taskdata->ompt_task_info.frame.exit_frame_flags = 796 ompt_frame_application | ompt_frame_framepointer; 797 } 798 if (ompt_enabled.ompt_callback_task_create) { 799 ompt_task_info_t *parent_info = &(current_task->ompt_task_info); 800 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 801 &(parent_info->task_data), &(parent_info->frame), 802 &(taskdata->ompt_task_info.task_data), 803 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0, 804 return_address); 805 } 806 __ompt_task_start(task, current_task, gtid); 807 } 808 #endif // OMPT_SUPPORT 809 810 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid, 811 loc_ref, taskdata)); 812 } 813 814 #if OMPT_SUPPORT 815 OMPT_NOINLINE 816 static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 817 kmp_task_t *task, 818 void *frame_address, 819 void *return_address) { 820 __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address, 821 return_address); 822 } 823 #endif // OMPT_SUPPORT 824 825 // __kmpc_omp_task_begin_if0: report that a given serialized task has started 826 // execution 827 // 828 // loc_ref: source location information; points to beginning of task block. 829 // gtid: global thread number. 830 // task: task thunk for the started task. 831 void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid, 832 kmp_task_t *task) { 833 #if OMPT_SUPPORT 834 if (UNLIKELY(ompt_enabled.enabled)) { 835 OMPT_STORE_RETURN_ADDRESS(gtid); 836 __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task, 837 OMPT_GET_FRAME_ADDRESS(1), 838 OMPT_LOAD_RETURN_ADDRESS(gtid)); 839 return; 840 } 841 #endif 842 __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL); 843 } 844 845 #ifdef TASK_UNUSED 846 // __kmpc_omp_task_begin: report that a given task has started execution 847 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 848 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) { 849 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 850 851 KA_TRACE( 852 10, 853 ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n", 854 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task)); 855 856 __kmp_task_start(gtid, task, current_task); 857 858 KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid, 859 loc_ref, KMP_TASK_TO_TASKDATA(task))); 860 return; 861 } 862 #endif // TASK_UNUSED 863 864 // __kmp_free_task: free the current task space and the space for shareds 865 // 866 // gtid: Global thread ID of calling thread 867 // taskdata: task to free 868 // thread: thread data structure of caller 869 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata, 870 kmp_info_t *thread) { 871 KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid, 872 taskdata)); 873 874 // Check to make sure all flags and counters have the correct values 875 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 876 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0); 877 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1); 878 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 879 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 || 880 taskdata->td_flags.task_serial == 1); 881 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0); 882 kmp_task_t *task = KMP_TASKDATA_TO_TASK(taskdata); 883 // Clear data to not be re-used later by mistake. 884 task->data1.destructors = NULL; 885 task->data2.priority = 0; 886 887 taskdata->td_flags.freed = 1; 888 // deallocate the taskdata and shared variable blocks associated with this task 889 #if USE_FAST_MEMORY 890 __kmp_fast_free(thread, taskdata); 891 #else /* ! USE_FAST_MEMORY */ 892 __kmp_thread_free(thread, taskdata); 893 #endif 894 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata)); 895 } 896 897 // __kmp_free_task_and_ancestors: free the current task and ancestors without 898 // children 899 // 900 // gtid: Global thread ID of calling thread 901 // taskdata: task to free 902 // thread: thread data structure of caller 903 static void __kmp_free_task_and_ancestors(kmp_int32 gtid, 904 kmp_taskdata_t *taskdata, 905 kmp_info_t *thread) { 906 // Proxy tasks must always be allowed to free their parents 907 // because they can be run in background even in serial mode. 908 kmp_int32 team_serial = 909 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) && 910 !taskdata->td_flags.proxy; 911 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 912 913 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 914 KMP_DEBUG_ASSERT(children >= 0); 915 916 // Now, go up the ancestor tree to see if any ancestors can now be freed. 917 while (children == 0) { 918 kmp_taskdata_t *parent_taskdata = taskdata->td_parent; 919 920 KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete " 921 "and freeing itself\n", 922 gtid, taskdata)); 923 924 // --- Deallocate my ancestor task --- 925 __kmp_free_task(gtid, taskdata, thread); 926 927 taskdata = parent_taskdata; 928 929 if (team_serial) 930 return; 931 // Stop checking ancestors at implicit task instead of walking up ancestor 932 // tree to avoid premature deallocation of ancestors. 933 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) { 934 if (taskdata->td_dephash) { // do we need to cleanup dephash? 935 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks); 936 kmp_tasking_flags_t flags_old = taskdata->td_flags; 937 if (children == 0 && flags_old.complete == 1) { 938 kmp_tasking_flags_t flags_new = flags_old; 939 flags_new.complete = 0; 940 if (KMP_COMPARE_AND_STORE_ACQ32( 941 RCAST(kmp_int32 *, &taskdata->td_flags), 942 *RCAST(kmp_int32 *, &flags_old), 943 *RCAST(kmp_int32 *, &flags_new))) { 944 KA_TRACE(100, ("__kmp_free_task_and_ancestors: T#%d cleans " 945 "dephash of implicit task %p\n", 946 gtid, taskdata)); 947 // cleanup dephash of finished implicit task 948 __kmp_dephash_free_entries(thread, taskdata->td_dephash); 949 } 950 } 951 } 952 return; 953 } 954 // Predecrement simulated by "- 1" calculation 955 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 956 KMP_DEBUG_ASSERT(children >= 0); 957 } 958 959 KA_TRACE( 960 20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; " 961 "not freeing it yet\n", 962 gtid, taskdata, children)); 963 } 964 965 // Only need to keep track of child task counts if any of the following: 966 // 1. team parallel and tasking not serialized; 967 // 2. it is a proxy or detachable or hidden helper task 968 // 3. the children counter of its parent task is greater than 0. 969 // The reason for the 3rd one is for serialized team that found detached task, 970 // hidden helper task, T. In this case, the execution of T is still deferred, 971 // and it is also possible that a regular task depends on T. In this case, if we 972 // don't track the children, task synchronization will be broken. 973 static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) { 974 kmp_tasking_flags_t flags = taskdata->td_flags; 975 bool ret = !(flags.team_serial || flags.tasking_ser); 976 ret = ret || flags.proxy == TASK_PROXY || 977 flags.detachable == TASK_DETACHABLE || flags.hidden_helper; 978 ret = ret || 979 KMP_ATOMIC_LD_ACQ(&taskdata->td_parent->td_incomplete_child_tasks) > 0; 980 return ret; 981 } 982 983 // __kmp_task_finish: bookkeeping to do when a task finishes execution 984 // 985 // gtid: global thread ID for calling thread 986 // task: task to be finished 987 // resumed_task: task to be resumed. (may be NULL if task is serialized) 988 // 989 // template<ompt>: effectively ompt_enabled.enabled!=0 990 // the version with ompt=false is inlined, allowing to optimize away all ompt 991 // code in this case 992 template <bool ompt> 993 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task, 994 kmp_taskdata_t *resumed_task) { 995 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 996 kmp_info_t *thread = __kmp_threads[gtid]; 997 kmp_task_team_t *task_team = 998 thread->th.th_task_team; // might be NULL for serial teams... 999 #if KMP_DEBUG 1000 kmp_int32 children = 0; 1001 #endif 1002 KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming " 1003 "task %p\n", 1004 gtid, taskdata, resumed_task)); 1005 1006 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 1007 1008 // Pop task from stack if tied 1009 #ifdef BUILD_TIED_TASK_STACK 1010 if (taskdata->td_flags.tiedness == TASK_TIED) { 1011 __kmp_pop_task_stack(gtid, thread, taskdata); 1012 } 1013 #endif /* BUILD_TIED_TASK_STACK */ 1014 1015 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { 1016 // untied task needs to check the counter so that the task structure is not 1017 // freed prematurely 1018 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1; 1019 KA_TRACE( 1020 20, 1021 ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n", 1022 gtid, counter, taskdata)); 1023 if (counter > 0) { 1024 // untied task is not done, to be continued possibly by other thread, do 1025 // not free it now 1026 if (resumed_task == NULL) { 1027 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial); 1028 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 1029 // task is the parent 1030 } 1031 thread->th.th_current_task = resumed_task; // restore current_task 1032 resumed_task->td_flags.executing = 1; // resume previous task 1033 KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, " 1034 "resuming task %p\n", 1035 gtid, taskdata, resumed_task)); 1036 return; 1037 } 1038 } 1039 1040 // bookkeeping for resuming task: 1041 // GEH - note tasking_ser => task_serial 1042 KMP_DEBUG_ASSERT( 1043 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == 1044 taskdata->td_flags.task_serial); 1045 if (taskdata->td_flags.task_serial) { 1046 if (resumed_task == NULL) { 1047 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 1048 // task is the parent 1049 } 1050 } else { 1051 KMP_DEBUG_ASSERT(resumed_task != 1052 NULL); // verify that resumed task is passed as argument 1053 } 1054 1055 /* If the tasks' destructor thunk flag has been set, we need to invoke the 1056 destructor thunk that has been generated by the compiler. The code is 1057 placed here, since at this point other tasks might have been released 1058 hence overlapping the destructor invocations with some other work in the 1059 released tasks. The OpenMP spec is not specific on when the destructors 1060 are invoked, so we should be free to choose. */ 1061 if (UNLIKELY(taskdata->td_flags.destructors_thunk)) { 1062 kmp_routine_entry_t destr_thunk = task->data1.destructors; 1063 KMP_ASSERT(destr_thunk); 1064 destr_thunk(gtid, task); 1065 } 1066 1067 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 1068 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); 1069 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 1070 1071 bool completed = true; 1072 if (UNLIKELY(taskdata->td_flags.detachable == TASK_DETACHABLE)) { 1073 if (taskdata->td_allow_completion_event.type == 1074 KMP_EVENT_ALLOW_COMPLETION) { 1075 // event hasn't been fulfilled yet. Try to detach task. 1076 __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 1077 if (taskdata->td_allow_completion_event.type == 1078 KMP_EVENT_ALLOW_COMPLETION) { 1079 // task finished execution 1080 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); 1081 taskdata->td_flags.executing = 0; // suspend the finishing task 1082 1083 #if OMPT_SUPPORT 1084 // For a detached task, which is not completed, we switch back 1085 // the omp_fulfill_event signals completion 1086 // locking is necessary to avoid a race with ompt_task_late_fulfill 1087 if (ompt) 1088 __ompt_task_finish(task, resumed_task, ompt_task_detach); 1089 #endif 1090 1091 // no access to taskdata after this point! 1092 // __kmp_fulfill_event might free taskdata at any time from now 1093 1094 taskdata->td_flags.proxy = TASK_PROXY; // proxify! 1095 completed = false; 1096 } 1097 __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 1098 } 1099 } 1100 1101 // Tasks with valid target async handles must be re-enqueued. 1102 if (taskdata->td_target_data.async_handle != NULL) { 1103 // Note: no need to translate gtid to its shadow. If the current thread is a 1104 // hidden helper one, then the gtid is already correct. Otherwise, hidden 1105 // helper threads are disabled, and gtid refers to a OpenMP thread. 1106 __kmpc_give_task(task, __kmp_tid_from_gtid(gtid)); 1107 if (KMP_HIDDEN_HELPER_THREAD(gtid)) 1108 __kmp_hidden_helper_worker_thread_signal(); 1109 completed = false; 1110 } 1111 1112 if (completed) { 1113 taskdata->td_flags.complete = 1; // mark the task as completed 1114 1115 #if OMPT_SUPPORT 1116 // This is not a detached task, we are done here 1117 if (ompt) 1118 __ompt_task_finish(task, resumed_task, ompt_task_complete); 1119 #endif 1120 // TODO: What would be the balance between the conditions in the function 1121 // and an atomic operation? 1122 if (__kmp_track_children_task(taskdata)) { 1123 __kmp_release_deps(gtid, taskdata); 1124 // Predecrement simulated by "- 1" calculation 1125 #if KMP_DEBUG 1126 children = -1 + 1127 #endif 1128 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks); 1129 KMP_DEBUG_ASSERT(children >= 0); 1130 if (taskdata->td_taskgroup) 1131 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 1132 } else if (task_team && (task_team->tt.tt_found_proxy_tasks || 1133 task_team->tt.tt_hidden_helper_task_encountered)) { 1134 // if we found proxy or hidden helper tasks there could exist a dependency 1135 // chain with the proxy task as origin 1136 __kmp_release_deps(gtid, taskdata); 1137 } 1138 // td_flags.executing must be marked as 0 after __kmp_release_deps has been 1139 // called. Othertwise, if a task is executed immediately from the 1140 // release_deps code, the flag will be reset to 1 again by this same 1141 // function 1142 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); 1143 taskdata->td_flags.executing = 0; // suspend the finishing task 1144 1145 // Decrement the counter of hidden helper tasks to be executed. 1146 if (taskdata->td_flags.hidden_helper) { 1147 // Hidden helper tasks can only be executed by hidden helper threads. 1148 KMP_ASSERT(KMP_HIDDEN_HELPER_THREAD(gtid)); 1149 KMP_ATOMIC_DEC(&__kmp_unexecuted_hidden_helper_tasks); 1150 } 1151 } 1152 1153 KA_TRACE( 1154 20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n", 1155 gtid, taskdata, children)); 1156 1157 // Free this task and then ancestor tasks if they have no children. 1158 // Restore th_current_task first as suggested by John: 1159 // johnmc: if an asynchronous inquiry peers into the runtime system 1160 // it doesn't see the freed task as the current task. 1161 thread->th.th_current_task = resumed_task; 1162 if (completed) 1163 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 1164 1165 // TODO: GEH - make sure root team implicit task is initialized properly. 1166 // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 ); 1167 resumed_task->td_flags.executing = 1; // resume previous task 1168 1169 KA_TRACE( 1170 10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n", 1171 gtid, taskdata, resumed_task)); 1172 1173 return; 1174 } 1175 1176 template <bool ompt> 1177 static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref, 1178 kmp_int32 gtid, 1179 kmp_task_t *task) { 1180 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n", 1181 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 1182 KMP_DEBUG_ASSERT(gtid >= 0); 1183 // this routine will provide task to resume 1184 __kmp_task_finish<ompt>(gtid, task, NULL); 1185 1186 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n", 1187 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 1188 1189 #if OMPT_SUPPORT 1190 if (ompt) { 1191 ompt_frame_t *ompt_frame; 1192 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 1193 ompt_frame->enter_frame = ompt_data_none; 1194 ompt_frame->enter_frame_flags = 1195 ompt_frame_runtime | ompt_frame_framepointer; 1196 } 1197 #endif 1198 1199 return; 1200 } 1201 1202 #if OMPT_SUPPORT 1203 OMPT_NOINLINE 1204 void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 1205 kmp_task_t *task) { 1206 __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task); 1207 } 1208 #endif // OMPT_SUPPORT 1209 1210 // __kmpc_omp_task_complete_if0: report that a task has completed execution 1211 // 1212 // loc_ref: source location information; points to end of task block. 1213 // gtid: global thread number. 1214 // task: task thunk for the completed task. 1215 void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid, 1216 kmp_task_t *task) { 1217 #if OMPT_SUPPORT 1218 if (UNLIKELY(ompt_enabled.enabled)) { 1219 __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task); 1220 return; 1221 } 1222 #endif 1223 __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task); 1224 } 1225 1226 #ifdef TASK_UNUSED 1227 // __kmpc_omp_task_complete: report that a task has completed execution 1228 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 1229 void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid, 1230 kmp_task_t *task) { 1231 KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid, 1232 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1233 1234 __kmp_task_finish<false>(gtid, task, 1235 NULL); // Not sure how to find task to resume 1236 1237 KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid, 1238 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1239 return; 1240 } 1241 #endif // TASK_UNUSED 1242 1243 // __kmp_init_implicit_task: Initialize the appropriate fields in the implicit 1244 // task for a given thread 1245 // 1246 // loc_ref: reference to source location of parallel region 1247 // this_thr: thread data structure corresponding to implicit task 1248 // team: team for this_thr 1249 // tid: thread id of given thread within team 1250 // set_curr_task: TRUE if need to push current task to thread 1251 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to 1252 // have already been done elsewhere. 1253 // TODO: Get better loc_ref. Value passed in may be NULL 1254 void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr, 1255 kmp_team_t *team, int tid, int set_curr_task) { 1256 kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid]; 1257 1258 KF_TRACE( 1259 10, 1260 ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n", 1261 tid, team, task, set_curr_task ? "TRUE" : "FALSE")); 1262 1263 task->td_task_id = KMP_GEN_TASK_ID(); 1264 task->td_team = team; 1265 // task->td_parent = NULL; // fix for CQ230101 (broken parent task info 1266 // in debugger) 1267 task->td_ident = loc_ref; 1268 task->td_taskwait_ident = NULL; 1269 task->td_taskwait_counter = 0; 1270 task->td_taskwait_thread = 0; 1271 1272 task->td_flags.tiedness = TASK_TIED; 1273 task->td_flags.tasktype = TASK_IMPLICIT; 1274 task->td_flags.proxy = TASK_FULL; 1275 1276 // All implicit tasks are executed immediately, not deferred 1277 task->td_flags.task_serial = 1; 1278 task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1279 task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1280 1281 task->td_flags.started = 1; 1282 task->td_flags.executing = 1; 1283 task->td_flags.complete = 0; 1284 task->td_flags.freed = 0; 1285 1286 task->td_depnode = NULL; 1287 task->td_last_tied = task; 1288 task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1289 1290 if (set_curr_task) { // only do this init first time thread is created 1291 KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0); 1292 // Not used: don't need to deallocate implicit task 1293 KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0); 1294 task->td_taskgroup = NULL; // An implicit task does not have taskgroup 1295 task->td_dephash = NULL; 1296 __kmp_push_current_task_to_thread(this_thr, team, tid); 1297 } else { 1298 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0); 1299 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0); 1300 } 1301 1302 #if OMPT_SUPPORT 1303 if (UNLIKELY(ompt_enabled.enabled)) 1304 __ompt_task_init(task, tid); 1305 #endif 1306 1307 KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid, 1308 team, task)); 1309 } 1310 1311 // __kmp_finish_implicit_task: Release resources associated to implicit tasks 1312 // at the end of parallel regions. Some resources are kept for reuse in the next 1313 // parallel region. 1314 // 1315 // thread: thread data structure corresponding to implicit task 1316 void __kmp_finish_implicit_task(kmp_info_t *thread) { 1317 kmp_taskdata_t *task = thread->th.th_current_task; 1318 if (task->td_dephash) { 1319 int children; 1320 task->td_flags.complete = 1; 1321 children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks); 1322 kmp_tasking_flags_t flags_old = task->td_flags; 1323 if (children == 0 && flags_old.complete == 1) { 1324 kmp_tasking_flags_t flags_new = flags_old; 1325 flags_new.complete = 0; 1326 if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags), 1327 *RCAST(kmp_int32 *, &flags_old), 1328 *RCAST(kmp_int32 *, &flags_new))) { 1329 KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans " 1330 "dephash of implicit task %p\n", 1331 thread->th.th_info.ds.ds_gtid, task)); 1332 __kmp_dephash_free_entries(thread, task->td_dephash); 1333 } 1334 } 1335 } 1336 } 1337 1338 // __kmp_free_implicit_task: Release resources associated to implicit tasks 1339 // when these are destroyed regions 1340 // 1341 // thread: thread data structure corresponding to implicit task 1342 void __kmp_free_implicit_task(kmp_info_t *thread) { 1343 kmp_taskdata_t *task = thread->th.th_current_task; 1344 if (task && task->td_dephash) { 1345 __kmp_dephash_free(thread, task->td_dephash); 1346 task->td_dephash = NULL; 1347 } 1348 } 1349 1350 // Round up a size to a power of two specified by val: Used to insert padding 1351 // between structures co-allocated using a single malloc() call 1352 static size_t __kmp_round_up_to_val(size_t size, size_t val) { 1353 if (size & (val - 1)) { 1354 size &= ~(val - 1); 1355 if (size <= KMP_SIZE_T_MAX - val) { 1356 size += val; // Round up if there is no overflow. 1357 } 1358 } 1359 return size; 1360 } // __kmp_round_up_to_va 1361 1362 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task 1363 // 1364 // loc_ref: source location information 1365 // gtid: global thread number. 1366 // flags: include tiedness & task type (explicit vs. implicit) of the ''new'' 1367 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine. 1368 // sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including 1369 // private vars accessed in task. 1370 // sizeof_shareds: Size in bytes of array of pointers to shared vars accessed 1371 // in task. 1372 // task_entry: Pointer to task code entry point generated by compiler. 1373 // returns: a pointer to the allocated kmp_task_t structure (task). 1374 kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1375 kmp_tasking_flags_t *flags, 1376 size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1377 kmp_routine_entry_t task_entry) { 1378 kmp_task_t *task; 1379 kmp_taskdata_t *taskdata; 1380 kmp_info_t *thread = __kmp_threads[gtid]; 1381 kmp_team_t *team = thread->th.th_team; 1382 kmp_taskdata_t *parent_task = thread->th.th_current_task; 1383 size_t shareds_offset; 1384 1385 if (UNLIKELY(!TCR_4(__kmp_init_middle))) 1386 __kmp_middle_initialize(); 1387 1388 if (flags->hidden_helper) { 1389 if (__kmp_enable_hidden_helper) { 1390 if (!TCR_4(__kmp_init_hidden_helper)) 1391 __kmp_hidden_helper_initialize(); 1392 } else { 1393 // If the hidden helper task is not enabled, reset the flag to FALSE. 1394 flags->hidden_helper = FALSE; 1395 } 1396 } 1397 1398 KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) " 1399 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1400 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t, 1401 sizeof_shareds, task_entry)); 1402 1403 KMP_DEBUG_ASSERT(parent_task); 1404 if (parent_task->td_flags.final) { 1405 if (flags->merged_if0) { 1406 } 1407 flags->final = 1; 1408 } 1409 1410 if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) { 1411 // Untied task encountered causes the TSC algorithm to check entire deque of 1412 // the victim thread. If no untied task encountered, then checking the head 1413 // of the deque should be enough. 1414 KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1); 1415 } 1416 1417 // Detachable tasks are not proxy tasks yet but could be in the future. Doing 1418 // the tasking setup 1419 // when that happens is too late. 1420 if (UNLIKELY(flags->proxy == TASK_PROXY || 1421 flags->detachable == TASK_DETACHABLE || flags->hidden_helper)) { 1422 if (flags->proxy == TASK_PROXY) { 1423 flags->tiedness = TASK_UNTIED; 1424 flags->merged_if0 = 1; 1425 } 1426 /* are we running in a sequential parallel or tskm_immediate_exec... we need 1427 tasking support enabled */ 1428 if ((thread->th.th_task_team) == NULL) { 1429 /* This should only happen if the team is serialized 1430 setup a task team and propagate it to the thread */ 1431 KMP_DEBUG_ASSERT(team->t.t_serialized); 1432 KA_TRACE(30, 1433 ("T#%d creating task team in __kmp_task_alloc for proxy task\n", 1434 gtid)); 1435 // 1 indicates setup the current team regardless of nthreads 1436 __kmp_task_team_setup(thread, team, 1); 1437 thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state]; 1438 } 1439 kmp_task_team_t *task_team = thread->th.th_task_team; 1440 1441 /* tasking must be enabled now as the task might not be pushed */ 1442 if (!KMP_TASKING_ENABLED(task_team)) { 1443 KA_TRACE( 1444 30, 1445 ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid)); 1446 __kmp_enable_tasking(task_team, thread); 1447 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 1448 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 1449 // No lock needed since only owner can allocate 1450 if (thread_data->td.td_deque == NULL) { 1451 __kmp_alloc_task_deque(thread, thread_data); 1452 } 1453 } 1454 1455 if ((flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE) && 1456 task_team->tt.tt_found_proxy_tasks == FALSE) 1457 TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE); 1458 if (flags->hidden_helper && 1459 task_team->tt.tt_hidden_helper_task_encountered == FALSE) 1460 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, TRUE); 1461 } 1462 1463 // Calculate shared structure offset including padding after kmp_task_t struct 1464 // to align pointers in shared struct 1465 shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t; 1466 shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *)); 1467 1468 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 1469 KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid, 1470 shareds_offset)); 1471 KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid, 1472 sizeof_shareds)); 1473 1474 // Avoid double allocation here by combining shareds with taskdata 1475 #if USE_FAST_MEMORY 1476 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset + 1477 sizeof_shareds); 1478 #else /* ! USE_FAST_MEMORY */ 1479 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset + 1480 sizeof_shareds); 1481 #endif /* USE_FAST_MEMORY */ 1482 1483 task = KMP_TASKDATA_TO_TASK(taskdata); 1484 1485 // Make sure task & taskdata are aligned appropriately 1486 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD 1487 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0); 1488 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0); 1489 #else 1490 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0); 1491 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0); 1492 #endif 1493 if (sizeof_shareds > 0) { 1494 // Avoid double allocation here by combining shareds with taskdata 1495 task->shareds = &((char *)taskdata)[shareds_offset]; 1496 // Make sure shareds struct is aligned to pointer size 1497 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 1498 0); 1499 } else { 1500 task->shareds = NULL; 1501 } 1502 task->routine = task_entry; 1503 task->part_id = 0; // AC: Always start with 0 part id 1504 1505 taskdata->td_task_id = KMP_GEN_TASK_ID(); 1506 taskdata->td_team = thread->th.th_team; 1507 taskdata->td_alloc_thread = thread; 1508 taskdata->td_parent = parent_task; 1509 taskdata->td_level = parent_task->td_level + 1; // increment nesting level 1510 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0); 1511 taskdata->td_ident = loc_ref; 1512 taskdata->td_taskwait_ident = NULL; 1513 taskdata->td_taskwait_counter = 0; 1514 taskdata->td_taskwait_thread = 0; 1515 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL); 1516 // avoid copying icvs for proxy tasks 1517 if (flags->proxy == TASK_FULL) 1518 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs); 1519 1520 taskdata->td_flags = *flags; 1521 taskdata->td_task_team = thread->th.th_task_team; 1522 taskdata->td_size_alloc = shareds_offset + sizeof_shareds; 1523 taskdata->td_flags.tasktype = TASK_EXPLICIT; 1524 // If it is hidden helper task, we need to set the team and task team 1525 // correspondingly. 1526 if (flags->hidden_helper) { 1527 kmp_info_t *shadow_thread = __kmp_threads[KMP_GTID_TO_SHADOW_GTID(gtid)]; 1528 taskdata->td_team = shadow_thread->th.th_team; 1529 taskdata->td_task_team = shadow_thread->th.th_task_team; 1530 } 1531 1532 // GEH - TODO: fix this to copy parent task's value of tasking_ser flag 1533 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1534 1535 // GEH - TODO: fix this to copy parent task's value of team_serial flag 1536 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1537 1538 // GEH - Note we serialize the task if the team is serialized to make sure 1539 // implicit parallel region tasks are not left until program termination to 1540 // execute. Also, it helps locality to execute immediately. 1541 1542 taskdata->td_flags.task_serial = 1543 (parent_task->td_flags.final || taskdata->td_flags.team_serial || 1544 taskdata->td_flags.tasking_ser || flags->merged_if0); 1545 1546 taskdata->td_flags.started = 0; 1547 taskdata->td_flags.executing = 0; 1548 taskdata->td_flags.complete = 0; 1549 taskdata->td_flags.freed = 0; 1550 1551 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0); 1552 // start at one because counts current task and children 1553 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1); 1554 taskdata->td_taskgroup = 1555 parent_task->td_taskgroup; // task inherits taskgroup from the parent task 1556 taskdata->td_dephash = NULL; 1557 taskdata->td_depnode = NULL; 1558 taskdata->td_target_data.async_handle = NULL; 1559 if (flags->tiedness == TASK_UNTIED) 1560 taskdata->td_last_tied = NULL; // will be set when the task is scheduled 1561 else 1562 taskdata->td_last_tied = taskdata; 1563 taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1564 #if OMPT_SUPPORT 1565 if (UNLIKELY(ompt_enabled.enabled)) 1566 __ompt_task_init(taskdata, gtid); 1567 #endif 1568 // TODO: What would be the balance between the conditions in the function and 1569 // an atomic operation? 1570 if (__kmp_track_children_task(taskdata)) { 1571 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 1572 if (parent_task->td_taskgroup) 1573 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 1574 // Only need to keep track of allocated child tasks for explicit tasks since 1575 // implicit not deallocated 1576 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) { 1577 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 1578 } 1579 if (flags->hidden_helper) { 1580 taskdata->td_flags.task_serial = FALSE; 1581 // Increment the number of hidden helper tasks to be executed 1582 KMP_ATOMIC_INC(&__kmp_unexecuted_hidden_helper_tasks); 1583 } 1584 } 1585 1586 KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n", 1587 gtid, taskdata, taskdata->td_parent)); 1588 1589 return task; 1590 } 1591 1592 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1593 kmp_int32 flags, size_t sizeof_kmp_task_t, 1594 size_t sizeof_shareds, 1595 kmp_routine_entry_t task_entry) { 1596 kmp_task_t *retval; 1597 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1598 __kmp_assert_valid_gtid(gtid); 1599 input_flags->native = FALSE; 1600 // __kmp_task_alloc() sets up all other runtime flags 1601 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) " 1602 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1603 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied", 1604 input_flags->proxy ? "proxy" : "", 1605 input_flags->detachable ? "detachable" : "", sizeof_kmp_task_t, 1606 sizeof_shareds, task_entry)); 1607 1608 retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t, 1609 sizeof_shareds, task_entry); 1610 1611 KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval)); 1612 1613 return retval; 1614 } 1615 1616 kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1617 kmp_int32 flags, 1618 size_t sizeof_kmp_task_t, 1619 size_t sizeof_shareds, 1620 kmp_routine_entry_t task_entry, 1621 kmp_int64 device_id) { 1622 auto &input_flags = reinterpret_cast<kmp_tasking_flags_t &>(flags); 1623 // target task is untied defined in the specification 1624 input_flags.tiedness = TASK_UNTIED; 1625 1626 if (__kmp_enable_hidden_helper) 1627 input_flags.hidden_helper = TRUE; 1628 1629 return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t, 1630 sizeof_shareds, task_entry); 1631 } 1632 1633 /*! 1634 @ingroup TASKING 1635 @param loc_ref location of the original task directive 1636 @param gtid Global Thread ID of encountering thread 1637 @param new_task task thunk allocated by __kmpc_omp_task_alloc() for the ''new 1638 task'' 1639 @param naffins Number of affinity items 1640 @param affin_list List of affinity items 1641 @return Returns non-zero if registering affinity information was not successful. 1642 Returns 0 if registration was successful 1643 This entry registers the affinity information attached to a task with the task 1644 thunk structure kmp_taskdata_t. 1645 */ 1646 kmp_int32 1647 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, 1648 kmp_task_t *new_task, kmp_int32 naffins, 1649 kmp_task_affinity_info_t *affin_list) { 1650 return 0; 1651 } 1652 1653 // __kmp_invoke_task: invoke the specified task 1654 // 1655 // gtid: global thread ID of caller 1656 // task: the task to invoke 1657 // current_task: the task to resume after task invocation 1658 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task, 1659 kmp_taskdata_t *current_task) { 1660 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 1661 kmp_info_t *thread; 1662 int discard = 0 /* false */; 1663 KA_TRACE( 1664 30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n", 1665 gtid, taskdata, current_task)); 1666 KMP_DEBUG_ASSERT(task); 1667 if (UNLIKELY(taskdata->td_flags.proxy == TASK_PROXY && 1668 taskdata->td_flags.complete == 1)) { 1669 // This is a proxy task that was already completed but it needs to run 1670 // its bottom-half finish 1671 KA_TRACE( 1672 30, 1673 ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n", 1674 gtid, taskdata)); 1675 1676 __kmp_bottom_half_finish_proxy(gtid, task); 1677 1678 KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for " 1679 "proxy task %p, resuming task %p\n", 1680 gtid, taskdata, current_task)); 1681 1682 return; 1683 } 1684 1685 #if OMPT_SUPPORT 1686 // For untied tasks, the first task executed only calls __kmpc_omp_task and 1687 // does not execute code. 1688 ompt_thread_info_t oldInfo; 1689 if (UNLIKELY(ompt_enabled.enabled)) { 1690 // Store the threads states and restore them after the task 1691 thread = __kmp_threads[gtid]; 1692 oldInfo = thread->th.ompt_thread_info; 1693 thread->th.ompt_thread_info.wait_id = 0; 1694 thread->th.ompt_thread_info.state = (thread->th.th_team_serialized) 1695 ? ompt_state_work_serial 1696 : ompt_state_work_parallel; 1697 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1698 } 1699 #endif 1700 1701 // Proxy tasks are not handled by the runtime 1702 if (taskdata->td_flags.proxy != TASK_PROXY) { 1703 __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded 1704 } 1705 1706 // TODO: cancel tasks if the parallel region has also been cancelled 1707 // TODO: check if this sequence can be hoisted above __kmp_task_start 1708 // if cancellation has been enabled for this run ... 1709 if (UNLIKELY(__kmp_omp_cancellation)) { 1710 thread = __kmp_threads[gtid]; 1711 kmp_team_t *this_team = thread->th.th_team; 1712 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 1713 if ((taskgroup && taskgroup->cancel_request) || 1714 (this_team->t.t_cancel_request == cancel_parallel)) { 1715 #if OMPT_SUPPORT && OMPT_OPTIONAL 1716 ompt_data_t *task_data; 1717 if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) { 1718 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); 1719 ompt_callbacks.ompt_callback(ompt_callback_cancel)( 1720 task_data, 1721 ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup 1722 : ompt_cancel_parallel) | 1723 ompt_cancel_discarded_task, 1724 NULL); 1725 } 1726 #endif 1727 KMP_COUNT_BLOCK(TASK_cancelled); 1728 // this task belongs to a task group and we need to cancel it 1729 discard = 1 /* true */; 1730 } 1731 } 1732 1733 // Invoke the task routine and pass in relevant data. 1734 // Thunks generated by gcc take a different argument list. 1735 if (!discard) { 1736 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 1737 taskdata->td_last_tied = current_task->td_last_tied; 1738 KMP_DEBUG_ASSERT(taskdata->td_last_tied); 1739 } 1740 #if KMP_STATS_ENABLED 1741 KMP_COUNT_BLOCK(TASK_executed); 1742 switch (KMP_GET_THREAD_STATE()) { 1743 case FORK_JOIN_BARRIER: 1744 KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar); 1745 break; 1746 case PLAIN_BARRIER: 1747 KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar); 1748 break; 1749 case TASKYIELD: 1750 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield); 1751 break; 1752 case TASKWAIT: 1753 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait); 1754 break; 1755 case TASKGROUP: 1756 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup); 1757 break; 1758 default: 1759 KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate); 1760 break; 1761 } 1762 #endif // KMP_STATS_ENABLED 1763 1764 // OMPT task begin 1765 #if OMPT_SUPPORT 1766 if (UNLIKELY(ompt_enabled.enabled)) 1767 __ompt_task_start(task, current_task, gtid); 1768 #endif 1769 #if OMPT_SUPPORT && OMPT_OPTIONAL 1770 if (UNLIKELY(ompt_enabled.ompt_callback_dispatch && 1771 taskdata->ompt_task_info.dispatch_chunk.iterations > 0)) { 1772 ompt_data_t instance = ompt_data_none; 1773 instance.ptr = &(taskdata->ompt_task_info.dispatch_chunk); 1774 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 1775 ompt_callbacks.ompt_callback(ompt_callback_dispatch)( 1776 &(team_info->parallel_data), &(taskdata->ompt_task_info.task_data), 1777 ompt_dispatch_taskloop_chunk, instance); 1778 taskdata->ompt_task_info.dispatch_chunk = {0, 0}; 1779 } 1780 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1781 1782 #if OMPD_SUPPORT 1783 if (ompd_state & OMPD_ENABLE_BP) 1784 ompd_bp_task_begin(); 1785 #endif 1786 1787 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1788 kmp_uint64 cur_time; 1789 kmp_int32 kmp_itt_count_task = 1790 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial && 1791 current_task->td_flags.tasktype == TASK_IMPLICIT; 1792 if (kmp_itt_count_task) { 1793 thread = __kmp_threads[gtid]; 1794 // Time outer level explicit task on barrier for adjusting imbalance time 1795 if (thread->th.th_bar_arrive_time) 1796 cur_time = __itt_get_timestamp(); 1797 else 1798 kmp_itt_count_task = 0; // thread is not on a barrier - skip timing 1799 } 1800 KMP_FSYNC_ACQUIRED(taskdata); // acquired self (new task) 1801 #endif 1802 1803 #if ENABLE_LIBOMPTARGET 1804 if (taskdata->td_target_data.async_handle != NULL) { 1805 // If we have a valid target async handle, that means that we have already 1806 // executed the task routine once. We must query for the handle completion 1807 // instead of re-executing the routine. 1808 __tgt_target_nowait_query(&taskdata->td_target_data.async_handle); 1809 } else 1810 #endif 1811 if (task->routine != NULL) { 1812 #ifdef KMP_GOMP_COMPAT 1813 if (taskdata->td_flags.native) { 1814 ((void (*)(void *))(*(task->routine)))(task->shareds); 1815 } else 1816 #endif /* KMP_GOMP_COMPAT */ 1817 { 1818 (*(task->routine))(gtid, task); 1819 } 1820 } 1821 KMP_POP_PARTITIONED_TIMER(); 1822 1823 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1824 if (kmp_itt_count_task) { 1825 // Barrier imbalance - adjust arrive time with the task duration 1826 thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time); 1827 } 1828 KMP_FSYNC_CANCEL(taskdata); // destroy self (just executed) 1829 KMP_FSYNC_RELEASING(taskdata->td_parent); // releasing parent 1830 #endif 1831 } 1832 1833 #if OMPD_SUPPORT 1834 if (ompd_state & OMPD_ENABLE_BP) 1835 ompd_bp_task_end(); 1836 #endif 1837 1838 // Proxy tasks are not handled by the runtime 1839 if (taskdata->td_flags.proxy != TASK_PROXY) { 1840 #if OMPT_SUPPORT 1841 if (UNLIKELY(ompt_enabled.enabled)) { 1842 thread->th.ompt_thread_info = oldInfo; 1843 if (taskdata->td_flags.tiedness == TASK_TIED) { 1844 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1845 } 1846 __kmp_task_finish<true>(gtid, task, current_task); 1847 } else 1848 #endif 1849 __kmp_task_finish<false>(gtid, task, current_task); 1850 } 1851 1852 KA_TRACE( 1853 30, 1854 ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n", 1855 gtid, taskdata, current_task)); 1856 return; 1857 } 1858 1859 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution 1860 // 1861 // loc_ref: location of original task pragma (ignored) 1862 // gtid: Global Thread ID of encountering thread 1863 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task'' 1864 // Returns: 1865 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1866 // be resumed later. 1867 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1868 // resumed later. 1869 kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid, 1870 kmp_task_t *new_task) { 1871 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1872 1873 KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid, 1874 loc_ref, new_taskdata)); 1875 1876 #if OMPT_SUPPORT 1877 kmp_taskdata_t *parent; 1878 if (UNLIKELY(ompt_enabled.enabled)) { 1879 parent = new_taskdata->td_parent; 1880 if (ompt_enabled.ompt_callback_task_create) { 1881 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1882 &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame), 1883 &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0, 1884 OMPT_GET_RETURN_ADDRESS(0)); 1885 } 1886 } 1887 #endif 1888 1889 /* Should we execute the new task or queue it? For now, let's just always try 1890 to queue it. If the queue fills up, then we'll execute it. */ 1891 1892 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1893 { // Execute this task immediately 1894 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1895 new_taskdata->td_flags.task_serial = 1; 1896 __kmp_invoke_task(gtid, new_task, current_task); 1897 } 1898 1899 KA_TRACE( 1900 10, 1901 ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: " 1902 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", 1903 gtid, loc_ref, new_taskdata)); 1904 1905 #if OMPT_SUPPORT 1906 if (UNLIKELY(ompt_enabled.enabled)) { 1907 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1908 } 1909 #endif 1910 return TASK_CURRENT_NOT_QUEUED; 1911 } 1912 1913 // __kmp_omp_task: Schedule a non-thread-switchable task for execution 1914 // 1915 // gtid: Global Thread ID of encountering thread 1916 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc() 1917 // serialize_immediate: if TRUE then if the task is executed immediately its 1918 // execution will be serialized 1919 // Returns: 1920 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1921 // be resumed later. 1922 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1923 // resumed later. 1924 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task, 1925 bool serialize_immediate) { 1926 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1927 1928 /* Should we execute the new task or queue it? For now, let's just always try 1929 to queue it. If the queue fills up, then we'll execute it. */ 1930 if (new_taskdata->td_flags.proxy == TASK_PROXY || 1931 __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1932 { // Execute this task immediately 1933 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1934 if (serialize_immediate) 1935 new_taskdata->td_flags.task_serial = 1; 1936 __kmp_invoke_task(gtid, new_task, current_task); 1937 } else if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME && 1938 __kmp_wpolicy_passive) { 1939 kmp_info_t *this_thr = __kmp_threads[gtid]; 1940 kmp_team_t *team = this_thr->th.th_team; 1941 kmp_int32 nthreads = this_thr->th.th_team_nproc; 1942 for (int i = 0; i < nthreads; ++i) { 1943 kmp_info_t *thread = team->t.t_threads[i]; 1944 if (thread == this_thr) 1945 continue; 1946 if (thread->th.th_sleep_loc != NULL) { 1947 __kmp_null_resume_wrapper(thread); 1948 break; // awake one thread at a time 1949 } 1950 } 1951 } 1952 return TASK_CURRENT_NOT_QUEUED; 1953 } 1954 1955 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a 1956 // non-thread-switchable task from the parent thread only! 1957 // 1958 // loc_ref: location of original task pragma (ignored) 1959 // gtid: Global Thread ID of encountering thread 1960 // new_task: non-thread-switchable task thunk allocated by 1961 // __kmp_omp_task_alloc() 1962 // Returns: 1963 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1964 // be resumed later. 1965 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1966 // resumed later. 1967 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid, 1968 kmp_task_t *new_task) { 1969 kmp_int32 res; 1970 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 1971 1972 #if KMP_DEBUG || OMPT_SUPPORT 1973 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1974 #endif 1975 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 1976 new_taskdata)); 1977 __kmp_assert_valid_gtid(gtid); 1978 1979 #if OMPT_SUPPORT 1980 kmp_taskdata_t *parent = NULL; 1981 if (UNLIKELY(ompt_enabled.enabled)) { 1982 if (!new_taskdata->td_flags.started) { 1983 OMPT_STORE_RETURN_ADDRESS(gtid); 1984 parent = new_taskdata->td_parent; 1985 if (!parent->ompt_task_info.frame.enter_frame.ptr) { 1986 parent->ompt_task_info.frame.enter_frame.ptr = 1987 OMPT_GET_FRAME_ADDRESS(0); 1988 } 1989 if (ompt_enabled.ompt_callback_task_create) { 1990 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1991 &(parent->ompt_task_info.task_data), 1992 &(parent->ompt_task_info.frame), 1993 &(new_taskdata->ompt_task_info.task_data), 1994 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 1995 OMPT_LOAD_RETURN_ADDRESS(gtid)); 1996 } 1997 } else { 1998 // We are scheduling the continuation of an UNTIED task. 1999 // Scheduling back to the parent task. 2000 __ompt_task_finish(new_task, 2001 new_taskdata->ompt_task_info.scheduling_parent, 2002 ompt_task_switch); 2003 new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 2004 } 2005 } 2006 #endif 2007 2008 res = __kmp_omp_task(gtid, new_task, true); 2009 2010 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 2011 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 2012 gtid, loc_ref, new_taskdata)); 2013 #if OMPT_SUPPORT 2014 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 2015 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 2016 } 2017 #endif 2018 return res; 2019 } 2020 2021 // __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule 2022 // a taskloop task with the correct OMPT return address 2023 // 2024 // loc_ref: location of original task pragma (ignored) 2025 // gtid: Global Thread ID of encountering thread 2026 // new_task: non-thread-switchable task thunk allocated by 2027 // __kmp_omp_task_alloc() 2028 // codeptr_ra: return address for OMPT callback 2029 // Returns: 2030 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 2031 // be resumed later. 2032 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 2033 // resumed later. 2034 kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid, 2035 kmp_task_t *new_task, void *codeptr_ra) { 2036 kmp_int32 res; 2037 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 2038 2039 #if KMP_DEBUG || OMPT_SUPPORT 2040 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 2041 #endif 2042 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 2043 new_taskdata)); 2044 2045 #if OMPT_SUPPORT 2046 kmp_taskdata_t *parent = NULL; 2047 if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) { 2048 parent = new_taskdata->td_parent; 2049 if (!parent->ompt_task_info.frame.enter_frame.ptr) 2050 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 2051 if (ompt_enabled.ompt_callback_task_create) { 2052 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 2053 &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame), 2054 &(new_taskdata->ompt_task_info.task_data), 2055 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 2056 codeptr_ra); 2057 } 2058 } 2059 #endif 2060 2061 res = __kmp_omp_task(gtid, new_task, true); 2062 2063 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 2064 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 2065 gtid, loc_ref, new_taskdata)); 2066 #if OMPT_SUPPORT 2067 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 2068 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 2069 } 2070 #endif 2071 return res; 2072 } 2073 2074 template <bool ompt> 2075 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid, 2076 void *frame_address, 2077 void *return_address) { 2078 kmp_taskdata_t *taskdata = nullptr; 2079 kmp_info_t *thread; 2080 int thread_finished = FALSE; 2081 KMP_SET_THREAD_STATE_BLOCK(TASKWAIT); 2082 2083 KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref)); 2084 KMP_DEBUG_ASSERT(gtid >= 0); 2085 2086 if (__kmp_tasking_mode != tskm_immediate_exec) { 2087 thread = __kmp_threads[gtid]; 2088 taskdata = thread->th.th_current_task; 2089 2090 #if OMPT_SUPPORT && OMPT_OPTIONAL 2091 ompt_data_t *my_task_data; 2092 ompt_data_t *my_parallel_data; 2093 2094 if (ompt) { 2095 my_task_data = &(taskdata->ompt_task_info.task_data); 2096 my_parallel_data = OMPT_CUR_TEAM_DATA(thread); 2097 2098 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address; 2099 2100 if (ompt_enabled.ompt_callback_sync_region) { 2101 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2102 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 2103 my_task_data, return_address); 2104 } 2105 2106 if (ompt_enabled.ompt_callback_sync_region_wait) { 2107 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2108 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 2109 my_task_data, return_address); 2110 } 2111 } 2112 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 2113 2114 // Debugger: The taskwait is active. Store location and thread encountered the 2115 // taskwait. 2116 #if USE_ITT_BUILD 2117 // Note: These values are used by ITT events as well. 2118 #endif /* USE_ITT_BUILD */ 2119 taskdata->td_taskwait_counter += 1; 2120 taskdata->td_taskwait_ident = loc_ref; 2121 taskdata->td_taskwait_thread = gtid + 1; 2122 2123 #if USE_ITT_BUILD 2124 void *itt_sync_obj = NULL; 2125 #if USE_ITT_NOTIFY 2126 KMP_ITT_TASKWAIT_STARTING(itt_sync_obj); 2127 #endif /* USE_ITT_NOTIFY */ 2128 #endif /* USE_ITT_BUILD */ 2129 2130 bool must_wait = 2131 !taskdata->td_flags.team_serial && !taskdata->td_flags.final; 2132 2133 must_wait = must_wait || (thread->th.th_task_team != NULL && 2134 thread->th.th_task_team->tt.tt_found_proxy_tasks); 2135 // If hidden helper thread is encountered, we must enable wait here. 2136 must_wait = 2137 must_wait || 2138 (__kmp_enable_hidden_helper && thread->th.th_task_team != NULL && 2139 thread->th.th_task_team->tt.tt_hidden_helper_task_encountered); 2140 2141 if (must_wait) { 2142 kmp_flag_32<false, false> flag( 2143 RCAST(std::atomic<kmp_uint32> *, 2144 &(taskdata->td_incomplete_child_tasks)), 2145 0U); 2146 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) { 2147 flag.execute_tasks(thread, gtid, FALSE, 2148 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 2149 __kmp_task_stealing_constraint); 2150 } 2151 } 2152 #if USE_ITT_BUILD 2153 KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj); 2154 KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with children 2155 #endif /* USE_ITT_BUILD */ 2156 2157 // Debugger: The taskwait is completed. Location remains, but thread is 2158 // negated. 2159 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 2160 2161 #if OMPT_SUPPORT && OMPT_OPTIONAL 2162 if (ompt) { 2163 if (ompt_enabled.ompt_callback_sync_region_wait) { 2164 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2165 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 2166 my_task_data, return_address); 2167 } 2168 if (ompt_enabled.ompt_callback_sync_region) { 2169 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2170 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 2171 my_task_data, return_address); 2172 } 2173 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none; 2174 } 2175 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 2176 2177 } 2178 2179 KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, " 2180 "returning TASK_CURRENT_NOT_QUEUED\n", 2181 gtid, taskdata)); 2182 2183 return TASK_CURRENT_NOT_QUEUED; 2184 } 2185 2186 #if OMPT_SUPPORT && OMPT_OPTIONAL 2187 OMPT_NOINLINE 2188 static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid, 2189 void *frame_address, 2190 void *return_address) { 2191 return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address, 2192 return_address); 2193 } 2194 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 2195 2196 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are 2197 // complete 2198 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) { 2199 #if OMPT_SUPPORT && OMPT_OPTIONAL 2200 if (UNLIKELY(ompt_enabled.enabled)) { 2201 OMPT_STORE_RETURN_ADDRESS(gtid); 2202 return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0), 2203 OMPT_LOAD_RETURN_ADDRESS(gtid)); 2204 } 2205 #endif 2206 return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL); 2207 } 2208 2209 // __kmpc_omp_taskyield: switch to a different task 2210 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) { 2211 kmp_taskdata_t *taskdata = NULL; 2212 kmp_info_t *thread; 2213 int thread_finished = FALSE; 2214 2215 KMP_COUNT_BLOCK(OMP_TASKYIELD); 2216 KMP_SET_THREAD_STATE_BLOCK(TASKYIELD); 2217 2218 KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n", 2219 gtid, loc_ref, end_part)); 2220 __kmp_assert_valid_gtid(gtid); 2221 2222 if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) { 2223 thread = __kmp_threads[gtid]; 2224 taskdata = thread->th.th_current_task; 2225 // Should we model this as a task wait or not? 2226 // Debugger: The taskwait is active. Store location and thread encountered the 2227 // taskwait. 2228 #if USE_ITT_BUILD 2229 // Note: These values are used by ITT events as well. 2230 #endif /* USE_ITT_BUILD */ 2231 taskdata->td_taskwait_counter += 1; 2232 taskdata->td_taskwait_ident = loc_ref; 2233 taskdata->td_taskwait_thread = gtid + 1; 2234 2235 #if USE_ITT_BUILD 2236 void *itt_sync_obj = NULL; 2237 #if USE_ITT_NOTIFY 2238 KMP_ITT_TASKWAIT_STARTING(itt_sync_obj); 2239 #endif /* USE_ITT_NOTIFY */ 2240 #endif /* USE_ITT_BUILD */ 2241 if (!taskdata->td_flags.team_serial) { 2242 kmp_task_team_t *task_team = thread->th.th_task_team; 2243 if (task_team != NULL) { 2244 if (KMP_TASKING_ENABLED(task_team)) { 2245 #if OMPT_SUPPORT 2246 if (UNLIKELY(ompt_enabled.enabled)) 2247 thread->th.ompt_thread_info.ompt_task_yielded = 1; 2248 #endif 2249 __kmp_execute_tasks_32( 2250 thread, gtid, (kmp_flag_32<> *)NULL, FALSE, 2251 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 2252 __kmp_task_stealing_constraint); 2253 #if OMPT_SUPPORT 2254 if (UNLIKELY(ompt_enabled.enabled)) 2255 thread->th.ompt_thread_info.ompt_task_yielded = 0; 2256 #endif 2257 } 2258 } 2259 } 2260 #if USE_ITT_BUILD 2261 KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj); 2262 #endif /* USE_ITT_BUILD */ 2263 2264 // Debugger: The taskwait is completed. Location remains, but thread is 2265 // negated. 2266 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 2267 } 2268 2269 KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, " 2270 "returning TASK_CURRENT_NOT_QUEUED\n", 2271 gtid, taskdata)); 2272 2273 return TASK_CURRENT_NOT_QUEUED; 2274 } 2275 2276 // Task Reduction implementation 2277 // 2278 // Note: initial implementation didn't take into account the possibility 2279 // to specify omp_orig for initializer of the UDR (user defined reduction). 2280 // Corrected implementation takes into account the omp_orig object. 2281 // Compiler is free to use old implementation if omp_orig is not specified. 2282 2283 /*! 2284 @ingroup BASIC_TYPES 2285 @{ 2286 */ 2287 2288 /*! 2289 Flags for special info per task reduction item. 2290 */ 2291 typedef struct kmp_taskred_flags { 2292 /*! 1 - use lazy alloc/init (e.g. big objects, num tasks < num threads) */ 2293 unsigned lazy_priv : 1; 2294 unsigned reserved31 : 31; 2295 } kmp_taskred_flags_t; 2296 2297 /*! 2298 Internal struct for reduction data item related info set up by compiler. 2299 */ 2300 typedef struct kmp_task_red_input { 2301 void *reduce_shar; /**< shared between tasks item to reduce into */ 2302 size_t reduce_size; /**< size of data item in bytes */ 2303 // three compiler-generated routines (init, fini are optional): 2304 void *reduce_init; /**< data initialization routine (single parameter) */ 2305 void *reduce_fini; /**< data finalization routine */ 2306 void *reduce_comb; /**< data combiner routine */ 2307 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2308 } kmp_task_red_input_t; 2309 2310 /*! 2311 Internal struct for reduction data item related info saved by the library. 2312 */ 2313 typedef struct kmp_taskred_data { 2314 void *reduce_shar; /**< shared between tasks item to reduce into */ 2315 size_t reduce_size; /**< size of data item */ 2316 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2317 void *reduce_priv; /**< array of thread specific items */ 2318 void *reduce_pend; /**< end of private data for faster comparison op */ 2319 // three compiler-generated routines (init, fini are optional): 2320 void *reduce_comb; /**< data combiner routine */ 2321 void *reduce_init; /**< data initialization routine (two parameters) */ 2322 void *reduce_fini; /**< data finalization routine */ 2323 void *reduce_orig; /**< original item (can be used in UDR initializer) */ 2324 } kmp_taskred_data_t; 2325 2326 /*! 2327 Internal struct for reduction data item related info set up by compiler. 2328 2329 New interface: added reduce_orig field to provide omp_orig for UDR initializer. 2330 */ 2331 typedef struct kmp_taskred_input { 2332 void *reduce_shar; /**< shared between tasks item to reduce into */ 2333 void *reduce_orig; /**< original reduction item used for initialization */ 2334 size_t reduce_size; /**< size of data item */ 2335 // three compiler-generated routines (init, fini are optional): 2336 void *reduce_init; /**< data initialization routine (two parameters) */ 2337 void *reduce_fini; /**< data finalization routine */ 2338 void *reduce_comb; /**< data combiner routine */ 2339 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2340 } kmp_taskred_input_t; 2341 /*! 2342 @} 2343 */ 2344 2345 template <typename T> void __kmp_assign_orig(kmp_taskred_data_t &item, T &src); 2346 template <> 2347 void __kmp_assign_orig<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2348 kmp_task_red_input_t &src) { 2349 item.reduce_orig = NULL; 2350 } 2351 template <> 2352 void __kmp_assign_orig<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2353 kmp_taskred_input_t &src) { 2354 if (src.reduce_orig != NULL) { 2355 item.reduce_orig = src.reduce_orig; 2356 } else { 2357 item.reduce_orig = src.reduce_shar; 2358 } // non-NULL reduce_orig means new interface used 2359 } 2360 2361 template <typename T> void __kmp_call_init(kmp_taskred_data_t &item, size_t j); 2362 template <> 2363 void __kmp_call_init<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2364 size_t offset) { 2365 ((void (*)(void *))item.reduce_init)((char *)(item.reduce_priv) + offset); 2366 } 2367 template <> 2368 void __kmp_call_init<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2369 size_t offset) { 2370 ((void (*)(void *, void *))item.reduce_init)( 2371 (char *)(item.reduce_priv) + offset, item.reduce_orig); 2372 } 2373 2374 template <typename T> 2375 void *__kmp_task_reduction_init(int gtid, int num, T *data) { 2376 __kmp_assert_valid_gtid(gtid); 2377 kmp_info_t *thread = __kmp_threads[gtid]; 2378 kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup; 2379 kmp_uint32 nth = thread->th.th_team_nproc; 2380 kmp_taskred_data_t *arr; 2381 2382 // check input data just in case 2383 KMP_ASSERT(tg != NULL); 2384 KMP_ASSERT(data != NULL); 2385 KMP_ASSERT(num > 0); 2386 if (nth == 1) { 2387 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n", 2388 gtid, tg)); 2389 return (void *)tg; 2390 } 2391 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n", 2392 gtid, tg, num)); 2393 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2394 thread, num * sizeof(kmp_taskred_data_t)); 2395 for (int i = 0; i < num; ++i) { 2396 size_t size = data[i].reduce_size - 1; 2397 // round the size up to cache line per thread-specific item 2398 size += CACHE_LINE - size % CACHE_LINE; 2399 KMP_ASSERT(data[i].reduce_comb != NULL); // combiner is mandatory 2400 arr[i].reduce_shar = data[i].reduce_shar; 2401 arr[i].reduce_size = size; 2402 arr[i].flags = data[i].flags; 2403 arr[i].reduce_comb = data[i].reduce_comb; 2404 arr[i].reduce_init = data[i].reduce_init; 2405 arr[i].reduce_fini = data[i].reduce_fini; 2406 __kmp_assign_orig<T>(arr[i], data[i]); 2407 if (!arr[i].flags.lazy_priv) { 2408 // allocate cache-line aligned block and fill it with zeros 2409 arr[i].reduce_priv = __kmp_allocate(nth * size); 2410 arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size; 2411 if (arr[i].reduce_init != NULL) { 2412 // initialize all thread-specific items 2413 for (size_t j = 0; j < nth; ++j) { 2414 __kmp_call_init<T>(arr[i], j * size); 2415 } 2416 } 2417 } else { 2418 // only allocate space for pointers now, 2419 // objects will be lazily allocated/initialized if/when requested 2420 // note that __kmp_allocate zeroes the allocated memory 2421 arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *)); 2422 } 2423 } 2424 tg->reduce_data = (void *)arr; 2425 tg->reduce_num_data = num; 2426 return (void *)tg; 2427 } 2428 2429 /*! 2430 @ingroup TASKING 2431 @param gtid Global thread ID 2432 @param num Number of data items to reduce 2433 @param data Array of data for reduction 2434 @return The taskgroup identifier 2435 2436 Initialize task reduction for the taskgroup. 2437 2438 Note: this entry supposes the optional compiler-generated initializer routine 2439 has single parameter - pointer to object to be initialized. That means 2440 the reduction either does not use omp_orig object, or the omp_orig is accessible 2441 without help of the runtime library. 2442 */ 2443 void *__kmpc_task_reduction_init(int gtid, int num, void *data) { 2444 return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data); 2445 } 2446 2447 /*! 2448 @ingroup TASKING 2449 @param gtid Global thread ID 2450 @param num Number of data items to reduce 2451 @param data Array of data for reduction 2452 @return The taskgroup identifier 2453 2454 Initialize task reduction for the taskgroup. 2455 2456 Note: this entry supposes the optional compiler-generated initializer routine 2457 has two parameters, pointer to object to be initialized and pointer to omp_orig 2458 */ 2459 void *__kmpc_taskred_init(int gtid, int num, void *data) { 2460 return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data); 2461 } 2462 2463 // Copy task reduction data (except for shared pointers). 2464 template <typename T> 2465 void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data, 2466 kmp_taskgroup_t *tg, void *reduce_data) { 2467 kmp_taskred_data_t *arr; 2468 KA_TRACE(20, ("__kmp_task_reduction_init_copy: Th %p, init taskgroup %p," 2469 " from data %p\n", 2470 thr, tg, reduce_data)); 2471 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2472 thr, num * sizeof(kmp_taskred_data_t)); 2473 // threads will share private copies, thunk routines, sizes, flags, etc.: 2474 KMP_MEMCPY(arr, reduce_data, num * sizeof(kmp_taskred_data_t)); 2475 for (int i = 0; i < num; ++i) { 2476 arr[i].reduce_shar = data[i].reduce_shar; // init unique shared pointers 2477 } 2478 tg->reduce_data = (void *)arr; 2479 tg->reduce_num_data = num; 2480 } 2481 2482 /*! 2483 @ingroup TASKING 2484 @param gtid Global thread ID 2485 @param tskgrp The taskgroup ID (optional) 2486 @param data Shared location of the item 2487 @return The pointer to per-thread data 2488 2489 Get thread-specific location of data item 2490 */ 2491 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) { 2492 __kmp_assert_valid_gtid(gtid); 2493 kmp_info_t *thread = __kmp_threads[gtid]; 2494 kmp_int32 nth = thread->th.th_team_nproc; 2495 if (nth == 1) 2496 return data; // nothing to do 2497 2498 kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp; 2499 if (tg == NULL) 2500 tg = thread->th.th_current_task->td_taskgroup; 2501 KMP_ASSERT(tg != NULL); 2502 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)(tg->reduce_data); 2503 kmp_int32 num = tg->reduce_num_data; 2504 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 2505 2506 KMP_ASSERT(data != NULL); 2507 while (tg != NULL) { 2508 for (int i = 0; i < num; ++i) { 2509 if (!arr[i].flags.lazy_priv) { 2510 if (data == arr[i].reduce_shar || 2511 (data >= arr[i].reduce_priv && data < arr[i].reduce_pend)) 2512 return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size; 2513 } else { 2514 // check shared location first 2515 void **p_priv = (void **)(arr[i].reduce_priv); 2516 if (data == arr[i].reduce_shar) 2517 goto found; 2518 // check if we get some thread specific location as parameter 2519 for (int j = 0; j < nth; ++j) 2520 if (data == p_priv[j]) 2521 goto found; 2522 continue; // not found, continue search 2523 found: 2524 if (p_priv[tid] == NULL) { 2525 // allocate thread specific object lazily 2526 p_priv[tid] = __kmp_allocate(arr[i].reduce_size); 2527 if (arr[i].reduce_init != NULL) { 2528 if (arr[i].reduce_orig != NULL) { // new interface 2529 ((void (*)(void *, void *))arr[i].reduce_init)( 2530 p_priv[tid], arr[i].reduce_orig); 2531 } else { // old interface (single parameter) 2532 ((void (*)(void *))arr[i].reduce_init)(p_priv[tid]); 2533 } 2534 } 2535 } 2536 return p_priv[tid]; 2537 } 2538 } 2539 tg = tg->parent; 2540 arr = (kmp_taskred_data_t *)(tg->reduce_data); 2541 num = tg->reduce_num_data; 2542 } 2543 KMP_ASSERT2(0, "Unknown task reduction item"); 2544 return NULL; // ERROR, this line never executed 2545 } 2546 2547 // Finalize task reduction. 2548 // Called from __kmpc_end_taskgroup() 2549 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) { 2550 kmp_int32 nth = th->th.th_team_nproc; 2551 KMP_DEBUG_ASSERT(nth > 1); // should not be called if nth == 1 2552 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)tg->reduce_data; 2553 kmp_int32 num = tg->reduce_num_data; 2554 for (int i = 0; i < num; ++i) { 2555 void *sh_data = arr[i].reduce_shar; 2556 void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini); 2557 void (*f_comb)(void *, void *) = 2558 (void (*)(void *, void *))(arr[i].reduce_comb); 2559 if (!arr[i].flags.lazy_priv) { 2560 void *pr_data = arr[i].reduce_priv; 2561 size_t size = arr[i].reduce_size; 2562 for (int j = 0; j < nth; ++j) { 2563 void *priv_data = (char *)pr_data + j * size; 2564 f_comb(sh_data, priv_data); // combine results 2565 if (f_fini) 2566 f_fini(priv_data); // finalize if needed 2567 } 2568 } else { 2569 void **pr_data = (void **)(arr[i].reduce_priv); 2570 for (int j = 0; j < nth; ++j) { 2571 if (pr_data[j] != NULL) { 2572 f_comb(sh_data, pr_data[j]); // combine results 2573 if (f_fini) 2574 f_fini(pr_data[j]); // finalize if needed 2575 __kmp_free(pr_data[j]); 2576 } 2577 } 2578 } 2579 __kmp_free(arr[i].reduce_priv); 2580 } 2581 __kmp_thread_free(th, arr); 2582 tg->reduce_data = NULL; 2583 tg->reduce_num_data = 0; 2584 } 2585 2586 // Cleanup task reduction data for parallel or worksharing, 2587 // do not touch task private data other threads still working with. 2588 // Called from __kmpc_end_taskgroup() 2589 static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) { 2590 __kmp_thread_free(th, tg->reduce_data); 2591 tg->reduce_data = NULL; 2592 tg->reduce_num_data = 0; 2593 } 2594 2595 template <typename T> 2596 void *__kmp_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2597 int num, T *data) { 2598 __kmp_assert_valid_gtid(gtid); 2599 kmp_info_t *thr = __kmp_threads[gtid]; 2600 kmp_int32 nth = thr->th.th_team_nproc; 2601 __kmpc_taskgroup(loc, gtid); // form new taskgroup first 2602 if (nth == 1) { 2603 KA_TRACE(10, 2604 ("__kmpc_reduction_modifier_init: T#%d, tg %p, exiting nth=1\n", 2605 gtid, thr->th.th_current_task->td_taskgroup)); 2606 return (void *)thr->th.th_current_task->td_taskgroup; 2607 } 2608 kmp_team_t *team = thr->th.th_team; 2609 void *reduce_data; 2610 kmp_taskgroup_t *tg; 2611 reduce_data = KMP_ATOMIC_LD_RLX(&team->t.t_tg_reduce_data[is_ws]); 2612 if (reduce_data == NULL && 2613 __kmp_atomic_compare_store(&team->t.t_tg_reduce_data[is_ws], reduce_data, 2614 (void *)1)) { 2615 // single thread enters this block to initialize common reduction data 2616 KMP_DEBUG_ASSERT(reduce_data == NULL); 2617 // first initialize own data, then make a copy other threads can use 2618 tg = (kmp_taskgroup_t *)__kmp_task_reduction_init<T>(gtid, num, data); 2619 reduce_data = __kmp_thread_malloc(thr, num * sizeof(kmp_taskred_data_t)); 2620 KMP_MEMCPY(reduce_data, tg->reduce_data, num * sizeof(kmp_taskred_data_t)); 2621 // fini counters should be 0 at this point 2622 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[0]) == 0); 2623 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[1]) == 0); 2624 KMP_ATOMIC_ST_REL(&team->t.t_tg_reduce_data[is_ws], reduce_data); 2625 } else { 2626 while ( 2627 (reduce_data = KMP_ATOMIC_LD_ACQ(&team->t.t_tg_reduce_data[is_ws])) == 2628 (void *)1) { // wait for task reduction initialization 2629 KMP_CPU_PAUSE(); 2630 } 2631 KMP_DEBUG_ASSERT(reduce_data > (void *)1); // should be valid pointer here 2632 tg = thr->th.th_current_task->td_taskgroup; 2633 __kmp_task_reduction_init_copy<T>(thr, num, data, tg, reduce_data); 2634 } 2635 return tg; 2636 } 2637 2638 /*! 2639 @ingroup TASKING 2640 @param loc Source location info 2641 @param gtid Global thread ID 2642 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2643 @param num Number of data items to reduce 2644 @param data Array of data for reduction 2645 @return The taskgroup identifier 2646 2647 Initialize task reduction for a parallel or worksharing. 2648 2649 Note: this entry supposes the optional compiler-generated initializer routine 2650 has single parameter - pointer to object to be initialized. That means 2651 the reduction either does not use omp_orig object, or the omp_orig is accessible 2652 without help of the runtime library. 2653 */ 2654 void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2655 int num, void *data) { 2656 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2657 (kmp_task_red_input_t *)data); 2658 } 2659 2660 /*! 2661 @ingroup TASKING 2662 @param loc Source location info 2663 @param gtid Global thread ID 2664 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2665 @param num Number of data items to reduce 2666 @param data Array of data for reduction 2667 @return The taskgroup identifier 2668 2669 Initialize task reduction for a parallel or worksharing. 2670 2671 Note: this entry supposes the optional compiler-generated initializer routine 2672 has two parameters, pointer to object to be initialized and pointer to omp_orig 2673 */ 2674 void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, 2675 void *data) { 2676 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2677 (kmp_taskred_input_t *)data); 2678 } 2679 2680 /*! 2681 @ingroup TASKING 2682 @param loc Source location info 2683 @param gtid Global thread ID 2684 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2685 2686 Finalize task reduction for a parallel or worksharing. 2687 */ 2688 void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws) { 2689 __kmpc_end_taskgroup(loc, gtid); 2690 } 2691 2692 // __kmpc_taskgroup: Start a new taskgroup 2693 void __kmpc_taskgroup(ident_t *loc, int gtid) { 2694 __kmp_assert_valid_gtid(gtid); 2695 kmp_info_t *thread = __kmp_threads[gtid]; 2696 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2697 kmp_taskgroup_t *tg_new = 2698 (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t)); 2699 KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new)); 2700 KMP_ATOMIC_ST_RLX(&tg_new->count, 0); 2701 KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq); 2702 tg_new->parent = taskdata->td_taskgroup; 2703 tg_new->reduce_data = NULL; 2704 tg_new->reduce_num_data = 0; 2705 tg_new->gomp_data = NULL; 2706 taskdata->td_taskgroup = tg_new; 2707 2708 #if OMPT_SUPPORT && OMPT_OPTIONAL 2709 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2710 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2711 if (!codeptr) 2712 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2713 kmp_team_t *team = thread->th.th_team; 2714 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data; 2715 // FIXME: I think this is wrong for lwt! 2716 ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data; 2717 2718 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2719 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2720 &(my_task_data), codeptr); 2721 } 2722 #endif 2723 } 2724 2725 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task 2726 // and its descendants are complete 2727 void __kmpc_end_taskgroup(ident_t *loc, int gtid) { 2728 __kmp_assert_valid_gtid(gtid); 2729 kmp_info_t *thread = __kmp_threads[gtid]; 2730 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2731 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 2732 int thread_finished = FALSE; 2733 2734 #if OMPT_SUPPORT && OMPT_OPTIONAL 2735 kmp_team_t *team; 2736 ompt_data_t my_task_data; 2737 ompt_data_t my_parallel_data; 2738 void *codeptr = nullptr; 2739 if (UNLIKELY(ompt_enabled.enabled)) { 2740 team = thread->th.th_team; 2741 my_task_data = taskdata->ompt_task_info.task_data; 2742 // FIXME: I think this is wrong for lwt! 2743 my_parallel_data = team->t.ompt_team_info.parallel_data; 2744 codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2745 if (!codeptr) 2746 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2747 } 2748 #endif 2749 2750 KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc)); 2751 KMP_DEBUG_ASSERT(taskgroup != NULL); 2752 KMP_SET_THREAD_STATE_BLOCK(TASKGROUP); 2753 2754 if (__kmp_tasking_mode != tskm_immediate_exec) { 2755 // mark task as waiting not on a barrier 2756 taskdata->td_taskwait_counter += 1; 2757 taskdata->td_taskwait_ident = loc; 2758 taskdata->td_taskwait_thread = gtid + 1; 2759 #if USE_ITT_BUILD 2760 // For ITT the taskgroup wait is similar to taskwait until we need to 2761 // distinguish them 2762 void *itt_sync_obj = NULL; 2763 #if USE_ITT_NOTIFY 2764 KMP_ITT_TASKWAIT_STARTING(itt_sync_obj); 2765 #endif /* USE_ITT_NOTIFY */ 2766 #endif /* USE_ITT_BUILD */ 2767 2768 #if OMPT_SUPPORT && OMPT_OPTIONAL 2769 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2770 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2771 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2772 &(my_task_data), codeptr); 2773 } 2774 #endif 2775 2776 if (!taskdata->td_flags.team_serial || 2777 (thread->th.th_task_team != NULL && 2778 (thread->th.th_task_team->tt.tt_found_proxy_tasks || 2779 thread->th.th_task_team->tt.tt_hidden_helper_task_encountered))) { 2780 kmp_flag_32<false, false> flag( 2781 RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 0U); 2782 while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) { 2783 flag.execute_tasks(thread, gtid, FALSE, 2784 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 2785 __kmp_task_stealing_constraint); 2786 } 2787 } 2788 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting 2789 2790 #if OMPT_SUPPORT && OMPT_OPTIONAL 2791 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2792 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2793 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2794 &(my_task_data), codeptr); 2795 } 2796 #endif 2797 2798 #if USE_ITT_BUILD 2799 KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj); 2800 KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with descendants 2801 #endif /* USE_ITT_BUILD */ 2802 } 2803 KMP_DEBUG_ASSERT(taskgroup->count == 0); 2804 2805 if (taskgroup->reduce_data != NULL && 2806 !taskgroup->gomp_data) { // need to reduce? 2807 int cnt; 2808 void *reduce_data; 2809 kmp_team_t *t = thread->th.th_team; 2810 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)taskgroup->reduce_data; 2811 // check if <priv> data of the first reduction variable shared for the team 2812 void *priv0 = arr[0].reduce_priv; 2813 if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[0])) != NULL && 2814 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2815 // finishing task reduction on parallel 2816 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[0]); 2817 if (cnt == thread->th.th_team_nproc - 1) { 2818 // we are the last thread passing __kmpc_reduction_modifier_fini() 2819 // finalize task reduction: 2820 __kmp_task_reduction_fini(thread, taskgroup); 2821 // cleanup fields in the team structure: 2822 // TODO: is relaxed store enough here (whole barrier should follow)? 2823 __kmp_thread_free(thread, reduce_data); 2824 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[0], NULL); 2825 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[0], 0); 2826 } else { 2827 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2828 // so do not finalize reduction, just clean own copy of the data 2829 __kmp_task_reduction_clean(thread, taskgroup); 2830 } 2831 } else if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[1])) != 2832 NULL && 2833 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2834 // finishing task reduction on worksharing 2835 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[1]); 2836 if (cnt == thread->th.th_team_nproc - 1) { 2837 // we are the last thread passing __kmpc_reduction_modifier_fini() 2838 __kmp_task_reduction_fini(thread, taskgroup); 2839 // cleanup fields in team structure: 2840 // TODO: is relaxed store enough here (whole barrier should follow)? 2841 __kmp_thread_free(thread, reduce_data); 2842 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[1], NULL); 2843 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[1], 0); 2844 } else { 2845 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2846 // so do not finalize reduction, just clean own copy of the data 2847 __kmp_task_reduction_clean(thread, taskgroup); 2848 } 2849 } else { 2850 // finishing task reduction on taskgroup 2851 __kmp_task_reduction_fini(thread, taskgroup); 2852 } 2853 } 2854 // Restore parent taskgroup for the current task 2855 taskdata->td_taskgroup = taskgroup->parent; 2856 __kmp_thread_free(thread, taskgroup); 2857 2858 KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", 2859 gtid, taskdata)); 2860 2861 #if OMPT_SUPPORT && OMPT_OPTIONAL 2862 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2863 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2864 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2865 &(my_task_data), codeptr); 2866 } 2867 #endif 2868 } 2869 2870 static kmp_task_t *__kmp_get_priority_task(kmp_int32 gtid, 2871 kmp_task_team_t *task_team, 2872 kmp_int32 is_constrained) { 2873 kmp_task_t *task = NULL; 2874 kmp_taskdata_t *taskdata; 2875 kmp_taskdata_t *current; 2876 kmp_thread_data_t *thread_data; 2877 int ntasks = task_team->tt.tt_num_task_pri; 2878 if (ntasks == 0) { 2879 KA_TRACE( 2880 20, ("__kmp_get_priority_task(exit #1): T#%d No tasks to get\n", gtid)); 2881 return NULL; 2882 } 2883 do { 2884 // decrement num_tasks to "reserve" one task to get for execution 2885 if (__kmp_atomic_compare_store(&task_team->tt.tt_num_task_pri, ntasks, 2886 ntasks - 1)) 2887 break; 2888 } while (ntasks > 0); 2889 if (ntasks == 0) { 2890 KA_TRACE(20, ("__kmp_get_priority_task(exit #2): T#%d No tasks to get\n", 2891 __kmp_get_gtid())); 2892 return NULL; 2893 } 2894 // We got a "ticket" to get a "reserved" priority task 2895 int deque_ntasks; 2896 kmp_task_pri_t *list = task_team->tt.tt_task_pri_list; 2897 do { 2898 KMP_ASSERT(list != NULL); 2899 thread_data = &list->td; 2900 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 2901 deque_ntasks = thread_data->td.td_deque_ntasks; 2902 if (deque_ntasks == 0) { 2903 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2904 KA_TRACE(20, ("__kmp_get_priority_task: T#%d No tasks to get from %p\n", 2905 __kmp_get_gtid(), thread_data)); 2906 list = list->next; 2907 } 2908 } while (deque_ntasks == 0); 2909 KMP_DEBUG_ASSERT(deque_ntasks); 2910 int target = thread_data->td.td_deque_head; 2911 current = __kmp_threads[gtid]->th.th_current_task; 2912 taskdata = thread_data->td.td_deque[target]; 2913 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2914 // Bump head pointer and Wrap. 2915 thread_data->td.td_deque_head = 2916 (target + 1) & TASK_DEQUE_MASK(thread_data->td); 2917 } else { 2918 if (!task_team->tt.tt_untied_task_encountered) { 2919 // The TSC does not allow to steal victim task 2920 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2921 KA_TRACE(20, ("__kmp_get_priority_task(exit #3): T#%d could not get task " 2922 "from %p: task_team=%p ntasks=%d head=%u tail=%u\n", 2923 gtid, thread_data, task_team, deque_ntasks, target, 2924 thread_data->td.td_deque_tail)); 2925 task_team->tt.tt_num_task_pri++; // atomic inc, restore value 2926 return NULL; 2927 } 2928 int i; 2929 // walk through the deque trying to steal any task 2930 taskdata = NULL; 2931 for (i = 1; i < deque_ntasks; ++i) { 2932 target = (target + 1) & TASK_DEQUE_MASK(thread_data->td); 2933 taskdata = thread_data->td.td_deque[target]; 2934 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2935 break; // found task to execute 2936 } else { 2937 taskdata = NULL; 2938 } 2939 } 2940 if (taskdata == NULL) { 2941 // No appropriate candidate found to execute 2942 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2943 KA_TRACE( 2944 10, ("__kmp_get_priority_task(exit #4): T#%d could not get task from " 2945 "%p: task_team=%p ntasks=%d head=%u tail=%u\n", 2946 gtid, thread_data, task_team, deque_ntasks, 2947 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2948 task_team->tt.tt_num_task_pri++; // atomic inc, restore value 2949 return NULL; 2950 } 2951 int prev = target; 2952 for (i = i + 1; i < deque_ntasks; ++i) { 2953 // shift remaining tasks in the deque left by 1 2954 target = (target + 1) & TASK_DEQUE_MASK(thread_data->td); 2955 thread_data->td.td_deque[prev] = thread_data->td.td_deque[target]; 2956 prev = target; 2957 } 2958 KMP_DEBUG_ASSERT( 2959 thread_data->td.td_deque_tail == 2960 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(thread_data->td))); 2961 thread_data->td.td_deque_tail = target; // tail -= 1 (wrapped)) 2962 } 2963 thread_data->td.td_deque_ntasks = deque_ntasks - 1; 2964 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2965 task = KMP_TASKDATA_TO_TASK(taskdata); 2966 return task; 2967 } 2968 2969 // __kmp_remove_my_task: remove a task from my own deque 2970 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid, 2971 kmp_task_team_t *task_team, 2972 kmp_int32 is_constrained) { 2973 kmp_task_t *task; 2974 kmp_taskdata_t *taskdata; 2975 kmp_thread_data_t *thread_data; 2976 kmp_uint32 tail; 2977 2978 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2979 KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data != 2980 NULL); // Caller should check this condition 2981 2982 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 2983 2984 KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n", 2985 gtid, thread_data->td.td_deque_ntasks, 2986 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2987 2988 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 2989 KA_TRACE(10, 2990 ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: " 2991 "ntasks=%d head=%u tail=%u\n", 2992 gtid, thread_data->td.td_deque_ntasks, 2993 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2994 return NULL; 2995 } 2996 2997 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 2998 2999 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 3000 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3001 KA_TRACE(10, 3002 ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: " 3003 "ntasks=%d head=%u tail=%u\n", 3004 gtid, thread_data->td.td_deque_ntasks, 3005 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 3006 return NULL; 3007 } 3008 3009 tail = (thread_data->td.td_deque_tail - 1) & 3010 TASK_DEQUE_MASK(thread_data->td); // Wrap index. 3011 taskdata = thread_data->td.td_deque[tail]; 3012 3013 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata, 3014 thread->th.th_current_task)) { 3015 // The TSC does not allow to steal victim task 3016 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3017 KA_TRACE(10, 3018 ("__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: " 3019 "ntasks=%d head=%u tail=%u\n", 3020 gtid, thread_data->td.td_deque_ntasks, 3021 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 3022 return NULL; 3023 } 3024 3025 thread_data->td.td_deque_tail = tail; 3026 TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1); 3027 3028 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3029 3030 KA_TRACE(10, ("__kmp_remove_my_task(exit #4): T#%d task %p removed: " 3031 "ntasks=%d head=%u tail=%u\n", 3032 gtid, taskdata, thread_data->td.td_deque_ntasks, 3033 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 3034 3035 task = KMP_TASKDATA_TO_TASK(taskdata); 3036 return task; 3037 } 3038 3039 // __kmp_steal_task: remove a task from another thread's deque 3040 // Assume that calling thread has already checked existence of 3041 // task_team thread_data before calling this routine. 3042 static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid, 3043 kmp_task_team_t *task_team, 3044 std::atomic<kmp_int32> *unfinished_threads, 3045 int *thread_finished, 3046 kmp_int32 is_constrained) { 3047 kmp_task_t *task; 3048 kmp_taskdata_t *taskdata; 3049 kmp_taskdata_t *current; 3050 kmp_thread_data_t *victim_td, *threads_data; 3051 kmp_int32 target; 3052 kmp_int32 victim_tid; 3053 3054 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3055 3056 threads_data = task_team->tt.tt_threads_data; 3057 KMP_DEBUG_ASSERT(threads_data != NULL); // Caller should check this condition 3058 3059 victim_tid = victim_thr->th.th_info.ds.ds_tid; 3060 victim_td = &threads_data[victim_tid]; 3061 3062 KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: " 3063 "task_team=%p ntasks=%d head=%u tail=%u\n", 3064 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 3065 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 3066 victim_td->td.td_deque_tail)); 3067 3068 if (TCR_4(victim_td->td.td_deque_ntasks) == 0) { 3069 KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: " 3070 "task_team=%p ntasks=%d head=%u tail=%u\n", 3071 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 3072 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 3073 victim_td->td.td_deque_tail)); 3074 return NULL; 3075 } 3076 3077 __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock); 3078 3079 int ntasks = TCR_4(victim_td->td.td_deque_ntasks); 3080 // Check again after we acquire the lock 3081 if (ntasks == 0) { 3082 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 3083 KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: " 3084 "task_team=%p ntasks=%d head=%u tail=%u\n", 3085 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 3086 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 3087 return NULL; 3088 } 3089 3090 KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL); 3091 current = __kmp_threads[gtid]->th.th_current_task; 3092 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head]; 3093 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 3094 // Bump head pointer and Wrap. 3095 victim_td->td.td_deque_head = 3096 (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td); 3097 } else { 3098 if (!task_team->tt.tt_untied_task_encountered) { 3099 // The TSC does not allow to steal victim task 3100 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 3101 KA_TRACE(10, ("__kmp_steal_task(exit #3): T#%d could not steal from " 3102 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 3103 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 3104 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 3105 return NULL; 3106 } 3107 int i; 3108 // walk through victim's deque trying to steal any task 3109 target = victim_td->td.td_deque_head; 3110 taskdata = NULL; 3111 for (i = 1; i < ntasks; ++i) { 3112 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 3113 taskdata = victim_td->td.td_deque[target]; 3114 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 3115 break; // found victim task 3116 } else { 3117 taskdata = NULL; 3118 } 3119 } 3120 if (taskdata == NULL) { 3121 // No appropriate candidate to steal found 3122 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 3123 KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from " 3124 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 3125 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 3126 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 3127 return NULL; 3128 } 3129 int prev = target; 3130 for (i = i + 1; i < ntasks; ++i) { 3131 // shift remaining tasks in the deque left by 1 3132 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 3133 victim_td->td.td_deque[prev] = victim_td->td.td_deque[target]; 3134 prev = target; 3135 } 3136 KMP_DEBUG_ASSERT( 3137 victim_td->td.td_deque_tail == 3138 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td))); 3139 victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped)) 3140 } 3141 if (*thread_finished) { 3142 // We need to un-mark this victim as a finished victim. This must be done 3143 // before releasing the lock, or else other threads (starting with the 3144 // primary thread victim) might be prematurely released from the barrier!!! 3145 #if KMP_DEBUG 3146 kmp_int32 count = 3147 #endif 3148 KMP_ATOMIC_INC(unfinished_threads); 3149 KA_TRACE( 3150 20, 3151 ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n", 3152 gtid, count + 1, task_team)); 3153 *thread_finished = FALSE; 3154 } 3155 TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1); 3156 3157 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 3158 3159 KMP_COUNT_BLOCK(TASK_stolen); 3160 KA_TRACE(10, 3161 ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: " 3162 "task_team=%p ntasks=%d head=%u tail=%u\n", 3163 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, 3164 ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 3165 3166 task = KMP_TASKDATA_TO_TASK(taskdata); 3167 return task; 3168 } 3169 3170 // __kmp_execute_tasks_template: Choose and execute tasks until either the 3171 // condition is statisfied (return true) or there are none left (return false). 3172 // 3173 // final_spin is TRUE if this is the spin at the release barrier. 3174 // thread_finished indicates whether the thread is finished executing all 3175 // the tasks it has on its deque, and is at the release barrier. 3176 // spinner is the location on which to spin. 3177 // spinner == NULL means only execute a single task and return. 3178 // checker is the value to check to terminate the spin. 3179 template <class C> 3180 static inline int __kmp_execute_tasks_template( 3181 kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin, 3182 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3183 kmp_int32 is_constrained) { 3184 kmp_task_team_t *task_team = thread->th.th_task_team; 3185 kmp_thread_data_t *threads_data; 3186 kmp_task_t *task; 3187 kmp_info_t *other_thread; 3188 kmp_taskdata_t *current_task = thread->th.th_current_task; 3189 std::atomic<kmp_int32> *unfinished_threads; 3190 kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0, 3191 tid = thread->th.th_info.ds.ds_tid; 3192 3193 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3194 KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]); 3195 3196 if (task_team == NULL || current_task == NULL) 3197 return FALSE; 3198 3199 KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d " 3200 "*thread_finished=%d\n", 3201 gtid, final_spin, *thread_finished)); 3202 3203 thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 3204 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 3205 3206 KMP_DEBUG_ASSERT(threads_data != NULL); 3207 3208 nthreads = task_team->tt.tt_nproc; 3209 unfinished_threads = &(task_team->tt.tt_unfinished_threads); 3210 KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks || 3211 task_team->tt.tt_hidden_helper_task_encountered); 3212 KMP_DEBUG_ASSERT(*unfinished_threads >= 0); 3213 3214 while (1) { // Outer loop keeps trying to find tasks in case of single thread 3215 // getting tasks from target constructs 3216 while (1) { // Inner loop to find a task and execute it 3217 task = NULL; 3218 if (task_team->tt.tt_num_task_pri) { // get priority task first 3219 task = __kmp_get_priority_task(gtid, task_team, is_constrained); 3220 } 3221 if (task == NULL && use_own_tasks) { // check own queue next 3222 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained); 3223 } 3224 if ((task == NULL) && (nthreads > 1)) { // Steal a task finally 3225 int asleep = 1; 3226 use_own_tasks = 0; 3227 // Try to steal from the last place I stole from successfully. 3228 if (victim_tid == -2) { // haven't stolen anything yet 3229 victim_tid = threads_data[tid].td.td_deque_last_stolen; 3230 if (victim_tid != 3231 -1) // if we have a last stolen from victim, get the thread 3232 other_thread = threads_data[victim_tid].td.td_thr; 3233 } 3234 if (victim_tid != -1) { // found last victim 3235 asleep = 0; 3236 } else if (!new_victim) { // no recent steals and we haven't already 3237 // used a new victim; select a random thread 3238 do { // Find a different thread to steal work from. 3239 // Pick a random thread. Initial plan was to cycle through all the 3240 // threads, and only return if we tried to steal from every thread, 3241 // and failed. Arch says that's not such a great idea. 3242 victim_tid = __kmp_get_random(thread) % (nthreads - 1); 3243 if (victim_tid >= tid) { 3244 ++victim_tid; // Adjusts random distribution to exclude self 3245 } 3246 // Found a potential victim 3247 other_thread = threads_data[victim_tid].td.td_thr; 3248 // There is a slight chance that __kmp_enable_tasking() did not wake 3249 // up all threads waiting at the barrier. If victim is sleeping, 3250 // then wake it up. Since we were going to pay the cache miss 3251 // penalty for referencing another thread's kmp_info_t struct 3252 // anyway, 3253 // the check shouldn't cost too much performance at this point. In 3254 // extra barrier mode, tasks do not sleep at the separate tasking 3255 // barrier, so this isn't a problem. 3256 asleep = 0; 3257 if ((__kmp_tasking_mode == tskm_task_teams) && 3258 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) && 3259 (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc)) != 3260 NULL)) { 3261 asleep = 1; 3262 __kmp_null_resume_wrapper(other_thread); 3263 // A sleeping thread should not have any tasks on it's queue. 3264 // There is a slight possibility that it resumes, steals a task 3265 // from another thread, which spawns more tasks, all in the time 3266 // that it takes this thread to check => don't write an assertion 3267 // that the victim's queue is empty. Try stealing from a 3268 // different thread. 3269 } 3270 } while (asleep); 3271 } 3272 3273 if (!asleep) { 3274 // We have a victim to try to steal from 3275 task = __kmp_steal_task(other_thread, gtid, task_team, 3276 unfinished_threads, thread_finished, 3277 is_constrained); 3278 } 3279 if (task != NULL) { // set last stolen to victim 3280 if (threads_data[tid].td.td_deque_last_stolen != victim_tid) { 3281 threads_data[tid].td.td_deque_last_stolen = victim_tid; 3282 // The pre-refactored code did not try more than 1 successful new 3283 // vicitm, unless the last one generated more local tasks; 3284 // new_victim keeps track of this 3285 new_victim = 1; 3286 } 3287 } else { // No tasks found; unset last_stolen 3288 KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1); 3289 victim_tid = -2; // no successful victim found 3290 } 3291 } 3292 3293 if (task == NULL) 3294 break; // break out of tasking loop 3295 3296 // Found a task; execute it 3297 #if USE_ITT_BUILD && USE_ITT_NOTIFY 3298 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 3299 if (itt_sync_obj == NULL) { // we are at fork barrier where we could not 3300 // get the object reliably 3301 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 3302 } 3303 __kmp_itt_task_starting(itt_sync_obj); 3304 } 3305 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 3306 __kmp_invoke_task(gtid, task, current_task); 3307 #if USE_ITT_BUILD 3308 if (itt_sync_obj != NULL) 3309 __kmp_itt_task_finished(itt_sync_obj); 3310 #endif /* USE_ITT_BUILD */ 3311 // If this thread is only partway through the barrier and the condition is 3312 // met, then return now, so that the barrier gather/release pattern can 3313 // proceed. If this thread is in the last spin loop in the barrier, 3314 // waiting to be released, we know that the termination condition will not 3315 // be satisfied, so don't waste any cycles checking it. 3316 if (flag == NULL || (!final_spin && flag->done_check())) { 3317 KA_TRACE( 3318 15, 3319 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 3320 gtid)); 3321 return TRUE; 3322 } 3323 if (thread->th.th_task_team == NULL) { 3324 break; 3325 } 3326 KMP_YIELD(__kmp_library == library_throughput); // Yield before next task 3327 // If execution of a stolen task results in more tasks being placed on our 3328 // run queue, reset use_own_tasks 3329 if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) { 3330 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned " 3331 "other tasks, restart\n", 3332 gtid)); 3333 use_own_tasks = 1; 3334 new_victim = 0; 3335 } 3336 } 3337 3338 // The task source has been exhausted. If in final spin loop of barrier, 3339 // check if termination condition is satisfied. The work queue may be empty 3340 // but there might be proxy tasks still executing. 3341 if (final_spin && 3342 KMP_ATOMIC_LD_ACQ(¤t_task->td_incomplete_child_tasks) == 0) { 3343 // First, decrement the #unfinished threads, if that has not already been 3344 // done. This decrement might be to the spin location, and result in the 3345 // termination condition being satisfied. 3346 if (!*thread_finished) { 3347 #if KMP_DEBUG 3348 kmp_int32 count = -1 + 3349 #endif 3350 KMP_ATOMIC_DEC(unfinished_threads); 3351 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec " 3352 "unfinished_threads to %d task_team=%p\n", 3353 gtid, count, task_team)); 3354 *thread_finished = TRUE; 3355 } 3356 3357 // It is now unsafe to reference thread->th.th_team !!! 3358 // Decrementing task_team->tt.tt_unfinished_threads can allow the primary 3359 // thread to pass through the barrier, where it might reset each thread's 3360 // th.th_team field for the next parallel region. If we can steal more 3361 // work, we know that this has not happened yet. 3362 if (flag != NULL && flag->done_check()) { 3363 KA_TRACE( 3364 15, 3365 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 3366 gtid)); 3367 return TRUE; 3368 } 3369 } 3370 3371 // If this thread's task team is NULL, primary thread has recognized that 3372 // there are no more tasks; bail out 3373 if (thread->th.th_task_team == NULL) { 3374 KA_TRACE(15, 3375 ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid)); 3376 return FALSE; 3377 } 3378 3379 // Check the flag again to see if it has already done in case to be trapped 3380 // into infinite loop when a if0 task depends on a hidden helper task 3381 // outside any parallel region. Detached tasks are not impacted in this case 3382 // because the only thread executing this function has to execute the proxy 3383 // task so it is in another code path that has the same check. 3384 if (flag == NULL || (!final_spin && flag->done_check())) { 3385 KA_TRACE(15, 3386 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 3387 gtid)); 3388 return TRUE; 3389 } 3390 3391 // We could be getting tasks from target constructs; if this is the only 3392 // thread, keep trying to execute tasks from own queue 3393 if (nthreads == 1 && 3394 KMP_ATOMIC_LD_ACQ(¤t_task->td_incomplete_child_tasks)) 3395 use_own_tasks = 1; 3396 else { 3397 KA_TRACE(15, 3398 ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid)); 3399 return FALSE; 3400 } 3401 } 3402 } 3403 3404 template <bool C, bool S> 3405 int __kmp_execute_tasks_32( 3406 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32<C, S> *flag, int final_spin, 3407 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3408 kmp_int32 is_constrained) { 3409 return __kmp_execute_tasks_template( 3410 thread, gtid, flag, final_spin, 3411 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3412 } 3413 3414 template <bool C, bool S> 3415 int __kmp_execute_tasks_64( 3416 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64<C, S> *flag, int final_spin, 3417 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3418 kmp_int32 is_constrained) { 3419 return __kmp_execute_tasks_template( 3420 thread, gtid, flag, final_spin, 3421 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3422 } 3423 3424 template <bool C, bool S> 3425 int __kmp_atomic_execute_tasks_64( 3426 kmp_info_t *thread, kmp_int32 gtid, kmp_atomic_flag_64<C, S> *flag, 3427 int final_spin, int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3428 kmp_int32 is_constrained) { 3429 return __kmp_execute_tasks_template( 3430 thread, gtid, flag, final_spin, 3431 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3432 } 3433 3434 int __kmp_execute_tasks_oncore( 3435 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin, 3436 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3437 kmp_int32 is_constrained) { 3438 return __kmp_execute_tasks_template( 3439 thread, gtid, flag, final_spin, 3440 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3441 } 3442 3443 template int 3444 __kmp_execute_tasks_32<false, false>(kmp_info_t *, kmp_int32, 3445 kmp_flag_32<false, false> *, int, 3446 int *USE_ITT_BUILD_ARG(void *), kmp_int32); 3447 3448 template int __kmp_execute_tasks_64<false, true>(kmp_info_t *, kmp_int32, 3449 kmp_flag_64<false, true> *, 3450 int, 3451 int *USE_ITT_BUILD_ARG(void *), 3452 kmp_int32); 3453 3454 template int __kmp_execute_tasks_64<true, false>(kmp_info_t *, kmp_int32, 3455 kmp_flag_64<true, false> *, 3456 int, 3457 int *USE_ITT_BUILD_ARG(void *), 3458 kmp_int32); 3459 3460 template int __kmp_atomic_execute_tasks_64<false, true>( 3461 kmp_info_t *, kmp_int32, kmp_atomic_flag_64<false, true> *, int, 3462 int *USE_ITT_BUILD_ARG(void *), kmp_int32); 3463 3464 template int __kmp_atomic_execute_tasks_64<true, false>( 3465 kmp_info_t *, kmp_int32, kmp_atomic_flag_64<true, false> *, int, 3466 int *USE_ITT_BUILD_ARG(void *), kmp_int32); 3467 3468 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the 3469 // next barrier so they can assist in executing enqueued tasks. 3470 // First thread in allocates the task team atomically. 3471 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 3472 kmp_info_t *this_thr) { 3473 kmp_thread_data_t *threads_data; 3474 int nthreads, i, is_init_thread; 3475 3476 KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n", 3477 __kmp_gtid_from_thread(this_thr))); 3478 3479 KMP_DEBUG_ASSERT(task_team != NULL); 3480 KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL); 3481 3482 nthreads = task_team->tt.tt_nproc; 3483 KMP_DEBUG_ASSERT(nthreads > 0); 3484 KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc); 3485 3486 // Allocate or increase the size of threads_data if necessary 3487 is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team); 3488 3489 if (!is_init_thread) { 3490 // Some other thread already set up the array. 3491 KA_TRACE( 3492 20, 3493 ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n", 3494 __kmp_gtid_from_thread(this_thr))); 3495 return; 3496 } 3497 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 3498 KMP_DEBUG_ASSERT(threads_data != NULL); 3499 3500 if (__kmp_tasking_mode == tskm_task_teams && 3501 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) { 3502 // Release any threads sleeping at the barrier, so that they can steal 3503 // tasks and execute them. In extra barrier mode, tasks do not sleep 3504 // at the separate tasking barrier, so this isn't a problem. 3505 for (i = 0; i < nthreads; i++) { 3506 void *sleep_loc; 3507 kmp_info_t *thread = threads_data[i].td.td_thr; 3508 3509 if (i == this_thr->th.th_info.ds.ds_tid) { 3510 continue; 3511 } 3512 // Since we haven't locked the thread's suspend mutex lock at this 3513 // point, there is a small window where a thread might be putting 3514 // itself to sleep, but hasn't set the th_sleep_loc field yet. 3515 // To work around this, __kmp_execute_tasks_template() periodically checks 3516 // see if other threads are sleeping (using the same random mechanism that 3517 // is used for task stealing) and awakens them if they are. 3518 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3519 NULL) { 3520 KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n", 3521 __kmp_gtid_from_thread(this_thr), 3522 __kmp_gtid_from_thread(thread))); 3523 __kmp_null_resume_wrapper(thread); 3524 } else { 3525 KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n", 3526 __kmp_gtid_from_thread(this_thr), 3527 __kmp_gtid_from_thread(thread))); 3528 } 3529 } 3530 } 3531 3532 KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n", 3533 __kmp_gtid_from_thread(this_thr))); 3534 } 3535 3536 /* // TODO: Check the comment consistency 3537 * Utility routines for "task teams". A task team (kmp_task_t) is kind of 3538 * like a shadow of the kmp_team_t data struct, with a different lifetime. 3539 * After a child * thread checks into a barrier and calls __kmp_release() from 3540 * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no 3541 * longer assume that the kmp_team_t structure is intact (at any moment, the 3542 * primary thread may exit the barrier code and free the team data structure, 3543 * and return the threads to the thread pool). 3544 * 3545 * This does not work with the tasking code, as the thread is still 3546 * expected to participate in the execution of any tasks that may have been 3547 * spawned my a member of the team, and the thread still needs access to all 3548 * to each thread in the team, so that it can steal work from it. 3549 * 3550 * Enter the existence of the kmp_task_team_t struct. It employs a reference 3551 * counting mechanism, and is allocated by the primary thread before calling 3552 * __kmp_<barrier_kind>_release, and then is release by the last thread to 3553 * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes 3554 * of the kmp_task_team_t structs for consecutive barriers can overlap 3555 * (and will, unless the primary thread is the last thread to exit the barrier 3556 * release phase, which is not typical). The existence of such a struct is 3557 * useful outside the context of tasking. 3558 * 3559 * We currently use the existence of the threads array as an indicator that 3560 * tasks were spawned since the last barrier. If the structure is to be 3561 * useful outside the context of tasking, then this will have to change, but 3562 * not setting the field minimizes the performance impact of tasking on 3563 * barriers, when no explicit tasks were spawned (pushed, actually). 3564 */ 3565 3566 static kmp_task_team_t *__kmp_free_task_teams = 3567 NULL; // Free list for task_team data structures 3568 // Lock for task team data structures 3569 kmp_bootstrap_lock_t __kmp_task_team_lock = 3570 KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock); 3571 3572 // __kmp_alloc_task_deque: 3573 // Allocates a task deque for a particular thread, and initialize the necessary 3574 // data structures relating to the deque. This only happens once per thread 3575 // per task team since task teams are recycled. No lock is needed during 3576 // allocation since each thread allocates its own deque. 3577 static void __kmp_alloc_task_deque(kmp_info_t *thread, 3578 kmp_thread_data_t *thread_data) { 3579 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock); 3580 KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL); 3581 3582 // Initialize last stolen task field to "none" 3583 thread_data->td.td_deque_last_stolen = -1; 3584 3585 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0); 3586 KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0); 3587 KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0); 3588 3589 KE_TRACE( 3590 10, 3591 ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n", 3592 __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data)); 3593 // Allocate space for task deque, and zero the deque 3594 // Cannot use __kmp_thread_calloc() because threads not around for 3595 // kmp_reap_task_team( ). 3596 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate( 3597 INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *)); 3598 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE; 3599 } 3600 3601 // __kmp_free_task_deque: 3602 // Deallocates a task deque for a particular thread. Happens at library 3603 // deallocation so don't need to reset all thread data fields. 3604 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) { 3605 if (thread_data->td.td_deque != NULL) { 3606 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3607 TCW_4(thread_data->td.td_deque_ntasks, 0); 3608 __kmp_free(thread_data->td.td_deque); 3609 thread_data->td.td_deque = NULL; 3610 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3611 } 3612 3613 #ifdef BUILD_TIED_TASK_STACK 3614 // GEH: Figure out what to do here for td_susp_tied_tasks 3615 if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) { 3616 __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data); 3617 } 3618 #endif // BUILD_TIED_TASK_STACK 3619 } 3620 3621 // __kmp_realloc_task_threads_data: 3622 // Allocates a threads_data array for a task team, either by allocating an 3623 // initial array or enlarging an existing array. Only the first thread to get 3624 // the lock allocs or enlarges the array and re-initializes the array elements. 3625 // That thread returns "TRUE", the rest return "FALSE". 3626 // Assumes that the new array size is given by task_team -> tt.tt_nproc. 3627 // The current size is given by task_team -> tt.tt_max_threads. 3628 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 3629 kmp_task_team_t *task_team) { 3630 kmp_thread_data_t **threads_data_p; 3631 kmp_int32 nthreads, maxthreads; 3632 int is_init_thread = FALSE; 3633 3634 if (TCR_4(task_team->tt.tt_found_tasks)) { 3635 // Already reallocated and initialized. 3636 return FALSE; 3637 } 3638 3639 threads_data_p = &task_team->tt.tt_threads_data; 3640 nthreads = task_team->tt.tt_nproc; 3641 maxthreads = task_team->tt.tt_max_threads; 3642 3643 // All threads must lock when they encounter the first task of the implicit 3644 // task region to make sure threads_data fields are (re)initialized before 3645 // used. 3646 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3647 3648 if (!TCR_4(task_team->tt.tt_found_tasks)) { 3649 // first thread to enable tasking 3650 kmp_team_t *team = thread->th.th_team; 3651 int i; 3652 3653 is_init_thread = TRUE; 3654 if (maxthreads < nthreads) { 3655 3656 if (*threads_data_p != NULL) { 3657 kmp_thread_data_t *old_data = *threads_data_p; 3658 kmp_thread_data_t *new_data = NULL; 3659 3660 KE_TRACE( 3661 10, 3662 ("__kmp_realloc_task_threads_data: T#%d reallocating " 3663 "threads data for task_team %p, new_size = %d, old_size = %d\n", 3664 __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads)); 3665 // Reallocate threads_data to have more elements than current array 3666 // Cannot use __kmp_thread_realloc() because threads not around for 3667 // kmp_reap_task_team( ). Note all new array entries are initialized 3668 // to zero by __kmp_allocate(). 3669 new_data = (kmp_thread_data_t *)__kmp_allocate( 3670 nthreads * sizeof(kmp_thread_data_t)); 3671 // copy old data to new data 3672 KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t), 3673 (void *)old_data, maxthreads * sizeof(kmp_thread_data_t)); 3674 3675 #ifdef BUILD_TIED_TASK_STACK 3676 // GEH: Figure out if this is the right thing to do 3677 for (i = maxthreads; i < nthreads; i++) { 3678 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3679 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3680 } 3681 #endif // BUILD_TIED_TASK_STACK 3682 // Install the new data and free the old data 3683 (*threads_data_p) = new_data; 3684 __kmp_free(old_data); 3685 } else { 3686 KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating " 3687 "threads data for task_team %p, size = %d\n", 3688 __kmp_gtid_from_thread(thread), task_team, nthreads)); 3689 // Make the initial allocate for threads_data array, and zero entries 3690 // Cannot use __kmp_thread_calloc() because threads not around for 3691 // kmp_reap_task_team( ). 3692 *threads_data_p = (kmp_thread_data_t *)__kmp_allocate( 3693 nthreads * sizeof(kmp_thread_data_t)); 3694 #ifdef BUILD_TIED_TASK_STACK 3695 // GEH: Figure out if this is the right thing to do 3696 for (i = 0; i < nthreads; i++) { 3697 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3698 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3699 } 3700 #endif // BUILD_TIED_TASK_STACK 3701 } 3702 task_team->tt.tt_max_threads = nthreads; 3703 } else { 3704 // If array has (more than) enough elements, go ahead and use it 3705 KMP_DEBUG_ASSERT(*threads_data_p != NULL); 3706 } 3707 3708 // initialize threads_data pointers back to thread_info structures 3709 for (i = 0; i < nthreads; i++) { 3710 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3711 thread_data->td.td_thr = team->t.t_threads[i]; 3712 3713 if (thread_data->td.td_deque_last_stolen >= nthreads) { 3714 // The last stolen field survives across teams / barrier, and the number 3715 // of threads may have changed. It's possible (likely?) that a new 3716 // parallel region will exhibit the same behavior as previous region. 3717 thread_data->td.td_deque_last_stolen = -1; 3718 } 3719 } 3720 3721 KMP_MB(); 3722 TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE); 3723 } 3724 3725 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3726 return is_init_thread; 3727 } 3728 3729 // __kmp_free_task_threads_data: 3730 // Deallocates a threads_data array for a task team, including any attached 3731 // tasking deques. Only occurs at library shutdown. 3732 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) { 3733 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3734 if (task_team->tt.tt_threads_data != NULL) { 3735 int i; 3736 for (i = 0; i < task_team->tt.tt_max_threads; i++) { 3737 __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]); 3738 } 3739 __kmp_free(task_team->tt.tt_threads_data); 3740 task_team->tt.tt_threads_data = NULL; 3741 } 3742 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3743 } 3744 3745 // __kmp_free_task_pri_list: 3746 // Deallocates tasking deques used for priority tasks. 3747 // Only occurs at library shutdown. 3748 static void __kmp_free_task_pri_list(kmp_task_team_t *task_team) { 3749 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock); 3750 if (task_team->tt.tt_task_pri_list != NULL) { 3751 kmp_task_pri_t *list = task_team->tt.tt_task_pri_list; 3752 while (list != NULL) { 3753 kmp_task_pri_t *next = list->next; 3754 __kmp_free_task_deque(&list->td); 3755 __kmp_free(list); 3756 list = next; 3757 } 3758 task_team->tt.tt_task_pri_list = NULL; 3759 } 3760 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock); 3761 } 3762 3763 // __kmp_allocate_task_team: 3764 // Allocates a task team associated with a specific team, taking it from 3765 // the global task team free list if possible. Also initializes data 3766 // structures. 3767 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread, 3768 kmp_team_t *team) { 3769 kmp_task_team_t *task_team = NULL; 3770 int nthreads; 3771 3772 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n", 3773 (thread ? __kmp_gtid_from_thread(thread) : -1), team)); 3774 3775 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3776 // Take a task team from the task team pool 3777 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3778 if (__kmp_free_task_teams != NULL) { 3779 task_team = __kmp_free_task_teams; 3780 TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next); 3781 task_team->tt.tt_next = NULL; 3782 } 3783 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3784 } 3785 3786 if (task_team == NULL) { 3787 KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating " 3788 "task team for team %p\n", 3789 __kmp_gtid_from_thread(thread), team)); 3790 // Allocate a new task team if one is not available. Cannot use 3791 // __kmp_thread_malloc because threads not around for kmp_reap_task_team. 3792 task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t)); 3793 __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock); 3794 __kmp_init_bootstrap_lock(&task_team->tt.tt_task_pri_lock); 3795 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG 3796 // suppress race conditions detection on synchronization flags in debug mode 3797 // this helps to analyze library internals eliminating false positives 3798 __itt_suppress_mark_range( 3799 __itt_suppress_range, __itt_suppress_threading_errors, 3800 &task_team->tt.tt_found_tasks, sizeof(task_team->tt.tt_found_tasks)); 3801 __itt_suppress_mark_range(__itt_suppress_range, 3802 __itt_suppress_threading_errors, 3803 CCAST(kmp_uint32 *, &task_team->tt.tt_active), 3804 sizeof(task_team->tt.tt_active)); 3805 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */ 3806 // Note: __kmp_allocate zeroes returned memory, othewise we would need: 3807 // task_team->tt.tt_threads_data = NULL; 3808 // task_team->tt.tt_max_threads = 0; 3809 // task_team->tt.tt_next = NULL; 3810 } 3811 3812 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3813 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3814 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); 3815 task_team->tt.tt_nproc = nthreads = team->t.t_nproc; 3816 3817 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads); 3818 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); 3819 TCW_4(task_team->tt.tt_active, TRUE); 3820 3821 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p " 3822 "unfinished_threads init'd to %d\n", 3823 (thread ? __kmp_gtid_from_thread(thread) : -1), task_team, 3824 KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads))); 3825 return task_team; 3826 } 3827 3828 // __kmp_free_task_team: 3829 // Frees the task team associated with a specific thread, and adds it 3830 // to the global task team free list. 3831 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) { 3832 KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n", 3833 thread ? __kmp_gtid_from_thread(thread) : -1, task_team)); 3834 3835 // Put task team back on free list 3836 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3837 3838 KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL); 3839 task_team->tt.tt_next = __kmp_free_task_teams; 3840 TCW_PTR(__kmp_free_task_teams, task_team); 3841 3842 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3843 } 3844 3845 // __kmp_reap_task_teams: 3846 // Free all the task teams on the task team free list. 3847 // Should only be done during library shutdown. 3848 // Cannot do anything that needs a thread structure or gtid since they are 3849 // already gone. 3850 void __kmp_reap_task_teams(void) { 3851 kmp_task_team_t *task_team; 3852 3853 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3854 // Free all task_teams on the free list 3855 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3856 while ((task_team = __kmp_free_task_teams) != NULL) { 3857 __kmp_free_task_teams = task_team->tt.tt_next; 3858 task_team->tt.tt_next = NULL; 3859 3860 // Free threads_data if necessary 3861 if (task_team->tt.tt_threads_data != NULL) { 3862 __kmp_free_task_threads_data(task_team); 3863 } 3864 if (task_team->tt.tt_task_pri_list != NULL) { 3865 __kmp_free_task_pri_list(task_team); 3866 } 3867 __kmp_free(task_team); 3868 } 3869 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3870 } 3871 } 3872 3873 // __kmp_wait_to_unref_task_teams: 3874 // Some threads could still be in the fork barrier release code, possibly 3875 // trying to steal tasks. Wait for each thread to unreference its task team. 3876 void __kmp_wait_to_unref_task_teams(void) { 3877 kmp_info_t *thread; 3878 kmp_uint32 spins; 3879 kmp_uint64 time; 3880 int done; 3881 3882 KMP_INIT_YIELD(spins); 3883 KMP_INIT_BACKOFF(time); 3884 3885 for (;;) { 3886 done = TRUE; 3887 3888 // TODO: GEH - this may be is wrong because some sync would be necessary 3889 // in case threads are added to the pool during the traversal. Need to 3890 // verify that lock for thread pool is held when calling this routine. 3891 for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL; 3892 thread = thread->th.th_next_pool) { 3893 #if KMP_OS_WINDOWS 3894 DWORD exit_val; 3895 #endif 3896 if (TCR_PTR(thread->th.th_task_team) == NULL) { 3897 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n", 3898 __kmp_gtid_from_thread(thread))); 3899 continue; 3900 } 3901 #if KMP_OS_WINDOWS 3902 // TODO: GEH - add this check for Linux* OS / OS X* as well? 3903 if (!__kmp_is_thread_alive(thread, &exit_val)) { 3904 thread->th.th_task_team = NULL; 3905 continue; 3906 } 3907 #endif 3908 3909 done = FALSE; // Because th_task_team pointer is not NULL for this thread 3910 3911 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to " 3912 "unreference task_team\n", 3913 __kmp_gtid_from_thread(thread))); 3914 3915 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 3916 void *sleep_loc; 3917 // If the thread is sleeping, awaken it. 3918 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3919 NULL) { 3920 KA_TRACE( 3921 10, 3922 ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n", 3923 __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread))); 3924 __kmp_null_resume_wrapper(thread); 3925 } 3926 } 3927 } 3928 if (done) { 3929 break; 3930 } 3931 3932 // If oversubscribed or have waited a bit, yield. 3933 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time); 3934 } 3935 } 3936 3937 // __kmp_task_team_setup: Create a task_team for the current team, but use 3938 // an already created, unused one if it already exists. 3939 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) { 3940 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3941 3942 // If this task_team hasn't been created yet, allocate it. It will be used in 3943 // the region after the next. 3944 // If it exists, it is the current task team and shouldn't be touched yet as 3945 // it may still be in use. 3946 if (team->t.t_task_team[this_thr->th.th_task_state] == NULL && 3947 (always || team->t.t_nproc > 1)) { 3948 team->t.t_task_team[this_thr->th.th_task_state] = 3949 __kmp_allocate_task_team(this_thr, team); 3950 KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created new task_team %p" 3951 " for team %d at parity=%d\n", 3952 __kmp_gtid_from_thread(this_thr), 3953 team->t.t_task_team[this_thr->th.th_task_state], team->t.t_id, 3954 this_thr->th.th_task_state)); 3955 } 3956 3957 // After threads exit the release, they will call sync, and then point to this 3958 // other task_team; make sure it is allocated and properly initialized. As 3959 // threads spin in the barrier release phase, they will continue to use the 3960 // previous task_team struct(above), until they receive the signal to stop 3961 // checking for tasks (they can't safely reference the kmp_team_t struct, 3962 // which could be reallocated by the primary thread). No task teams are formed 3963 // for serialized teams. 3964 if (team->t.t_nproc > 1) { 3965 int other_team = 1 - this_thr->th.th_task_state; 3966 KMP_DEBUG_ASSERT(other_team >= 0 && other_team < 2); 3967 if (team->t.t_task_team[other_team] == NULL) { // setup other team as well 3968 team->t.t_task_team[other_team] = 3969 __kmp_allocate_task_team(this_thr, team); 3970 KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created second new " 3971 "task_team %p for team %d at parity=%d\n", 3972 __kmp_gtid_from_thread(this_thr), 3973 team->t.t_task_team[other_team], team->t.t_id, other_team)); 3974 } else { // Leave the old task team struct in place for the upcoming region; 3975 // adjust as needed 3976 kmp_task_team_t *task_team = team->t.t_task_team[other_team]; 3977 if (!task_team->tt.tt_active || 3978 team->t.t_nproc != task_team->tt.tt_nproc) { 3979 TCW_4(task_team->tt.tt_nproc, team->t.t_nproc); 3980 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3981 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3982 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); 3983 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, 3984 team->t.t_nproc); 3985 TCW_4(task_team->tt.tt_active, TRUE); 3986 } 3987 // if team size has changed, the first thread to enable tasking will 3988 // realloc threads_data if necessary 3989 KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d reset next task_team " 3990 "%p for team %d at parity=%d\n", 3991 __kmp_gtid_from_thread(this_thr), 3992 team->t.t_task_team[other_team], team->t.t_id, other_team)); 3993 } 3994 } 3995 3996 // For regular thread, task enabling should be called when the task is going 3997 // to be pushed to a dequeue. However, for the hidden helper thread, we need 3998 // it ahead of time so that some operations can be performed without race 3999 // condition. 4000 if (this_thr == __kmp_hidden_helper_main_thread) { 4001 for (int i = 0; i < 2; ++i) { 4002 kmp_task_team_t *task_team = team->t.t_task_team[i]; 4003 if (KMP_TASKING_ENABLED(task_team)) { 4004 continue; 4005 } 4006 __kmp_enable_tasking(task_team, this_thr); 4007 for (int j = 0; j < task_team->tt.tt_nproc; ++j) { 4008 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[j]; 4009 if (thread_data->td.td_deque == NULL) { 4010 __kmp_alloc_task_deque(__kmp_hidden_helper_threads[j], thread_data); 4011 } 4012 } 4013 } 4014 } 4015 } 4016 4017 // __kmp_task_team_sync: Propagation of task team data from team to threads 4018 // which happens just after the release phase of a team barrier. This may be 4019 // called by any thread, but only for teams with # threads > 1. 4020 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) { 4021 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 4022 4023 // Toggle the th_task_state field, to switch which task_team this thread 4024 // refers to 4025 this_thr->th.th_task_state = (kmp_uint8)(1 - this_thr->th.th_task_state); 4026 4027 // It is now safe to propagate the task team pointer from the team struct to 4028 // the current thread. 4029 TCW_PTR(this_thr->th.th_task_team, 4030 team->t.t_task_team[this_thr->th.th_task_state]); 4031 KA_TRACE(20, 4032 ("__kmp_task_team_sync: Thread T#%d task team switched to task_team " 4033 "%p from Team #%d (parity=%d)\n", 4034 __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team, 4035 team->t.t_id, this_thr->th.th_task_state)); 4036 } 4037 4038 // __kmp_task_team_wait: Primary thread waits for outstanding tasks after the 4039 // barrier gather phase. Only called by primary thread if #threads in team > 1 4040 // or if proxy tasks were created. 4041 // 4042 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off 4043 // by passing in 0 optionally as the last argument. When wait is zero, primary 4044 // thread does not wait for unfinished_threads to reach 0. 4045 void __kmp_task_team_wait( 4046 kmp_info_t *this_thr, 4047 kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) { 4048 kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state]; 4049 4050 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 4051 KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team); 4052 4053 if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) { 4054 if (wait) { 4055 KA_TRACE(20, ("__kmp_task_team_wait: Primary T#%d waiting for all tasks " 4056 "(for unfinished_threads to reach 0) on task_team = %p\n", 4057 __kmp_gtid_from_thread(this_thr), task_team)); 4058 // Worker threads may have dropped through to release phase, but could 4059 // still be executing tasks. Wait here for tasks to complete. To avoid 4060 // memory contention, only primary thread checks termination condition. 4061 kmp_flag_32<false, false> flag( 4062 RCAST(std::atomic<kmp_uint32> *, 4063 &task_team->tt.tt_unfinished_threads), 4064 0U); 4065 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 4066 } 4067 // Deactivate the old task team, so that the worker threads will stop 4068 // referencing it while spinning. 4069 KA_TRACE( 4070 20, 4071 ("__kmp_task_team_wait: Primary T#%d deactivating task_team %p: " 4072 "setting active to false, setting local and team's pointer to NULL\n", 4073 __kmp_gtid_from_thread(this_thr), task_team)); 4074 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 || 4075 task_team->tt.tt_found_proxy_tasks == TRUE || 4076 task_team->tt.tt_hidden_helper_task_encountered == TRUE); 4077 TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE); 4078 TCW_SYNC_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); 4079 KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0); 4080 TCW_SYNC_4(task_team->tt.tt_active, FALSE); 4081 KMP_MB(); 4082 4083 TCW_PTR(this_thr->th.th_task_team, NULL); 4084 } 4085 } 4086 4087 // __kmp_tasking_barrier: 4088 // This routine is called only when __kmp_tasking_mode == tskm_extra_barrier. 4089 // Internal function to execute all tasks prior to a regular barrier or a join 4090 // barrier. It is a full barrier itself, which unfortunately turns regular 4091 // barriers into double barriers and join barriers into 1 1/2 barriers. 4092 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) { 4093 std::atomic<kmp_uint32> *spin = RCAST( 4094 std::atomic<kmp_uint32> *, 4095 &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads); 4096 int flag = FALSE; 4097 KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier); 4098 4099 #if USE_ITT_BUILD 4100 KMP_FSYNC_SPIN_INIT(spin, NULL); 4101 #endif /* USE_ITT_BUILD */ 4102 kmp_flag_32<false, false> spin_flag(spin, 0U); 4103 while (!spin_flag.execute_tasks(thread, gtid, TRUE, 4104 &flag USE_ITT_BUILD_ARG(NULL), 0)) { 4105 #if USE_ITT_BUILD 4106 // TODO: What about itt_sync_obj?? 4107 KMP_FSYNC_SPIN_PREPARE(RCAST(void *, spin)); 4108 #endif /* USE_ITT_BUILD */ 4109 4110 if (TCR_4(__kmp_global.g.g_done)) { 4111 if (__kmp_global.g.g_abort) 4112 __kmp_abort_thread(); 4113 break; 4114 } 4115 KMP_YIELD(TRUE); 4116 } 4117 #if USE_ITT_BUILD 4118 KMP_FSYNC_SPIN_ACQUIRED(RCAST(void *, spin)); 4119 #endif /* USE_ITT_BUILD */ 4120 } 4121 4122 // __kmp_give_task puts a task into a given thread queue if: 4123 // - the queue for that thread was created 4124 // - there's space in that queue 4125 // Because of this, __kmp_push_task needs to check if there's space after 4126 // getting the lock 4127 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task, 4128 kmp_int32 pass) { 4129 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4130 kmp_task_team_t *task_team = taskdata->td_task_team; 4131 4132 KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n", 4133 taskdata, tid)); 4134 4135 // If task_team is NULL something went really bad... 4136 KMP_DEBUG_ASSERT(task_team != NULL); 4137 4138 bool result = false; 4139 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 4140 4141 if (thread_data->td.td_deque == NULL) { 4142 // There's no queue in this thread, go find another one 4143 // We're guaranteed that at least one thread has a queue 4144 KA_TRACE(30, 4145 ("__kmp_give_task: thread %d has no queue while giving task %p.\n", 4146 tid, taskdata)); 4147 return result; 4148 } 4149 4150 if (TCR_4(thread_data->td.td_deque_ntasks) >= 4151 TASK_DEQUE_SIZE(thread_data->td)) { 4152 KA_TRACE( 4153 30, 4154 ("__kmp_give_task: queue is full while giving task %p to thread %d.\n", 4155 taskdata, tid)); 4156 4157 // if this deque is bigger than the pass ratio give a chance to another 4158 // thread 4159 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 4160 return result; 4161 4162 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 4163 if (TCR_4(thread_data->td.td_deque_ntasks) >= 4164 TASK_DEQUE_SIZE(thread_data->td)) { 4165 // expand deque to push the task which is not allowed to execute 4166 __kmp_realloc_task_deque(thread, thread_data); 4167 } 4168 4169 } else { 4170 4171 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 4172 4173 if (TCR_4(thread_data->td.td_deque_ntasks) >= 4174 TASK_DEQUE_SIZE(thread_data->td)) { 4175 KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to " 4176 "thread %d.\n", 4177 taskdata, tid)); 4178 4179 // if this deque is bigger than the pass ratio give a chance to another 4180 // thread 4181 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 4182 goto release_and_exit; 4183 4184 __kmp_realloc_task_deque(thread, thread_data); 4185 } 4186 } 4187 4188 // lock is held here, and there is space in the deque 4189 4190 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; 4191 // Wrap index. 4192 thread_data->td.td_deque_tail = 4193 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 4194 TCW_4(thread_data->td.td_deque_ntasks, 4195 TCR_4(thread_data->td.td_deque_ntasks) + 1); 4196 4197 result = true; 4198 KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n", 4199 taskdata, tid)); 4200 4201 release_and_exit: 4202 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 4203 4204 return result; 4205 } 4206 4207 #define PROXY_TASK_FLAG 0x40000000 4208 /* The finish of the proxy tasks is divided in two pieces: 4209 - the top half is the one that can be done from a thread outside the team 4210 - the bottom half must be run from a thread within the team 4211 4212 In order to run the bottom half the task gets queued back into one of the 4213 threads of the team. Once the td_incomplete_child_task counter of the parent 4214 is decremented the threads can leave the barriers. So, the bottom half needs 4215 to be queued before the counter is decremented. The top half is therefore 4216 divided in two parts: 4217 - things that can be run before queuing the bottom half 4218 - things that must be run after queuing the bottom half 4219 4220 This creates a second race as the bottom half can free the task before the 4221 second top half is executed. To avoid this we use the 4222 td_incomplete_child_task of the proxy task to synchronize the top and bottom 4223 half. */ 4224 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 4225 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 4226 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 4227 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 4228 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 4229 4230 taskdata->td_flags.complete = 1; // mark the task as completed 4231 4232 if (taskdata->td_taskgroup) 4233 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 4234 4235 // Create an imaginary children for this task so the bottom half cannot 4236 // release the task before we have completed the second top half 4237 KMP_ATOMIC_OR(&taskdata->td_incomplete_child_tasks, PROXY_TASK_FLAG); 4238 } 4239 4240 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 4241 #if KMP_DEBUG 4242 kmp_int32 children = 0; 4243 // Predecrement simulated by "- 1" calculation 4244 children = -1 + 4245 #endif 4246 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks); 4247 KMP_DEBUG_ASSERT(children >= 0); 4248 4249 // Remove the imaginary children 4250 KMP_ATOMIC_AND(&taskdata->td_incomplete_child_tasks, ~PROXY_TASK_FLAG); 4251 } 4252 4253 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) { 4254 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 4255 kmp_info_t *thread = __kmp_threads[gtid]; 4256 4257 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 4258 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 4259 1); // top half must run before bottom half 4260 4261 // We need to wait to make sure the top half is finished 4262 // Spinning here should be ok as this should happen quickly 4263 while ((KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) & 4264 PROXY_TASK_FLAG) > 0) 4265 ; 4266 4267 __kmp_release_deps(gtid, taskdata); 4268 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 4269 } 4270 4271 /*! 4272 @ingroup TASKING 4273 @param gtid Global Thread ID of encountering thread 4274 @param ptask Task which execution is completed 4275 4276 Execute the completion of a proxy task from a thread of that is part of the 4277 team. Run first and bottom halves directly. 4278 */ 4279 void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) { 4280 KMP_DEBUG_ASSERT(ptask != NULL); 4281 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 4282 KA_TRACE( 4283 10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n", 4284 gtid, taskdata)); 4285 __kmp_assert_valid_gtid(gtid); 4286 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 4287 4288 __kmp_first_top_half_finish_proxy(taskdata); 4289 __kmp_second_top_half_finish_proxy(taskdata); 4290 __kmp_bottom_half_finish_proxy(gtid, ptask); 4291 4292 KA_TRACE(10, 4293 ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n", 4294 gtid, taskdata)); 4295 } 4296 4297 void __kmpc_give_task(kmp_task_t *ptask, kmp_int32 start = 0) { 4298 KMP_DEBUG_ASSERT(ptask != NULL); 4299 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 4300 4301 // Enqueue task to complete bottom half completion from a thread within the 4302 // corresponding team 4303 kmp_team_t *team = taskdata->td_team; 4304 kmp_int32 nthreads = team->t.t_nproc; 4305 kmp_info_t *thread; 4306 4307 // This should be similar to start_k = __kmp_get_random( thread ) % nthreads 4308 // but we cannot use __kmp_get_random here 4309 kmp_int32 start_k = start % nthreads; 4310 kmp_int32 pass = 1; 4311 kmp_int32 k = start_k; 4312 4313 do { 4314 // For now we're just linearly trying to find a thread 4315 thread = team->t.t_threads[k]; 4316 k = (k + 1) % nthreads; 4317 4318 // we did a full pass through all the threads 4319 if (k == start_k) 4320 pass = pass << 1; 4321 4322 } while (!__kmp_give_task(thread, k, ptask, pass)); 4323 4324 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME && __kmp_wpolicy_passive) { 4325 // awake at least one thread to execute given task 4326 for (int i = 0; i < nthreads; ++i) { 4327 thread = team->t.t_threads[i]; 4328 if (thread->th.th_sleep_loc != NULL) { 4329 __kmp_null_resume_wrapper(thread); 4330 break; 4331 } 4332 } 4333 } 4334 } 4335 4336 /*! 4337 @ingroup TASKING 4338 @param ptask Task which execution is completed 4339 4340 Execute the completion of a proxy task from a thread that could not belong to 4341 the team. 4342 */ 4343 void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) { 4344 KMP_DEBUG_ASSERT(ptask != NULL); 4345 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 4346 4347 KA_TRACE( 4348 10, 4349 ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n", 4350 taskdata)); 4351 4352 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 4353 4354 __kmp_first_top_half_finish_proxy(taskdata); 4355 4356 __kmpc_give_task(ptask); 4357 4358 __kmp_second_top_half_finish_proxy(taskdata); 4359 4360 KA_TRACE( 4361 10, 4362 ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n", 4363 taskdata)); 4364 } 4365 4366 kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref, int gtid, 4367 kmp_task_t *task) { 4368 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task); 4369 if (td->td_allow_completion_event.type == KMP_EVENT_UNINITIALIZED) { 4370 td->td_allow_completion_event.type = KMP_EVENT_ALLOW_COMPLETION; 4371 td->td_allow_completion_event.ed.task = task; 4372 __kmp_init_tas_lock(&td->td_allow_completion_event.lock); 4373 } 4374 return &td->td_allow_completion_event; 4375 } 4376 4377 void __kmp_fulfill_event(kmp_event_t *event) { 4378 if (event->type == KMP_EVENT_ALLOW_COMPLETION) { 4379 kmp_task_t *ptask = event->ed.task; 4380 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 4381 bool detached = false; 4382 int gtid = __kmp_get_gtid(); 4383 4384 // The associated task might have completed or could be completing at this 4385 // point. 4386 // We need to take the lock to avoid races 4387 __kmp_acquire_tas_lock(&event->lock, gtid); 4388 if (taskdata->td_flags.proxy == TASK_PROXY) { 4389 detached = true; 4390 } else { 4391 #if OMPT_SUPPORT 4392 // The OMPT event must occur under mutual exclusion, 4393 // otherwise the tool might access ptask after free 4394 if (UNLIKELY(ompt_enabled.enabled)) 4395 __ompt_task_finish(ptask, NULL, ompt_task_early_fulfill); 4396 #endif 4397 } 4398 event->type = KMP_EVENT_UNINITIALIZED; 4399 __kmp_release_tas_lock(&event->lock, gtid); 4400 4401 if (detached) { 4402 #if OMPT_SUPPORT 4403 // We free ptask afterwards and know the task is finished, 4404 // so locking is not necessary 4405 if (UNLIKELY(ompt_enabled.enabled)) 4406 __ompt_task_finish(ptask, NULL, ompt_task_late_fulfill); 4407 #endif 4408 // If the task detached complete the proxy task 4409 if (gtid >= 0) { 4410 kmp_team_t *team = taskdata->td_team; 4411 kmp_info_t *thread = __kmp_get_thread(); 4412 if (thread->th.th_team == team) { 4413 __kmpc_proxy_task_completed(gtid, ptask); 4414 return; 4415 } 4416 } 4417 4418 // fallback 4419 __kmpc_proxy_task_completed_ooo(ptask); 4420 } 4421 } 4422 } 4423 4424 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task 4425 // for taskloop 4426 // 4427 // thread: allocating thread 4428 // task_src: pointer to source task to be duplicated 4429 // returns: a pointer to the allocated kmp_task_t structure (task). 4430 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) { 4431 kmp_task_t *task; 4432 kmp_taskdata_t *taskdata; 4433 kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src); 4434 kmp_taskdata_t *parent_task = taskdata_src->td_parent; // same parent task 4435 size_t shareds_offset; 4436 size_t task_size; 4437 4438 KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread, 4439 task_src)); 4440 KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy == 4441 TASK_FULL); // it should not be proxy task 4442 KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT); 4443 task_size = taskdata_src->td_size_alloc; 4444 4445 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 4446 KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread, 4447 task_size)); 4448 #if USE_FAST_MEMORY 4449 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size); 4450 #else 4451 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size); 4452 #endif /* USE_FAST_MEMORY */ 4453 KMP_MEMCPY(taskdata, taskdata_src, task_size); 4454 4455 task = KMP_TASKDATA_TO_TASK(taskdata); 4456 4457 // Initialize new task (only specific fields not affected by memcpy) 4458 taskdata->td_task_id = KMP_GEN_TASK_ID(); 4459 if (task->shareds != NULL) { // need setup shareds pointer 4460 shareds_offset = (char *)task_src->shareds - (char *)taskdata_src; 4461 task->shareds = &((char *)taskdata)[shareds_offset]; 4462 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 4463 0); 4464 } 4465 taskdata->td_alloc_thread = thread; 4466 taskdata->td_parent = parent_task; 4467 // task inherits the taskgroup from the parent task 4468 taskdata->td_taskgroup = parent_task->td_taskgroup; 4469 // tied task needs to initialize the td_last_tied at creation, 4470 // untied one does this when it is scheduled for execution 4471 if (taskdata->td_flags.tiedness == TASK_TIED) 4472 taskdata->td_last_tied = taskdata; 4473 4474 // Only need to keep track of child task counts if team parallel and tasking 4475 // not serialized 4476 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { 4477 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 4478 if (parent_task->td_taskgroup) 4479 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 4480 // Only need to keep track of allocated child tasks for explicit tasks since 4481 // implicit not deallocated 4482 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) 4483 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 4484 } 4485 4486 KA_TRACE(20, 4487 ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n", 4488 thread, taskdata, taskdata->td_parent)); 4489 #if OMPT_SUPPORT 4490 if (UNLIKELY(ompt_enabled.enabled)) 4491 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid); 4492 #endif 4493 return task; 4494 } 4495 4496 // Routine optionally generated by the compiler for setting the lastprivate flag 4497 // and calling needed constructors for private/firstprivate objects 4498 // (used to form taskloop tasks from pattern task) 4499 // Parameters: dest task, src task, lastprivate flag. 4500 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32); 4501 4502 KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8); 4503 4504 // class to encapsulate manipulating loop bounds in a taskloop task. 4505 // this abstracts away the Intel vs GOMP taskloop interface for setting/getting 4506 // the loop bound variables. 4507 class kmp_taskloop_bounds_t { 4508 kmp_task_t *task; 4509 const kmp_taskdata_t *taskdata; 4510 size_t lower_offset; 4511 size_t upper_offset; 4512 4513 public: 4514 kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub) 4515 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)), 4516 lower_offset((char *)lb - (char *)task), 4517 upper_offset((char *)ub - (char *)task) { 4518 KMP_DEBUG_ASSERT((char *)lb > (char *)_task); 4519 KMP_DEBUG_ASSERT((char *)ub > (char *)_task); 4520 } 4521 kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds) 4522 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)), 4523 lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {} 4524 size_t get_lower_offset() const { return lower_offset; } 4525 size_t get_upper_offset() const { return upper_offset; } 4526 kmp_uint64 get_lb() const { 4527 kmp_int64 retval; 4528 #if defined(KMP_GOMP_COMPAT) 4529 // Intel task just returns the lower bound normally 4530 if (!taskdata->td_flags.native) { 4531 retval = *(kmp_int64 *)((char *)task + lower_offset); 4532 } else { 4533 // GOMP task has to take into account the sizeof(long) 4534 if (taskdata->td_size_loop_bounds == 4) { 4535 kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds); 4536 retval = (kmp_int64)*lb; 4537 } else { 4538 kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds); 4539 retval = (kmp_int64)*lb; 4540 } 4541 } 4542 #else 4543 (void)taskdata; 4544 retval = *(kmp_int64 *)((char *)task + lower_offset); 4545 #endif // defined(KMP_GOMP_COMPAT) 4546 return retval; 4547 } 4548 kmp_uint64 get_ub() const { 4549 kmp_int64 retval; 4550 #if defined(KMP_GOMP_COMPAT) 4551 // Intel task just returns the upper bound normally 4552 if (!taskdata->td_flags.native) { 4553 retval = *(kmp_int64 *)((char *)task + upper_offset); 4554 } else { 4555 // GOMP task has to take into account the sizeof(long) 4556 if (taskdata->td_size_loop_bounds == 4) { 4557 kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1; 4558 retval = (kmp_int64)*ub; 4559 } else { 4560 kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1; 4561 retval = (kmp_int64)*ub; 4562 } 4563 } 4564 #else 4565 retval = *(kmp_int64 *)((char *)task + upper_offset); 4566 #endif // defined(KMP_GOMP_COMPAT) 4567 return retval; 4568 } 4569 void set_lb(kmp_uint64 lb) { 4570 #if defined(KMP_GOMP_COMPAT) 4571 // Intel task just sets the lower bound normally 4572 if (!taskdata->td_flags.native) { 4573 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4574 } else { 4575 // GOMP task has to take into account the sizeof(long) 4576 if (taskdata->td_size_loop_bounds == 4) { 4577 kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds); 4578 *lower = (kmp_uint32)lb; 4579 } else { 4580 kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds); 4581 *lower = (kmp_uint64)lb; 4582 } 4583 } 4584 #else 4585 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4586 #endif // defined(KMP_GOMP_COMPAT) 4587 } 4588 void set_ub(kmp_uint64 ub) { 4589 #if defined(KMP_GOMP_COMPAT) 4590 // Intel task just sets the upper bound normally 4591 if (!taskdata->td_flags.native) { 4592 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4593 } else { 4594 // GOMP task has to take into account the sizeof(long) 4595 if (taskdata->td_size_loop_bounds == 4) { 4596 kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1; 4597 *upper = (kmp_uint32)ub; 4598 } else { 4599 kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1; 4600 *upper = (kmp_uint64)ub; 4601 } 4602 } 4603 #else 4604 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4605 #endif // defined(KMP_GOMP_COMPAT) 4606 } 4607 }; 4608 4609 // __kmp_taskloop_linear: Start tasks of the taskloop linearly 4610 // 4611 // loc Source location information 4612 // gtid Global thread ID 4613 // task Pattern task, exposes the loop iteration range 4614 // lb Pointer to loop lower bound in task structure 4615 // ub Pointer to loop upper bound in task structure 4616 // st Loop stride 4617 // ub_glob Global upper bound (used for lastprivate check) 4618 // num_tasks Number of tasks to execute 4619 // grainsize Number of loop iterations per task 4620 // extras Number of chunks with grainsize+1 iterations 4621 // last_chunk Reduction of grainsize for last task 4622 // tc Iterations count 4623 // task_dup Tasks duplication routine 4624 // codeptr_ra Return address for OMPT events 4625 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, 4626 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4627 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4628 kmp_uint64 grainsize, kmp_uint64 extras, 4629 kmp_int64 last_chunk, kmp_uint64 tc, 4630 #if OMPT_SUPPORT 4631 void *codeptr_ra, 4632 #endif 4633 void *task_dup) { 4634 KMP_COUNT_BLOCK(OMP_TASKLOOP); 4635 KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling); 4636 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4637 // compiler provides global bounds here 4638 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4639 kmp_uint64 lower = task_bounds.get_lb(); 4640 kmp_uint64 upper = task_bounds.get_ub(); 4641 kmp_uint64 i; 4642 kmp_info_t *thread = __kmp_threads[gtid]; 4643 kmp_taskdata_t *current_task = thread->th.th_current_task; 4644 kmp_task_t *next_task; 4645 kmp_int32 lastpriv = 0; 4646 4647 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + 4648 (last_chunk < 0 ? last_chunk : extras)); 4649 KMP_DEBUG_ASSERT(num_tasks > extras); 4650 KMP_DEBUG_ASSERT(num_tasks > 0); 4651 KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, " 4652 "extras %lld, last_chunk %lld, i=%lld,%lld(%d)%lld, dup %p\n", 4653 gtid, num_tasks, grainsize, extras, last_chunk, lower, upper, 4654 ub_glob, st, task_dup)); 4655 4656 // Launch num_tasks tasks, assign grainsize iterations each task 4657 for (i = 0; i < num_tasks; ++i) { 4658 kmp_uint64 chunk_minus_1; 4659 if (extras == 0) { 4660 chunk_minus_1 = grainsize - 1; 4661 } else { 4662 chunk_minus_1 = grainsize; 4663 --extras; // first extras iterations get bigger chunk (grainsize+1) 4664 } 4665 upper = lower + st * chunk_minus_1; 4666 if (upper > *ub) { 4667 upper = *ub; 4668 } 4669 if (i == num_tasks - 1) { 4670 // schedule the last task, set lastprivate flag if needed 4671 if (st == 1) { // most common case 4672 KMP_DEBUG_ASSERT(upper == *ub); 4673 if (upper == ub_glob) 4674 lastpriv = 1; 4675 } else if (st > 0) { // positive loop stride 4676 KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper); 4677 if ((kmp_uint64)st > ub_glob - upper) 4678 lastpriv = 1; 4679 } else { // negative loop stride 4680 KMP_DEBUG_ASSERT(upper + st < *ub); 4681 if (upper - ub_glob < (kmp_uint64)(-st)) 4682 lastpriv = 1; 4683 } 4684 } 4685 next_task = __kmp_task_dup_alloc(thread, task); // allocate new task 4686 kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task); 4687 kmp_taskloop_bounds_t next_task_bounds = 4688 kmp_taskloop_bounds_t(next_task, task_bounds); 4689 4690 // adjust task-specific bounds 4691 next_task_bounds.set_lb(lower); 4692 if (next_taskdata->td_flags.native) { 4693 next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1)); 4694 } else { 4695 next_task_bounds.set_ub(upper); 4696 } 4697 if (ptask_dup != NULL) // set lastprivate flag, construct firstprivates, 4698 // etc. 4699 ptask_dup(next_task, task, lastpriv); 4700 KA_TRACE(40, 4701 ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, " 4702 "upper %lld stride %lld, (offsets %p %p)\n", 4703 gtid, i, next_task, lower, upper, st, 4704 next_task_bounds.get_lower_offset(), 4705 next_task_bounds.get_upper_offset())); 4706 #if OMPT_SUPPORT 4707 __kmp_omp_taskloop_task(NULL, gtid, next_task, 4708 codeptr_ra); // schedule new task 4709 #if OMPT_OPTIONAL 4710 if (ompt_enabled.ompt_callback_dispatch) { 4711 OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk, 4712 lower, upper, st); 4713 } 4714 #endif // OMPT_OPTIONAL 4715 #else 4716 __kmp_omp_task(gtid, next_task, true); // schedule new task 4717 #endif 4718 lower = upper + st; // adjust lower bound for the next iteration 4719 } 4720 // free the pattern task and exit 4721 __kmp_task_start(gtid, task, current_task); // make internal bookkeeping 4722 // do not execute the pattern task, just do internal bookkeeping 4723 __kmp_task_finish<false>(gtid, task, current_task); 4724 } 4725 4726 // Structure to keep taskloop parameters for auxiliary task 4727 // kept in the shareds of the task structure. 4728 typedef struct __taskloop_params { 4729 kmp_task_t *task; 4730 kmp_uint64 *lb; 4731 kmp_uint64 *ub; 4732 void *task_dup; 4733 kmp_int64 st; 4734 kmp_uint64 ub_glob; 4735 kmp_uint64 num_tasks; 4736 kmp_uint64 grainsize; 4737 kmp_uint64 extras; 4738 kmp_int64 last_chunk; 4739 kmp_uint64 tc; 4740 kmp_uint64 num_t_min; 4741 #if OMPT_SUPPORT 4742 void *codeptr_ra; 4743 #endif 4744 } __taskloop_params_t; 4745 4746 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *, 4747 kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64, 4748 kmp_uint64, kmp_uint64, kmp_int64, kmp_uint64, 4749 kmp_uint64, 4750 #if OMPT_SUPPORT 4751 void *, 4752 #endif 4753 void *); 4754 4755 // Execute part of the taskloop submitted as a task. 4756 int __kmp_taskloop_task(int gtid, void *ptask) { 4757 __taskloop_params_t *p = 4758 (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds; 4759 kmp_task_t *task = p->task; 4760 kmp_uint64 *lb = p->lb; 4761 kmp_uint64 *ub = p->ub; 4762 void *task_dup = p->task_dup; 4763 // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4764 kmp_int64 st = p->st; 4765 kmp_uint64 ub_glob = p->ub_glob; 4766 kmp_uint64 num_tasks = p->num_tasks; 4767 kmp_uint64 grainsize = p->grainsize; 4768 kmp_uint64 extras = p->extras; 4769 kmp_int64 last_chunk = p->last_chunk; 4770 kmp_uint64 tc = p->tc; 4771 kmp_uint64 num_t_min = p->num_t_min; 4772 #if OMPT_SUPPORT 4773 void *codeptr_ra = p->codeptr_ra; 4774 #endif 4775 #if KMP_DEBUG 4776 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4777 KMP_DEBUG_ASSERT(task != NULL); 4778 KA_TRACE(20, 4779 ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize" 4780 " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n", 4781 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub, 4782 st, task_dup)); 4783 #endif 4784 KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min); 4785 if (num_tasks > num_t_min) 4786 __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4787 grainsize, extras, last_chunk, tc, num_t_min, 4788 #if OMPT_SUPPORT 4789 codeptr_ra, 4790 #endif 4791 task_dup); 4792 else 4793 __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4794 grainsize, extras, last_chunk, tc, 4795 #if OMPT_SUPPORT 4796 codeptr_ra, 4797 #endif 4798 task_dup); 4799 4800 KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid)); 4801 return 0; 4802 } 4803 4804 // Schedule part of the taskloop as a task, 4805 // execute the rest of the taskloop. 4806 // 4807 // loc Source location information 4808 // gtid Global thread ID 4809 // task Pattern task, exposes the loop iteration range 4810 // lb Pointer to loop lower bound in task structure 4811 // ub Pointer to loop upper bound in task structure 4812 // st Loop stride 4813 // ub_glob Global upper bound (used for lastprivate check) 4814 // num_tasks Number of tasks to execute 4815 // grainsize Number of loop iterations per task 4816 // extras Number of chunks with grainsize+1 iterations 4817 // last_chunk Reduction of grainsize for last task 4818 // tc Iterations count 4819 // num_t_min Threshold to launch tasks recursively 4820 // task_dup Tasks duplication routine 4821 // codeptr_ra Return address for OMPT events 4822 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task, 4823 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4824 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4825 kmp_uint64 grainsize, kmp_uint64 extras, 4826 kmp_int64 last_chunk, kmp_uint64 tc, 4827 kmp_uint64 num_t_min, 4828 #if OMPT_SUPPORT 4829 void *codeptr_ra, 4830 #endif 4831 void *task_dup) { 4832 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4833 KMP_DEBUG_ASSERT(task != NULL); 4834 KMP_DEBUG_ASSERT(num_tasks > num_t_min); 4835 KA_TRACE(20, 4836 ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize" 4837 " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n", 4838 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub, 4839 st, task_dup)); 4840 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4841 kmp_uint64 lower = *lb; 4842 kmp_info_t *thread = __kmp_threads[gtid]; 4843 // kmp_taskdata_t *current_task = thread->th.th_current_task; 4844 kmp_task_t *next_task; 4845 size_t lower_offset = 4846 (char *)lb - (char *)task; // remember offset of lb in the task structure 4847 size_t upper_offset = 4848 (char *)ub - (char *)task; // remember offset of ub in the task structure 4849 4850 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + 4851 (last_chunk < 0 ? last_chunk : extras)); 4852 KMP_DEBUG_ASSERT(num_tasks > extras); 4853 KMP_DEBUG_ASSERT(num_tasks > 0); 4854 4855 // split the loop in two halves 4856 kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1; 4857 kmp_int64 last_chunk0 = 0, last_chunk1 = 0; 4858 kmp_uint64 gr_size0 = grainsize; 4859 kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute 4860 kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task 4861 if (last_chunk < 0) { 4862 ext0 = ext1 = 0; 4863 last_chunk1 = last_chunk; 4864 tc0 = grainsize * n_tsk0; 4865 tc1 = tc - tc0; 4866 } else if (n_tsk0 <= extras) { 4867 gr_size0++; // integrate extras into grainsize 4868 ext0 = 0; // no extra iters in 1st half 4869 ext1 = extras - n_tsk0; // remaining extras 4870 tc0 = gr_size0 * n_tsk0; 4871 tc1 = tc - tc0; 4872 } else { // n_tsk0 > extras 4873 ext1 = 0; // no extra iters in 2nd half 4874 ext0 = extras; 4875 tc1 = grainsize * n_tsk1; 4876 tc0 = tc - tc1; 4877 } 4878 ub0 = lower + st * (tc0 - 1); 4879 lb1 = ub0 + st; 4880 4881 // create pattern task for 2nd half of the loop 4882 next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task 4883 // adjust lower bound (upper bound is not changed) for the 2nd half 4884 *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1; 4885 if (ptask_dup != NULL) // construct firstprivates, etc. 4886 ptask_dup(next_task, task, 0); 4887 *ub = ub0; // adjust upper bound for the 1st half 4888 4889 // create auxiliary task for 2nd half of the loop 4890 // make sure new task has same parent task as the pattern task 4891 kmp_taskdata_t *current_task = thread->th.th_current_task; 4892 thread->th.th_current_task = taskdata->td_parent; 4893 kmp_task_t *new_task = 4894 __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *), 4895 sizeof(__taskloop_params_t), &__kmp_taskloop_task); 4896 // restore current task 4897 thread->th.th_current_task = current_task; 4898 __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds; 4899 p->task = next_task; 4900 p->lb = (kmp_uint64 *)((char *)next_task + lower_offset); 4901 p->ub = (kmp_uint64 *)((char *)next_task + upper_offset); 4902 p->task_dup = task_dup; 4903 p->st = st; 4904 p->ub_glob = ub_glob; 4905 p->num_tasks = n_tsk1; 4906 p->grainsize = grainsize; 4907 p->extras = ext1; 4908 p->last_chunk = last_chunk1; 4909 p->tc = tc1; 4910 p->num_t_min = num_t_min; 4911 #if OMPT_SUPPORT 4912 p->codeptr_ra = codeptr_ra; 4913 #endif 4914 4915 #if OMPT_SUPPORT 4916 // schedule new task with correct return address for OMPT events 4917 __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra); 4918 #else 4919 __kmp_omp_task(gtid, new_task, true); // schedule new task 4920 #endif 4921 4922 // execute the 1st half of current subrange 4923 if (n_tsk0 > num_t_min) 4924 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0, 4925 ext0, last_chunk0, tc0, num_t_min, 4926 #if OMPT_SUPPORT 4927 codeptr_ra, 4928 #endif 4929 task_dup); 4930 else 4931 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, 4932 gr_size0, ext0, last_chunk0, tc0, 4933 #if OMPT_SUPPORT 4934 codeptr_ra, 4935 #endif 4936 task_dup); 4937 4938 KA_TRACE(40, ("__kmp_taskloop_recur(exit): T#%d\n", gtid)); 4939 } 4940 4941 static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, 4942 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4943 int nogroup, int sched, kmp_uint64 grainsize, 4944 int modifier, void *task_dup) { 4945 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4946 KMP_DEBUG_ASSERT(task != NULL); 4947 if (nogroup == 0) { 4948 #if OMPT_SUPPORT && OMPT_OPTIONAL 4949 OMPT_STORE_RETURN_ADDRESS(gtid); 4950 #endif 4951 __kmpc_taskgroup(loc, gtid); 4952 } 4953 4954 // ========================================================================= 4955 // calculate loop parameters 4956 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4957 kmp_uint64 tc; 4958 // compiler provides global bounds here 4959 kmp_uint64 lower = task_bounds.get_lb(); 4960 kmp_uint64 upper = task_bounds.get_ub(); 4961 kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag 4962 kmp_uint64 num_tasks = 0, extras = 0; 4963 kmp_int64 last_chunk = 4964 0; // reduce grainsize of last task by last_chunk in strict mode 4965 kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks; 4966 kmp_info_t *thread = __kmp_threads[gtid]; 4967 kmp_taskdata_t *current_task = thread->th.th_current_task; 4968 4969 KA_TRACE(20, ("__kmp_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, " 4970 "grain %llu(%d, %d), dup %p\n", 4971 gtid, taskdata, lower, upper, st, grainsize, sched, modifier, 4972 task_dup)); 4973 4974 // compute trip count 4975 if (st == 1) { // most common case 4976 tc = upper - lower + 1; 4977 } else if (st < 0) { 4978 tc = (lower - upper) / (-st) + 1; 4979 } else { // st > 0 4980 tc = (upper - lower) / st + 1; 4981 } 4982 if (tc == 0) { 4983 KA_TRACE(20, ("__kmp_taskloop(exit): T#%d zero-trip loop\n", gtid)); 4984 // free the pattern task and exit 4985 __kmp_task_start(gtid, task, current_task); 4986 // do not execute anything for zero-trip loop 4987 __kmp_task_finish<false>(gtid, task, current_task); 4988 return; 4989 } 4990 4991 #if OMPT_SUPPORT && OMPT_OPTIONAL 4992 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 4993 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 4994 if (ompt_enabled.ompt_callback_work) { 4995 ompt_callbacks.ompt_callback(ompt_callback_work)( 4996 ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data), 4997 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 4998 } 4999 #endif 5000 5001 if (num_tasks_min == 0) 5002 // TODO: can we choose better default heuristic? 5003 num_tasks_min = 5004 KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE); 5005 5006 // compute num_tasks/grainsize based on the input provided 5007 switch (sched) { 5008 case 0: // no schedule clause specified, we can choose the default 5009 // let's try to schedule (team_size*10) tasks 5010 grainsize = thread->th.th_team_nproc * 10; 5011 KMP_FALLTHROUGH(); 5012 case 2: // num_tasks provided 5013 if (grainsize > tc) { 5014 num_tasks = tc; // too big num_tasks requested, adjust values 5015 grainsize = 1; 5016 extras = 0; 5017 } else { 5018 num_tasks = grainsize; 5019 grainsize = tc / num_tasks; 5020 extras = tc % num_tasks; 5021 } 5022 break; 5023 case 1: // grainsize provided 5024 if (grainsize > tc) { 5025 num_tasks = 1; 5026 grainsize = tc; // too big grainsize requested, adjust values 5027 extras = 0; 5028 } else { 5029 if (modifier) { 5030 num_tasks = (tc + grainsize - 1) / grainsize; 5031 last_chunk = tc - (num_tasks * grainsize); 5032 extras = 0; 5033 } else { 5034 num_tasks = tc / grainsize; 5035 // adjust grainsize for balanced distribution of iterations 5036 grainsize = tc / num_tasks; 5037 extras = tc % num_tasks; 5038 } 5039 } 5040 break; 5041 default: 5042 KMP_ASSERT2(0, "unknown scheduling of taskloop"); 5043 } 5044 5045 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + 5046 (last_chunk < 0 ? last_chunk : extras)); 5047 KMP_DEBUG_ASSERT(num_tasks > extras); 5048 KMP_DEBUG_ASSERT(num_tasks > 0); 5049 // ========================================================================= 5050 5051 // check if clause value first 5052 // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native) 5053 if (if_val == 0) { // if(0) specified, mark task as serial 5054 taskdata->td_flags.task_serial = 1; 5055 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied 5056 // always start serial tasks linearly 5057 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 5058 grainsize, extras, last_chunk, tc, 5059 #if OMPT_SUPPORT 5060 OMPT_GET_RETURN_ADDRESS(0), 5061 #endif 5062 task_dup); 5063 // !taskdata->td_flags.native => currently force linear spawning of tasks 5064 // for GOMP_taskloop 5065 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) { 5066 KA_TRACE(20, ("__kmp_taskloop: T#%d, go recursive: tc %llu, #tasks %llu" 5067 "(%lld), grain %llu, extras %llu, last_chunk %lld\n", 5068 gtid, tc, num_tasks, num_tasks_min, grainsize, extras, 5069 last_chunk)); 5070 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 5071 grainsize, extras, last_chunk, tc, num_tasks_min, 5072 #if OMPT_SUPPORT 5073 OMPT_GET_RETURN_ADDRESS(0), 5074 #endif 5075 task_dup); 5076 } else { 5077 KA_TRACE(20, ("__kmp_taskloop: T#%d, go linear: tc %llu, #tasks %llu" 5078 "(%lld), grain %llu, extras %llu, last_chunk %lld\n", 5079 gtid, tc, num_tasks, num_tasks_min, grainsize, extras, 5080 last_chunk)); 5081 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 5082 grainsize, extras, last_chunk, tc, 5083 #if OMPT_SUPPORT 5084 OMPT_GET_RETURN_ADDRESS(0), 5085 #endif 5086 task_dup); 5087 } 5088 5089 #if OMPT_SUPPORT && OMPT_OPTIONAL 5090 if (ompt_enabled.ompt_callback_work) { 5091 ompt_callbacks.ompt_callback(ompt_callback_work)( 5092 ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data), 5093 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 5094 } 5095 #endif 5096 5097 if (nogroup == 0) { 5098 #if OMPT_SUPPORT && OMPT_OPTIONAL 5099 OMPT_STORE_RETURN_ADDRESS(gtid); 5100 #endif 5101 __kmpc_end_taskgroup(loc, gtid); 5102 } 5103 KA_TRACE(20, ("__kmp_taskloop(exit): T#%d\n", gtid)); 5104 } 5105 5106 /*! 5107 @ingroup TASKING 5108 @param loc Source location information 5109 @param gtid Global thread ID 5110 @param task Task structure 5111 @param if_val Value of the if clause 5112 @param lb Pointer to loop lower bound in task structure 5113 @param ub Pointer to loop upper bound in task structure 5114 @param st Loop stride 5115 @param nogroup Flag, 1 if nogroup clause specified, 0 otherwise 5116 @param sched Schedule specified 0/1/2 for none/grainsize/num_tasks 5117 @param grainsize Schedule value if specified 5118 @param task_dup Tasks duplication routine 5119 5120 Execute the taskloop construct. 5121 */ 5122 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, 5123 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, 5124 int sched, kmp_uint64 grainsize, void *task_dup) { 5125 __kmp_assert_valid_gtid(gtid); 5126 KA_TRACE(20, ("__kmpc_taskloop(enter): T#%d\n", gtid)); 5127 __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize, 5128 0, task_dup); 5129 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid)); 5130 } 5131 5132 /*! 5133 @ingroup TASKING 5134 @param loc Source location information 5135 @param gtid Global thread ID 5136 @param task Task structure 5137 @param if_val Value of the if clause 5138 @param lb Pointer to loop lower bound in task structure 5139 @param ub Pointer to loop upper bound in task structure 5140 @param st Loop stride 5141 @param nogroup Flag, 1 if nogroup clause specified, 0 otherwise 5142 @param sched Schedule specified 0/1/2 for none/grainsize/num_tasks 5143 @param grainsize Schedule value if specified 5144 @param modifier Modifier 'strict' for sched, 1 if present, 0 otherwise 5145 @param task_dup Tasks duplication routine 5146 5147 Execute the taskloop construct. 5148 */ 5149 void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val, 5150 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 5151 int nogroup, int sched, kmp_uint64 grainsize, 5152 int modifier, void *task_dup) { 5153 __kmp_assert_valid_gtid(gtid); 5154 KA_TRACE(20, ("__kmpc_taskloop_5(enter): T#%d\n", gtid)); 5155 __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize, 5156 modifier, task_dup); 5157 KA_TRACE(20, ("__kmpc_taskloop_5(exit): T#%d\n", gtid)); 5158 } 5159 5160 /*! 5161 @ingroup TASKING 5162 @param gtid Global Thread ID of current thread 5163 @return Returns a pointer to the thread's current task async handle. If no task 5164 is present or gtid is invalid, returns NULL. 5165 5166 Acqurires a pointer to the target async handle from the current task. 5167 */ 5168 void **__kmpc_omp_get_target_async_handle_ptr(kmp_int32 gtid) { 5169 if (gtid == KMP_GTID_DNE) 5170 return NULL; 5171 5172 kmp_info_t *thread = __kmp_thread_from_gtid(gtid); 5173 kmp_taskdata_t *taskdata = thread->th.th_current_task; 5174 5175 if (!taskdata) 5176 return NULL; 5177 5178 return &taskdata->td_target_data.async_handle; 5179 } 5180 5181 /*! 5182 @ingroup TASKING 5183 @param gtid Global Thread ID of current thread 5184 @return Returns TRUE if the current task being executed of the given thread has 5185 a task team allocated to it. Otherwise, returns FALSE. 5186 5187 Checks if the current thread has a task team. 5188 */ 5189 bool __kmpc_omp_has_task_team(kmp_int32 gtid) { 5190 if (gtid == KMP_GTID_DNE) 5191 return FALSE; 5192 5193 kmp_info_t *thread = __kmp_thread_from_gtid(gtid); 5194 kmp_taskdata_t *taskdata = thread->th.th_current_task; 5195 5196 if (!taskdata) 5197 return FALSE; 5198 5199 return taskdata->td_task_team != NULL; 5200 } 5201