1 /* 2 * kmp_dispatch_hier.h -- hierarchical scheduling methods and data structures 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef KMP_DISPATCH_HIER_H 14 #define KMP_DISPATCH_HIER_H 15 #include "kmp.h" 16 #include "kmp_dispatch.h" 17 18 // Layer type for scheduling hierarchy 19 enum kmp_hier_layer_e { 20 LAYER_THREAD = -1, 21 LAYER_L1, 22 LAYER_L2, 23 LAYER_L3, 24 LAYER_NUMA, 25 LAYER_LOOP, 26 LAYER_LAST 27 }; 28 29 // Convert hierarchy type (LAYER_L1, LAYER_L2, etc.) to C-style string 30 static inline const char *__kmp_get_hier_str(kmp_hier_layer_e type) { 31 switch (type) { 32 case kmp_hier_layer_e::LAYER_THREAD: 33 return "THREAD"; 34 case kmp_hier_layer_e::LAYER_L1: 35 return "L1"; 36 case kmp_hier_layer_e::LAYER_L2: 37 return "L2"; 38 case kmp_hier_layer_e::LAYER_L3: 39 return "L3"; 40 case kmp_hier_layer_e::LAYER_NUMA: 41 return "NUMA"; 42 case kmp_hier_layer_e::LAYER_LOOP: 43 return "WHOLE_LOOP"; 44 case kmp_hier_layer_e::LAYER_LAST: 45 return "LAST"; 46 } 47 KMP_ASSERT(0); 48 // Appease compilers, should never get here 49 return "ERROR"; 50 } 51 52 // Structure to store values parsed from OMP_SCHEDULE for scheduling hierarchy 53 typedef struct kmp_hier_sched_env_t { 54 int size; 55 int capacity; 56 enum sched_type *scheds; 57 kmp_int32 *small_chunks; 58 kmp_int64 *large_chunks; 59 kmp_hier_layer_e *layers; 60 // Append a level of the hierarchy 61 void append(enum sched_type sched, kmp_int32 chunk, kmp_hier_layer_e layer) { 62 if (capacity == 0) { 63 scheds = (enum sched_type *)__kmp_allocate(sizeof(enum sched_type) * 64 kmp_hier_layer_e::LAYER_LAST); 65 small_chunks = (kmp_int32 *)__kmp_allocate(sizeof(kmp_int32) * 66 kmp_hier_layer_e::LAYER_LAST); 67 large_chunks = (kmp_int64 *)__kmp_allocate(sizeof(kmp_int64) * 68 kmp_hier_layer_e::LAYER_LAST); 69 layers = (kmp_hier_layer_e *)__kmp_allocate(sizeof(kmp_hier_layer_e) * 70 kmp_hier_layer_e::LAYER_LAST); 71 capacity = kmp_hier_layer_e::LAYER_LAST; 72 } 73 int current_size = size; 74 KMP_DEBUG_ASSERT(current_size < kmp_hier_layer_e::LAYER_LAST); 75 scheds[current_size] = sched; 76 layers[current_size] = layer; 77 small_chunks[current_size] = chunk; 78 large_chunks[current_size] = (kmp_int64)chunk; 79 size++; 80 } 81 // Sort the hierarchy using selection sort, size will always be small 82 // (less than LAYER_LAST) so it is not necessary to use an nlog(n) algorithm 83 void sort() { 84 if (size <= 1) 85 return; 86 for (int i = 0; i < size; ++i) { 87 int switch_index = i; 88 for (int j = i + 1; j < size; ++j) { 89 if (layers[j] < layers[switch_index]) 90 switch_index = j; 91 } 92 if (switch_index != i) { 93 kmp_hier_layer_e temp1 = layers[i]; 94 enum sched_type temp2 = scheds[i]; 95 kmp_int32 temp3 = small_chunks[i]; 96 kmp_int64 temp4 = large_chunks[i]; 97 layers[i] = layers[switch_index]; 98 scheds[i] = scheds[switch_index]; 99 small_chunks[i] = small_chunks[switch_index]; 100 large_chunks[i] = large_chunks[switch_index]; 101 layers[switch_index] = temp1; 102 scheds[switch_index] = temp2; 103 small_chunks[switch_index] = temp3; 104 large_chunks[switch_index] = temp4; 105 } 106 } 107 } 108 // Free all memory 109 void deallocate() { 110 if (capacity > 0) { 111 __kmp_free(scheds); 112 __kmp_free(layers); 113 __kmp_free(small_chunks); 114 __kmp_free(large_chunks); 115 scheds = NULL; 116 layers = NULL; 117 small_chunks = NULL; 118 large_chunks = NULL; 119 } 120 size = 0; 121 capacity = 0; 122 } 123 } kmp_hier_sched_env_t; 124 125 extern int __kmp_dispatch_hand_threading; 126 extern kmp_hier_sched_env_t __kmp_hier_scheds; 127 128 // Sizes of layer arrays bounded by max number of detected L1s, L2s, etc. 129 extern int __kmp_hier_max_units[kmp_hier_layer_e::LAYER_LAST + 1]; 130 extern int __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_LAST + 1]; 131 132 extern int __kmp_dispatch_get_index(int tid, kmp_hier_layer_e type); 133 extern int __kmp_dispatch_get_id(int gtid, kmp_hier_layer_e type); 134 extern int __kmp_dispatch_get_t1_per_t2(kmp_hier_layer_e t1, 135 kmp_hier_layer_e t2); 136 extern void __kmp_dispatch_free_hierarchies(kmp_team_t *team); 137 138 template <typename T> struct kmp_hier_shared_bdata_t { 139 typedef typename traits_t<T>::signed_t ST; 140 volatile kmp_uint64 val[2]; 141 kmp_int32 status[2]; 142 T lb[2]; 143 T ub[2]; 144 ST st[2]; 145 dispatch_shared_info_template<T> sh[2]; 146 void zero() { 147 val[0] = val[1] = 0; 148 status[0] = status[1] = 0; 149 lb[0] = lb[1] = 0; 150 ub[0] = ub[1] = 0; 151 st[0] = st[1] = 0; 152 sh[0].u.s.iteration = sh[1].u.s.iteration = 0; 153 } 154 void set_next_hand_thread(T nlb, T nub, ST nst, kmp_int32 nstatus, 155 kmp_uint64 index) { 156 lb[1 - index] = nlb; 157 ub[1 - index] = nub; 158 st[1 - index] = nst; 159 status[1 - index] = nstatus; 160 } 161 void set_next(T nlb, T nub, ST nst, kmp_int32 nstatus, kmp_uint64 index) { 162 lb[1 - index] = nlb; 163 ub[1 - index] = nub; 164 st[1 - index] = nst; 165 status[1 - index] = nstatus; 166 sh[1 - index].u.s.iteration = 0; 167 } 168 169 kmp_int32 get_next_status(kmp_uint64 index) const { 170 return status[1 - index]; 171 } 172 T get_next_lb(kmp_uint64 index) const { return lb[1 - index]; } 173 T get_next_ub(kmp_uint64 index) const { return ub[1 - index]; } 174 ST get_next_st(kmp_uint64 index) const { return st[1 - index]; } 175 dispatch_shared_info_template<T> volatile *get_next_sh(kmp_uint64 index) { 176 return &(sh[1 - index]); 177 } 178 179 kmp_int32 get_curr_status(kmp_uint64 index) const { return status[index]; } 180 T get_curr_lb(kmp_uint64 index) const { return lb[index]; } 181 T get_curr_ub(kmp_uint64 index) const { return ub[index]; } 182 ST get_curr_st(kmp_uint64 index) const { return st[index]; } 183 dispatch_shared_info_template<T> volatile *get_curr_sh(kmp_uint64 index) { 184 return &(sh[index]); 185 } 186 }; 187 188 /* 189 * In the barrier implementations, num_active is the number of threads that are 190 * attached to the kmp_hier_top_unit_t structure in the scheduling hierarchy. 191 * bdata is the shared barrier data that resides on the kmp_hier_top_unit_t 192 * structure. tdata is the thread private data that resides on the thread 193 * data structure. 194 * 195 * The reset_shared() method is used to initialize the barrier data on the 196 * kmp_hier_top_unit_t hierarchy structure 197 * 198 * The reset_private() method is used to initialize the barrier data on the 199 * thread's private dispatch buffer structure 200 * 201 * The barrier() method takes an id, which is that thread's id for the 202 * kmp_hier_top_unit_t structure, and implements the barrier. All threads wait 203 * inside barrier() until all fellow threads who are attached to that 204 * kmp_hier_top_unit_t structure have arrived. 205 */ 206 207 // Core barrier implementation 208 // Can be used in a unit with between 2 to 8 threads 209 template <typename T> class core_barrier_impl { 210 static inline kmp_uint64 get_wait_val(int num_active) { 211 kmp_uint64 wait_val = 0LL; 212 switch (num_active) { 213 case 2: 214 wait_val = 0x0101LL; 215 break; 216 case 3: 217 wait_val = 0x010101LL; 218 break; 219 case 4: 220 wait_val = 0x01010101LL; 221 break; 222 case 5: 223 wait_val = 0x0101010101LL; 224 break; 225 case 6: 226 wait_val = 0x010101010101LL; 227 break; 228 case 7: 229 wait_val = 0x01010101010101LL; 230 break; 231 case 8: 232 wait_val = 0x0101010101010101LL; 233 break; 234 default: 235 // don't use the core_barrier_impl for more than 8 threads 236 KMP_ASSERT(0); 237 } 238 return wait_val; 239 } 240 241 public: 242 static void reset_private(kmp_int32 num_active, 243 kmp_hier_private_bdata_t *tdata); 244 static void reset_shared(kmp_int32 num_active, 245 kmp_hier_shared_bdata_t<T> *bdata); 246 static void barrier(kmp_int32 id, kmp_hier_shared_bdata_t<T> *bdata, 247 kmp_hier_private_bdata_t *tdata); 248 }; 249 250 template <typename T> 251 void core_barrier_impl<T>::reset_private(kmp_int32 num_active, 252 kmp_hier_private_bdata_t *tdata) { 253 tdata->num_active = num_active; 254 tdata->index = 0; 255 tdata->wait_val[0] = tdata->wait_val[1] = get_wait_val(num_active); 256 } 257 template <typename T> 258 void core_barrier_impl<T>::reset_shared(kmp_int32 num_active, 259 kmp_hier_shared_bdata_t<T> *bdata) { 260 bdata->val[0] = bdata->val[1] = 0LL; 261 bdata->status[0] = bdata->status[1] = 0LL; 262 } 263 template <typename T> 264 void core_barrier_impl<T>::barrier(kmp_int32 id, 265 kmp_hier_shared_bdata_t<T> *bdata, 266 kmp_hier_private_bdata_t *tdata) { 267 kmp_uint64 current_index = tdata->index; 268 kmp_uint64 next_index = 1 - current_index; 269 kmp_uint64 current_wait_value = tdata->wait_val[current_index]; 270 kmp_uint64 next_wait_value = 271 (current_wait_value ? 0 : get_wait_val(tdata->num_active)); 272 KD_TRACE(10, ("core_barrier_impl::barrier(): T#%d current_index:%llu " 273 "next_index:%llu curr_wait:%llu next_wait:%llu\n", 274 __kmp_get_gtid(), current_index, next_index, current_wait_value, 275 next_wait_value)); 276 char v = (current_wait_value ? 0x1 : 0x0); 277 (RCAST(volatile char *, &(bdata->val[current_index])))[id] = v; 278 __kmp_wait<kmp_uint64>(&(bdata->val[current_index]), current_wait_value, 279 __kmp_eq<kmp_uint64> USE_ITT_BUILD_ARG(NULL)); 280 tdata->wait_val[current_index] = next_wait_value; 281 tdata->index = next_index; 282 } 283 284 // Counter barrier implementation 285 // Can be used in a unit with arbitrary number of active threads 286 template <typename T> class counter_barrier_impl { 287 public: 288 static void reset_private(kmp_int32 num_active, 289 kmp_hier_private_bdata_t *tdata); 290 static void reset_shared(kmp_int32 num_active, 291 kmp_hier_shared_bdata_t<T> *bdata); 292 static void barrier(kmp_int32 id, kmp_hier_shared_bdata_t<T> *bdata, 293 kmp_hier_private_bdata_t *tdata); 294 }; 295 296 template <typename T> 297 void counter_barrier_impl<T>::reset_private(kmp_int32 num_active, 298 kmp_hier_private_bdata_t *tdata) { 299 tdata->num_active = num_active; 300 tdata->index = 0; 301 tdata->wait_val[0] = tdata->wait_val[1] = (kmp_uint64)num_active; 302 } 303 template <typename T> 304 void counter_barrier_impl<T>::reset_shared(kmp_int32 num_active, 305 kmp_hier_shared_bdata_t<T> *bdata) { 306 bdata->val[0] = bdata->val[1] = 0LL; 307 bdata->status[0] = bdata->status[1] = 0LL; 308 } 309 template <typename T> 310 void counter_barrier_impl<T>::barrier(kmp_int32 id, 311 kmp_hier_shared_bdata_t<T> *bdata, 312 kmp_hier_private_bdata_t *tdata) { 313 volatile kmp_int64 *val; 314 kmp_uint64 current_index = tdata->index; 315 kmp_uint64 next_index = 1 - current_index; 316 kmp_uint64 current_wait_value = tdata->wait_val[current_index]; 317 kmp_uint64 next_wait_value = current_wait_value + tdata->num_active; 318 319 KD_TRACE(10, ("counter_barrier_impl::barrier(): T#%d current_index:%llu " 320 "next_index:%llu curr_wait:%llu next_wait:%llu\n", 321 __kmp_get_gtid(), current_index, next_index, current_wait_value, 322 next_wait_value)); 323 val = RCAST(volatile kmp_int64 *, &(bdata->val[current_index])); 324 KMP_TEST_THEN_INC64(val); 325 __kmp_wait<kmp_uint64>(&(bdata->val[current_index]), current_wait_value, 326 __kmp_ge<kmp_uint64> USE_ITT_BUILD_ARG(NULL)); 327 tdata->wait_val[current_index] = next_wait_value; 328 tdata->index = next_index; 329 } 330 331 // Data associated with topology unit within a layer 332 // For example, one kmp_hier_top_unit_t corresponds to one L1 cache 333 template <typename T> struct kmp_hier_top_unit_t { 334 typedef typename traits_t<T>::signed_t ST; 335 typedef typename traits_t<T>::unsigned_t UT; 336 kmp_int32 active; // number of topology units that communicate with this unit 337 // chunk information (lower/upper bound, stride, etc.) 338 dispatch_private_info_template<T> hier_pr; 339 kmp_hier_top_unit_t<T> *hier_parent; // pointer to parent unit 340 kmp_hier_shared_bdata_t<T> hier_barrier; // shared barrier data for this unit 341 342 kmp_int32 get_hier_id() const { return hier_pr.hier_id; } 343 void reset_shared_barrier() { 344 KMP_DEBUG_ASSERT(active > 0); 345 if (active == 1) 346 return; 347 hier_barrier.zero(); 348 if (active >= 2 && active <= 8) { 349 core_barrier_impl<T>::reset_shared(active, &hier_barrier); 350 } else { 351 counter_barrier_impl<T>::reset_shared(active, &hier_barrier); 352 } 353 } 354 void reset_private_barrier(kmp_hier_private_bdata_t *tdata) { 355 KMP_DEBUG_ASSERT(tdata); 356 KMP_DEBUG_ASSERT(active > 0); 357 if (active == 1) 358 return; 359 if (active >= 2 && active <= 8) { 360 core_barrier_impl<T>::reset_private(active, tdata); 361 } else { 362 counter_barrier_impl<T>::reset_private(active, tdata); 363 } 364 } 365 void barrier(kmp_int32 id, kmp_hier_private_bdata_t *tdata) { 366 KMP_DEBUG_ASSERT(tdata); 367 KMP_DEBUG_ASSERT(active > 0); 368 KMP_DEBUG_ASSERT(id >= 0 && id < active); 369 if (active == 1) { 370 tdata->index = 1 - tdata->index; 371 return; 372 } 373 if (active >= 2 && active <= 8) { 374 core_barrier_impl<T>::barrier(id, &hier_barrier, tdata); 375 } else { 376 counter_barrier_impl<T>::barrier(id, &hier_barrier, tdata); 377 } 378 } 379 380 kmp_int32 get_next_status(kmp_uint64 index) const { 381 return hier_barrier.get_next_status(index); 382 } 383 T get_next_lb(kmp_uint64 index) const { 384 return hier_barrier.get_next_lb(index); 385 } 386 T get_next_ub(kmp_uint64 index) const { 387 return hier_barrier.get_next_ub(index); 388 } 389 ST get_next_st(kmp_uint64 index) const { 390 return hier_barrier.get_next_st(index); 391 } 392 dispatch_shared_info_template<T> volatile *get_next_sh(kmp_uint64 index) { 393 return hier_barrier.get_next_sh(index); 394 } 395 396 kmp_int32 get_curr_status(kmp_uint64 index) const { 397 return hier_barrier.get_curr_status(index); 398 } 399 T get_curr_lb(kmp_uint64 index) const { 400 return hier_barrier.get_curr_lb(index); 401 } 402 T get_curr_ub(kmp_uint64 index) const { 403 return hier_barrier.get_curr_ub(index); 404 } 405 ST get_curr_st(kmp_uint64 index) const { 406 return hier_barrier.get_curr_st(index); 407 } 408 dispatch_shared_info_template<T> volatile *get_curr_sh(kmp_uint64 index) { 409 return hier_barrier.get_curr_sh(index); 410 } 411 412 void set_next_hand_thread(T lb, T ub, ST st, kmp_int32 status, 413 kmp_uint64 index) { 414 hier_barrier.set_next_hand_thread(lb, ub, st, status, index); 415 } 416 void set_next(T lb, T ub, ST st, kmp_int32 status, kmp_uint64 index) { 417 hier_barrier.set_next(lb, ub, st, status, index); 418 } 419 dispatch_private_info_template<T> *get_my_pr() { return &hier_pr; } 420 kmp_hier_top_unit_t<T> *get_parent() { return hier_parent; } 421 dispatch_private_info_template<T> *get_parent_pr() { 422 return &(hier_parent->hier_pr); 423 } 424 425 kmp_int32 is_active() const { return active; } 426 kmp_int32 get_num_active() const { return active; } 427 #ifdef KMP_DEBUG 428 void print() { 429 KD_TRACE( 430 10, 431 (" kmp_hier_top_unit_t: active:%d pr:%p lb:%d ub:%d st:%d tc:%d\n", 432 active, &hier_pr, hier_pr.u.p.lb, hier_pr.u.p.ub, hier_pr.u.p.st, 433 hier_pr.u.p.tc)); 434 } 435 #endif 436 }; 437 438 // Information regarding a single layer within the scheduling hierarchy 439 template <typename T> struct kmp_hier_layer_info_t { 440 int num_active; // number of threads active in this level 441 kmp_hier_layer_e type; // LAYER_L1, LAYER_L2, etc. 442 enum sched_type sched; // static, dynamic, guided, etc. 443 typename traits_t<T>::signed_t chunk; // chunk size associated with schedule 444 int length; // length of the kmp_hier_top_unit_t array 445 446 #ifdef KMP_DEBUG 447 // Print this layer's information 448 void print() { 449 const char *t = __kmp_get_hier_str(type); 450 KD_TRACE( 451 10, 452 (" kmp_hier_layer_info_t: num_active:%d type:%s sched:%d chunk:%d " 453 "length:%d\n", 454 num_active, t, sched, chunk, length)); 455 } 456 #endif 457 }; 458 459 /* 460 * Structure to implement entire hierarchy 461 * 462 * The hierarchy is kept as an array of arrays to represent the different 463 * layers. Layer 0 is the lowest layer to layer num_layers - 1 which is the 464 * highest layer. 465 * Example: 466 * [ 2 ] -> [ L3 | L3 ] 467 * [ 1 ] -> [ L2 | L2 | L2 | L2 ] 468 * [ 0 ] -> [ L1 | L1 | L1 | L1 | L1 | L1 | L1 | L1 ] 469 * There is also an array of layer_info_t which has information regarding 470 * each layer 471 */ 472 template <typename T> struct kmp_hier_t { 473 public: 474 typedef typename traits_t<T>::unsigned_t UT; 475 typedef typename traits_t<T>::signed_t ST; 476 477 private: 478 int next_recurse(ident_t *loc, int gtid, kmp_hier_top_unit_t<T> *current, 479 kmp_int32 *p_last, T *p_lb, T *p_ub, ST *p_st, 480 kmp_int32 previous_id, int hier_level) { 481 int status; 482 kmp_info_t *th = __kmp_threads[gtid]; 483 auto parent = current->get_parent(); 484 bool last_layer = (hier_level == get_num_layers() - 1); 485 KMP_DEBUG_ASSERT(th); 486 kmp_hier_private_bdata_t *tdata = &(th->th.th_hier_bar_data[hier_level]); 487 KMP_DEBUG_ASSERT(current); 488 KMP_DEBUG_ASSERT(hier_level >= 0); 489 KMP_DEBUG_ASSERT(hier_level < get_num_layers()); 490 KMP_DEBUG_ASSERT(tdata); 491 KMP_DEBUG_ASSERT(parent || last_layer); 492 493 KD_TRACE( 494 1, ("kmp_hier_t.next_recurse(): T#%d (%d) called\n", gtid, hier_level)); 495 496 T hier_id = (T)current->get_hier_id(); 497 // Attempt to grab next iteration range for this level 498 if (previous_id == 0) { 499 KD_TRACE(1, ("kmp_hier_t.next_recurse(): T#%d (%d) is master of unit\n", 500 gtid, hier_level)); 501 kmp_int32 contains_last; 502 T my_lb, my_ub; 503 ST my_st; 504 T nproc; 505 dispatch_shared_info_template<T> volatile *my_sh; 506 dispatch_private_info_template<T> *my_pr; 507 if (last_layer) { 508 // last layer below the very top uses the single shared buffer 509 // from the team struct. 510 KD_TRACE(10, 511 ("kmp_hier_t.next_recurse(): T#%d (%d) using top level sh\n", 512 gtid, hier_level)); 513 my_sh = reinterpret_cast<dispatch_shared_info_template<T> volatile *>( 514 th->th.th_dispatch->th_dispatch_sh_current); 515 nproc = (T)get_top_level_nproc(); 516 } else { 517 // middle layers use the shared buffer inside the kmp_hier_top_unit_t 518 // structure 519 KD_TRACE(10, ("kmp_hier_t.next_recurse(): T#%d (%d) using hier sh\n", 520 gtid, hier_level)); 521 my_sh = 522 parent->get_curr_sh(th->th.th_hier_bar_data[hier_level + 1].index); 523 nproc = (T)parent->get_num_active(); 524 } 525 my_pr = current->get_my_pr(); 526 KMP_DEBUG_ASSERT(my_sh); 527 KMP_DEBUG_ASSERT(my_pr); 528 enum sched_type schedule = get_sched(hier_level); 529 ST chunk = (ST)get_chunk(hier_level); 530 status = __kmp_dispatch_next_algorithm<T>(gtid, my_pr, my_sh, 531 &contains_last, &my_lb, &my_ub, 532 &my_st, nproc, hier_id); 533 KD_TRACE( 534 10, 535 ("kmp_hier_t.next_recurse(): T#%d (%d) next_pr_sh() returned %d\n", 536 gtid, hier_level, status)); 537 // When no iterations are found (status == 0) and this is not the last 538 // layer, attempt to go up the hierarchy for more iterations 539 if (status == 0 && !last_layer) { 540 status = next_recurse(loc, gtid, parent, &contains_last, &my_lb, &my_ub, 541 &my_st, hier_id, hier_level + 1); 542 KD_TRACE( 543 10, 544 ("kmp_hier_t.next_recurse(): T#%d (%d) hier_next() returned %d\n", 545 gtid, hier_level, status)); 546 if (status == 1) { 547 kmp_hier_private_bdata_t *upper_tdata = 548 &(th->th.th_hier_bar_data[hier_level + 1]); 549 my_sh = parent->get_curr_sh(upper_tdata->index); 550 KD_TRACE(10, ("kmp_hier_t.next_recurse(): T#%d (%d) about to init\n", 551 gtid, hier_level)); 552 __kmp_dispatch_init_algorithm(loc, gtid, my_pr, schedule, 553 parent->get_curr_lb(upper_tdata->index), 554 parent->get_curr_ub(upper_tdata->index), 555 parent->get_curr_st(upper_tdata->index), 556 #if USE_ITT_BUILD 557 NULL, 558 #endif 559 chunk, nproc, hier_id); 560 status = __kmp_dispatch_next_algorithm<T>( 561 gtid, my_pr, my_sh, &contains_last, &my_lb, &my_ub, &my_st, nproc, 562 hier_id); 563 if (!status) { 564 KD_TRACE(10, ("kmp_hier_t.next_recurse(): T#%d (%d) status not 1 " 565 "setting to 2!\n", 566 gtid, hier_level)); 567 status = 2; 568 } 569 } 570 } 571 current->set_next(my_lb, my_ub, my_st, status, tdata->index); 572 // Propagate whether a unit holds the actual global last iteration 573 // The contains_last attribute is sent downwards from the top to the 574 // bottom of the hierarchy via the contains_last flag inside the 575 // private dispatch buffers in the hierarchy's middle layers 576 if (contains_last) { 577 // If the next_algorithm() method returns 1 for p_last and it is the 578 // last layer or our parent contains the last serial chunk, then the 579 // chunk must contain the last serial iteration. 580 if (last_layer || parent->hier_pr.flags.contains_last) { 581 KD_TRACE(10, ("kmp_hier_t.next_recurse(): T#%d (%d) Setting this pr " 582 "to contain last.\n", 583 gtid, hier_level)); 584 current->hier_pr.flags.contains_last = contains_last; 585 } 586 if (!current->hier_pr.flags.contains_last) 587 contains_last = FALSE; 588 } 589 if (p_last) 590 *p_last = contains_last; 591 } // if master thread of this unit 592 if (hier_level > 0 || !__kmp_dispatch_hand_threading) { 593 KD_TRACE(10, 594 ("kmp_hier_t.next_recurse(): T#%d (%d) going into barrier.\n", 595 gtid, hier_level)); 596 current->barrier(previous_id, tdata); 597 KD_TRACE(10, 598 ("kmp_hier_t.next_recurse(): T#%d (%d) released and exit %d\n", 599 gtid, hier_level, current->get_curr_status(tdata->index))); 600 } else { 601 KMP_DEBUG_ASSERT(previous_id == 0); 602 return status; 603 } 604 return current->get_curr_status(tdata->index); 605 } 606 607 public: 608 int top_level_nproc; 609 int num_layers; 610 bool valid; 611 int type_size; 612 kmp_hier_layer_info_t<T> *info; 613 kmp_hier_top_unit_t<T> **layers; 614 // Deallocate all memory from this hierarchy 615 void deallocate() { 616 for (int i = 0; i < num_layers; ++i) 617 if (layers[i] != NULL) { 618 __kmp_free(layers[i]); 619 } 620 if (layers != NULL) { 621 __kmp_free(layers); 622 layers = NULL; 623 } 624 if (info != NULL) { 625 __kmp_free(info); 626 info = NULL; 627 } 628 num_layers = 0; 629 valid = false; 630 } 631 // Returns true if reallocation is needed else false 632 bool need_to_reallocate(int n, const kmp_hier_layer_e *new_layers, 633 const enum sched_type *new_scheds, 634 const ST *new_chunks) const { 635 if (!valid || layers == NULL || info == NULL || 636 traits_t<T>::type_size != type_size || n != num_layers) 637 return true; 638 for (int i = 0; i < n; ++i) { 639 if (info[i].type != new_layers[i]) 640 return true; 641 if (info[i].sched != new_scheds[i]) 642 return true; 643 if (info[i].chunk != new_chunks[i]) 644 return true; 645 } 646 return false; 647 } 648 // A single thread should call this function while the other threads wait 649 // create a new scheduling hierarchy consisting of new_layers, new_scheds 650 // and new_chunks. These should come pre-sorted according to 651 // kmp_hier_layer_e value. This function will try to avoid reallocation 652 // if it can 653 void allocate_hier(int n, const kmp_hier_layer_e *new_layers, 654 const enum sched_type *new_scheds, const ST *new_chunks) { 655 top_level_nproc = 0; 656 if (!need_to_reallocate(n, new_layers, new_scheds, new_chunks)) { 657 KD_TRACE( 658 10, 659 ("kmp_hier_t<T>::allocate_hier: T#0 do not need to reallocate\n")); 660 for (int i = 0; i < n; ++i) { 661 info[i].num_active = 0; 662 for (int j = 0; j < get_length(i); ++j) 663 layers[i][j].active = 0; 664 } 665 return; 666 } 667 KD_TRACE(10, ("kmp_hier_t<T>::allocate_hier: T#0 full alloc\n")); 668 deallocate(); 669 type_size = traits_t<T>::type_size; 670 num_layers = n; 671 info = (kmp_hier_layer_info_t<T> *)__kmp_allocate( 672 sizeof(kmp_hier_layer_info_t<T>) * n); 673 layers = (kmp_hier_top_unit_t<T> **)__kmp_allocate( 674 sizeof(kmp_hier_top_unit_t<T> *) * n); 675 for (int i = 0; i < n; ++i) { 676 int max = 0; 677 kmp_hier_layer_e layer = new_layers[i]; 678 info[i].num_active = 0; 679 info[i].type = layer; 680 info[i].sched = new_scheds[i]; 681 info[i].chunk = new_chunks[i]; 682 max = __kmp_hier_max_units[layer + 1]; 683 if (max == 0) { 684 valid = false; 685 KMP_WARNING(HierSchedInvalid, __kmp_get_hier_str(layer)); 686 deallocate(); 687 return; 688 } 689 info[i].length = max; 690 layers[i] = (kmp_hier_top_unit_t<T> *)__kmp_allocate( 691 sizeof(kmp_hier_top_unit_t<T>) * max); 692 for (int j = 0; j < max; ++j) { 693 layers[i][j].active = 0; 694 layers[i][j].hier_pr.flags.use_hier = TRUE; 695 } 696 } 697 valid = true; 698 } 699 // loc - source file location 700 // gtid - global thread identifier 701 // pr - this thread's private dispatch buffer (corresponding with gtid) 702 // p_last (return value) - pointer to flag indicating this set of iterations 703 // contains last 704 // iteration 705 // p_lb (return value) - lower bound for this chunk of iterations 706 // p_ub (return value) - upper bound for this chunk of iterations 707 // p_st (return value) - stride for this chunk of iterations 708 // 709 // Returns 1 if there are more iterations to perform, 0 otherwise 710 int next(ident_t *loc, int gtid, dispatch_private_info_template<T> *pr, 711 kmp_int32 *p_last, T *p_lb, T *p_ub, ST *p_st) { 712 int status; 713 kmp_int32 contains_last = 0; 714 kmp_info_t *th = __kmp_threads[gtid]; 715 kmp_hier_private_bdata_t *tdata = &(th->th.th_hier_bar_data[0]); 716 auto parent = pr->get_parent(); 717 KMP_DEBUG_ASSERT(parent); 718 KMP_DEBUG_ASSERT(th); 719 KMP_DEBUG_ASSERT(tdata); 720 KMP_DEBUG_ASSERT(parent); 721 T nproc = (T)parent->get_num_active(); 722 T unit_id = (T)pr->get_hier_id(); 723 KD_TRACE( 724 10, 725 ("kmp_hier_t.next(): T#%d THREAD LEVEL nproc:%d unit_id:%d called\n", 726 gtid, nproc, unit_id)); 727 // Handthreading implementation 728 // Each iteration is performed by all threads on last unit (typically 729 // cores/tiles) 730 // e.g., threads 0,1,2,3 all execute iteration 0 731 // threads 0,1,2,3 all execute iteration 1 732 // threads 4,5,6,7 all execute iteration 2 733 // threads 4,5,6,7 all execute iteration 3 734 // ... etc. 735 if (__kmp_dispatch_hand_threading) { 736 KD_TRACE(10, 737 ("kmp_hier_t.next(): T#%d THREAD LEVEL using hand threading\n", 738 gtid)); 739 if (unit_id == 0) { 740 // For hand threading, the sh buffer on the lowest level is only ever 741 // modified and read by the master thread on that level. Because of 742 // this, we can always use the first sh buffer. 743 auto sh = &(parent->hier_barrier.sh[0]); 744 KMP_DEBUG_ASSERT(sh); 745 status = __kmp_dispatch_next_algorithm<T>( 746 gtid, pr, sh, &contains_last, p_lb, p_ub, p_st, nproc, unit_id); 747 if (!status) { 748 bool done = false; 749 while (!done) { 750 done = true; 751 status = next_recurse(loc, gtid, parent, &contains_last, p_lb, p_ub, 752 p_st, unit_id, 0); 753 if (status == 1) { 754 __kmp_dispatch_init_algorithm(loc, gtid, pr, pr->schedule, 755 parent->get_next_lb(tdata->index), 756 parent->get_next_ub(tdata->index), 757 parent->get_next_st(tdata->index), 758 #if USE_ITT_BUILD 759 NULL, 760 #endif 761 pr->u.p.parm1, nproc, unit_id); 762 sh->u.s.iteration = 0; 763 status = __kmp_dispatch_next_algorithm<T>( 764 gtid, pr, sh, &contains_last, p_lb, p_ub, p_st, nproc, 765 unit_id); 766 if (!status) { 767 KD_TRACE(10, 768 ("kmp_hier_t.next(): T#%d THREAD LEVEL status == 0 " 769 "after next_pr_sh()" 770 "trying again.\n", 771 gtid)); 772 done = false; 773 } 774 } else if (status == 2) { 775 KD_TRACE(10, ("kmp_hier_t.next(): T#%d THREAD LEVEL status == 2 " 776 "trying again.\n", 777 gtid)); 778 done = false; 779 } 780 } 781 } 782 parent->set_next_hand_thread(*p_lb, *p_ub, *p_st, status, tdata->index); 783 } // if master thread of lowest unit level 784 parent->barrier(pr->get_hier_id(), tdata); 785 if (unit_id != 0) { 786 *p_lb = parent->get_curr_lb(tdata->index); 787 *p_ub = parent->get_curr_ub(tdata->index); 788 *p_st = parent->get_curr_st(tdata->index); 789 status = parent->get_curr_status(tdata->index); 790 } 791 } else { 792 // Normal implementation 793 // Each thread grabs an iteration chunk and executes it (no cooperation) 794 auto sh = parent->get_curr_sh(tdata->index); 795 KMP_DEBUG_ASSERT(sh); 796 status = __kmp_dispatch_next_algorithm<T>( 797 gtid, pr, sh, &contains_last, p_lb, p_ub, p_st, nproc, unit_id); 798 KD_TRACE(10, 799 ("kmp_hier_t.next(): T#%d THREAD LEVEL next_algorithm status:%d " 800 "contains_last:%d p_lb:%d p_ub:%d p_st:%d\n", 801 gtid, status, contains_last, *p_lb, *p_ub, *p_st)); 802 if (!status) { 803 bool done = false; 804 while (!done) { 805 done = true; 806 status = next_recurse(loc, gtid, parent, &contains_last, p_lb, p_ub, 807 p_st, unit_id, 0); 808 if (status == 1) { 809 sh = parent->get_curr_sh(tdata->index); 810 __kmp_dispatch_init_algorithm(loc, gtid, pr, pr->schedule, 811 parent->get_curr_lb(tdata->index), 812 parent->get_curr_ub(tdata->index), 813 parent->get_curr_st(tdata->index), 814 #if USE_ITT_BUILD 815 NULL, 816 #endif 817 pr->u.p.parm1, nproc, unit_id); 818 status = __kmp_dispatch_next_algorithm<T>( 819 gtid, pr, sh, &contains_last, p_lb, p_ub, p_st, nproc, unit_id); 820 if (!status) { 821 KD_TRACE(10, ("kmp_hier_t.next(): T#%d THREAD LEVEL status == 0 " 822 "after next_pr_sh()" 823 "trying again.\n", 824 gtid)); 825 done = false; 826 } 827 } else if (status == 2) { 828 KD_TRACE(10, ("kmp_hier_t.next(): T#%d THREAD LEVEL status == 2 " 829 "trying again.\n", 830 gtid)); 831 done = false; 832 } 833 } 834 } 835 } 836 if (contains_last && !parent->hier_pr.flags.contains_last) { 837 KD_TRACE(10, ("kmp_hier_t.next(): T#%d THREAD LEVEL resetting " 838 "contains_last to FALSE\n", 839 gtid)); 840 contains_last = FALSE; 841 } 842 if (p_last) 843 *p_last = contains_last; 844 KD_TRACE(10, ("kmp_hier_t.next(): T#%d THREAD LEVEL exit status %d\n", gtid, 845 status)); 846 return status; 847 } 848 // These functions probe the layer info structure 849 // Returns the type of topology unit given level 850 kmp_hier_layer_e get_type(int level) const { 851 KMP_DEBUG_ASSERT(level >= 0); 852 KMP_DEBUG_ASSERT(level < num_layers); 853 return info[level].type; 854 } 855 // Returns the schedule type at given level 856 enum sched_type get_sched(int level) const { 857 KMP_DEBUG_ASSERT(level >= 0); 858 KMP_DEBUG_ASSERT(level < num_layers); 859 return info[level].sched; 860 } 861 // Returns the chunk size at given level 862 ST get_chunk(int level) const { 863 KMP_DEBUG_ASSERT(level >= 0); 864 KMP_DEBUG_ASSERT(level < num_layers); 865 return info[level].chunk; 866 } 867 // Returns the number of active threads at given level 868 int get_num_active(int level) const { 869 KMP_DEBUG_ASSERT(level >= 0); 870 KMP_DEBUG_ASSERT(level < num_layers); 871 return info[level].num_active; 872 } 873 // Returns the length of topology unit array at given level 874 int get_length(int level) const { 875 KMP_DEBUG_ASSERT(level >= 0); 876 KMP_DEBUG_ASSERT(level < num_layers); 877 return info[level].length; 878 } 879 // Returns the topology unit given the level and index 880 kmp_hier_top_unit_t<T> *get_unit(int level, int index) { 881 KMP_DEBUG_ASSERT(level >= 0); 882 KMP_DEBUG_ASSERT(level < num_layers); 883 KMP_DEBUG_ASSERT(index >= 0); 884 KMP_DEBUG_ASSERT(index < get_length(level)); 885 return &(layers[level][index]); 886 } 887 // Returns the number of layers in the hierarchy 888 int get_num_layers() const { return num_layers; } 889 // Returns the number of threads in the top layer 890 // This is necessary because we don't store a topology unit as 891 // the very top level and the scheduling algorithms need this information 892 int get_top_level_nproc() const { return top_level_nproc; } 893 // Return whether this hierarchy is valid or not 894 bool is_valid() const { return valid; } 895 #ifdef KMP_DEBUG 896 // Print the hierarchy 897 void print() { 898 KD_TRACE(10, ("kmp_hier_t:\n")); 899 for (int i = num_layers - 1; i >= 0; --i) { 900 KD_TRACE(10, ("Info[%d] = ", i)); 901 info[i].print(); 902 } 903 for (int i = num_layers - 1; i >= 0; --i) { 904 KD_TRACE(10, ("Layer[%d] =\n", i)); 905 for (int j = 0; j < info[i].length; ++j) { 906 layers[i][j].print(); 907 } 908 } 909 } 910 #endif 911 }; 912 913 template <typename T> 914 void __kmp_dispatch_init_hierarchy(ident_t *loc, int n, 915 kmp_hier_layer_e *new_layers, 916 enum sched_type *new_scheds, 917 typename traits_t<T>::signed_t *new_chunks, 918 T lb, T ub, 919 typename traits_t<T>::signed_t st) { 920 int tid, gtid, num_hw_threads, num_threads_per_layer1, active; 921 int my_buffer_index; 922 kmp_info_t *th; 923 kmp_team_t *team; 924 dispatch_private_info_template<T> *pr; 925 dispatch_shared_info_template<T> volatile *sh; 926 gtid = __kmp_entry_gtid(); 927 tid = __kmp_tid_from_gtid(gtid); 928 #ifdef KMP_DEBUG 929 KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d called: %d layer(s)\n", 930 gtid, n)); 931 for (int i = 0; i < n; ++i) { 932 const char *layer = __kmp_get_hier_str(new_layers[i]); 933 KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d: new_layers[%d] = %s, " 934 "new_scheds[%d] = %d, new_chunks[%d] = %u\n", 935 gtid, i, layer, i, (int)new_scheds[i], i, new_chunks[i])); 936 } 937 #endif // KMP_DEBUG 938 KMP_DEBUG_ASSERT(n > 0); 939 KMP_DEBUG_ASSERT(new_layers); 940 KMP_DEBUG_ASSERT(new_scheds); 941 KMP_DEBUG_ASSERT(new_chunks); 942 if (!TCR_4(__kmp_init_parallel)) 943 __kmp_parallel_initialize(); 944 __kmp_resume_if_soft_paused(); 945 946 th = __kmp_threads[gtid]; 947 team = th->th.th_team; 948 active = !team->t.t_serialized; 949 th->th.th_ident = loc; 950 num_hw_threads = __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1]; 951 KMP_DEBUG_ASSERT(th->th.th_dispatch == 952 &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid]); 953 my_buffer_index = th->th.th_dispatch->th_disp_index; 954 pr = reinterpret_cast<dispatch_private_info_template<T> *>( 955 &th->th.th_dispatch 956 ->th_disp_buffer[my_buffer_index % __kmp_dispatch_num_buffers]); 957 sh = reinterpret_cast<dispatch_shared_info_template<T> volatile *>( 958 &team->t.t_disp_buffer[my_buffer_index % __kmp_dispatch_num_buffers]); 959 if (!active) { 960 KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d not active parallel. " 961 "Using normal dispatch functions.\n", 962 gtid)); 963 KMP_DEBUG_ASSERT(pr); 964 pr->flags.use_hier = FALSE; 965 pr->flags.contains_last = FALSE; 966 return; 967 } 968 KMP_DEBUG_ASSERT(pr); 969 KMP_DEBUG_ASSERT(sh); 970 pr->flags.use_hier = TRUE; 971 pr->u.p.tc = 0; 972 // Have master allocate the hierarchy 973 if (__kmp_tid_from_gtid(gtid) == 0) { 974 KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d pr:%p sh:%p allocating " 975 "hierarchy\n", 976 gtid, pr, sh)); 977 if (sh->hier == NULL) { 978 sh->hier = (kmp_hier_t<T> *)__kmp_allocate(sizeof(kmp_hier_t<T>)); 979 } 980 sh->hier->allocate_hier(n, new_layers, new_scheds, new_chunks); 981 sh->u.s.iteration = 0; 982 } 983 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 984 // Check to make sure the hierarchy is valid 985 kmp_hier_t<T> *hier = sh->hier; 986 if (!sh->hier->is_valid()) { 987 pr->flags.use_hier = FALSE; 988 return; 989 } 990 // Have threads allocate their thread-private barrier data if it hasn't 991 // already been allocated 992 if (th->th.th_hier_bar_data == NULL) { 993 th->th.th_hier_bar_data = (kmp_hier_private_bdata_t *)__kmp_allocate( 994 sizeof(kmp_hier_private_bdata_t) * kmp_hier_layer_e::LAYER_LAST); 995 } 996 // Have threads "register" themselves by modifiying the active count for each 997 // level they are involved in. The active count will act as nthreads for that 998 // level regarding the scheduling algorithms 999 for (int i = 0; i < n; ++i) { 1000 int index = __kmp_dispatch_get_index(tid, hier->get_type(i)); 1001 kmp_hier_top_unit_t<T> *my_unit = hier->get_unit(i, index); 1002 // Setup the thread's private dispatch buffer's hierarchy pointers 1003 if (i == 0) 1004 pr->hier_parent = my_unit; 1005 // If this unit is already active, then increment active count and wait 1006 if (my_unit->is_active()) { 1007 KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d my_unit (%p) " 1008 "is already active (%d)\n", 1009 gtid, my_unit, my_unit->active)); 1010 KMP_TEST_THEN_INC32(&(my_unit->active)); 1011 break; 1012 } 1013 // Flag that this unit is active 1014 if (KMP_COMPARE_AND_STORE_ACQ32(&(my_unit->active), 0, 1)) { 1015 // Do not setup parent pointer for top level unit since it has no parent 1016 if (i < n - 1) { 1017 // Setup middle layer pointers to parents 1018 my_unit->get_my_pr()->hier_id = 1019 index % __kmp_dispatch_get_t1_per_t2(hier->get_type(i), 1020 hier->get_type(i + 1)); 1021 int parent_index = __kmp_dispatch_get_index(tid, hier->get_type(i + 1)); 1022 my_unit->hier_parent = hier->get_unit(i + 1, parent_index); 1023 } else { 1024 // Setup top layer information (no parent pointers are set) 1025 my_unit->get_my_pr()->hier_id = 1026 index % __kmp_dispatch_get_t1_per_t2(hier->get_type(i), 1027 kmp_hier_layer_e::LAYER_LOOP); 1028 KMP_TEST_THEN_INC32(&(hier->top_level_nproc)); 1029 my_unit->hier_parent = nullptr; 1030 } 1031 // Set trip count to 0 so that next() operation will initially climb up 1032 // the hierarchy to get more iterations (early exit in next() for tc == 0) 1033 my_unit->get_my_pr()->u.p.tc = 0; 1034 // Increment this layer's number of active units 1035 KMP_TEST_THEN_INC32(&(hier->info[i].num_active)); 1036 KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d my_unit (%p) " 1037 "incrementing num_active\n", 1038 gtid, my_unit)); 1039 } else { 1040 KMP_TEST_THEN_INC32(&(my_unit->active)); 1041 break; 1042 } 1043 } 1044 // Set this thread's id 1045 num_threads_per_layer1 = __kmp_dispatch_get_t1_per_t2( 1046 kmp_hier_layer_e::LAYER_THREAD, hier->get_type(0)); 1047 pr->hier_id = tid % num_threads_per_layer1; 1048 // For oversubscribed threads, increment their index within the lowest unit 1049 // This is done to prevent having two or more threads with id 0, id 1, etc. 1050 if (tid >= num_hw_threads) 1051 pr->hier_id += ((tid / num_hw_threads) * num_threads_per_layer1); 1052 KD_TRACE( 1053 10, ("__kmp_dispatch_init_hierarchy: T#%d setting lowest hier_id to %d\n", 1054 gtid, pr->hier_id)); 1055 1056 pr->flags.contains_last = FALSE; 1057 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 1058 1059 // Now that the number of active threads at each level is determined, 1060 // the barrier data for each unit can be initialized and the last layer's 1061 // loop information can be initialized. 1062 int prev_id = pr->get_hier_id(); 1063 for (int i = 0; i < n; ++i) { 1064 if (prev_id != 0) 1065 break; 1066 int index = __kmp_dispatch_get_index(tid, hier->get_type(i)); 1067 kmp_hier_top_unit_t<T> *my_unit = hier->get_unit(i, index); 1068 // Only master threads of this unit within the hierarchy do initialization 1069 KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d (%d) prev_id is 0\n", 1070 gtid, i)); 1071 my_unit->reset_shared_barrier(); 1072 my_unit->hier_pr.flags.contains_last = FALSE; 1073 // Last layer, initialize the private buffers with entire loop information 1074 // Now the next next_algorithim() call will get the first chunk of 1075 // iterations properly 1076 if (i == n - 1) { 1077 __kmp_dispatch_init_algorithm<T>( 1078 loc, gtid, my_unit->get_my_pr(), hier->get_sched(i), lb, ub, st, 1079 #if USE_ITT_BUILD 1080 NULL, 1081 #endif 1082 hier->get_chunk(i), hier->get_num_active(i), my_unit->get_hier_id()); 1083 } 1084 prev_id = my_unit->get_hier_id(); 1085 } 1086 // Initialize each layer of the thread's private barrier data 1087 kmp_hier_top_unit_t<T> *unit = pr->hier_parent; 1088 for (int i = 0; i < n && unit; ++i, unit = unit->get_parent()) { 1089 kmp_hier_private_bdata_t *tdata = &(th->th.th_hier_bar_data[i]); 1090 unit->reset_private_barrier(tdata); 1091 } 1092 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 1093 1094 #ifdef KMP_DEBUG 1095 if (__kmp_tid_from_gtid(gtid) == 0) { 1096 for (int i = 0; i < n; ++i) { 1097 KD_TRACE(10, 1098 ("__kmp_dispatch_init_hierarchy: T#%d active count[%d] = %d\n", 1099 gtid, i, hier->get_num_active(i))); 1100 } 1101 hier->print(); 1102 } 1103 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 1104 #endif // KMP_DEBUG 1105 } 1106 #endif 1107