1 /*
2 * kmp_affinity.cpp -- affinity management
3 */
4
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_str.h"
18 #include "kmp_wrapper_getpid.h"
19 #if KMP_USE_HIER_SCHED
20 #include "kmp_dispatch_hier.h"
21 #endif
22
23 // Store the real or imagined machine hierarchy here
24 static hierarchy_info machine_hierarchy;
25
__kmp_cleanup_hierarchy()26 void __kmp_cleanup_hierarchy() { machine_hierarchy.fini(); }
27
__kmp_get_hierarchy(kmp_uint32 nproc,kmp_bstate_t * thr_bar)28 void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar) {
29 kmp_uint32 depth;
30 // The test below is true if affinity is available, but set to "none". Need to
31 // init on first use of hierarchical barrier.
32 if (TCR_1(machine_hierarchy.uninitialized))
33 machine_hierarchy.init(NULL, nproc);
34
35 // Adjust the hierarchy in case num threads exceeds original
36 if (nproc > machine_hierarchy.base_num_threads)
37 machine_hierarchy.resize(nproc);
38
39 depth = machine_hierarchy.depth;
40 KMP_DEBUG_ASSERT(depth > 0);
41
42 thr_bar->depth = depth;
43 thr_bar->base_leaf_kids = (kmp_uint8)machine_hierarchy.numPerLevel[0] - 1;
44 thr_bar->skip_per_level = machine_hierarchy.skipPerLevel;
45 }
46
47 #if KMP_AFFINITY_SUPPORTED
48
49 bool KMPAffinity::picked_api = false;
50
operator new(size_t n)51 void *KMPAffinity::Mask::operator new(size_t n) { return __kmp_allocate(n); }
operator new[](size_t n)52 void *KMPAffinity::Mask::operator new[](size_t n) { return __kmp_allocate(n); }
operator delete(void * p)53 void KMPAffinity::Mask::operator delete(void *p) { __kmp_free(p); }
operator delete[](void * p)54 void KMPAffinity::Mask::operator delete[](void *p) { __kmp_free(p); }
operator new(size_t n)55 void *KMPAffinity::operator new(size_t n) { return __kmp_allocate(n); }
operator delete(void * p)56 void KMPAffinity::operator delete(void *p) { __kmp_free(p); }
57
pick_api()58 void KMPAffinity::pick_api() {
59 KMPAffinity *affinity_dispatch;
60 if (picked_api)
61 return;
62 #if KMP_USE_HWLOC
63 // Only use Hwloc if affinity isn't explicitly disabled and
64 // user requests Hwloc topology method
65 if (__kmp_affinity_top_method == affinity_top_method_hwloc &&
66 __kmp_affinity_type != affinity_disabled) {
67 affinity_dispatch = new KMPHwlocAffinity();
68 } else
69 #endif
70 {
71 affinity_dispatch = new KMPNativeAffinity();
72 }
73 __kmp_affinity_dispatch = affinity_dispatch;
74 picked_api = true;
75 }
76
destroy_api()77 void KMPAffinity::destroy_api() {
78 if (__kmp_affinity_dispatch != NULL) {
79 delete __kmp_affinity_dispatch;
80 __kmp_affinity_dispatch = NULL;
81 picked_api = false;
82 }
83 }
84
85 #define KMP_ADVANCE_SCAN(scan) \
86 while (*scan != '\0') { \
87 scan++; \
88 }
89
90 // Print the affinity mask to the character array in a pretty format.
91 // The format is a comma separated list of non-negative integers or integer
92 // ranges: e.g., 1,2,3-5,7,9-15
93 // The format can also be the string "{<empty>}" if no bits are set in mask
__kmp_affinity_print_mask(char * buf,int buf_len,kmp_affin_mask_t * mask)94 char *__kmp_affinity_print_mask(char *buf, int buf_len,
95 kmp_affin_mask_t *mask) {
96 int start = 0, finish = 0, previous = 0;
97 bool first_range;
98 KMP_ASSERT(buf);
99 KMP_ASSERT(buf_len >= 40);
100 KMP_ASSERT(mask);
101 char *scan = buf;
102 char *end = buf + buf_len - 1;
103
104 // Check for empty set.
105 if (mask->begin() == mask->end()) {
106 KMP_SNPRINTF(scan, end - scan + 1, "{<empty>}");
107 KMP_ADVANCE_SCAN(scan);
108 KMP_ASSERT(scan <= end);
109 return buf;
110 }
111
112 first_range = true;
113 start = mask->begin();
114 while (1) {
115 // Find next range
116 // [start, previous] is inclusive range of contiguous bits in mask
117 for (finish = mask->next(start), previous = start;
118 finish == previous + 1 && finish != mask->end();
119 finish = mask->next(finish)) {
120 previous = finish;
121 }
122
123 // The first range does not need a comma printed before it, but the rest
124 // of the ranges do need a comma beforehand
125 if (!first_range) {
126 KMP_SNPRINTF(scan, end - scan + 1, "%s", ",");
127 KMP_ADVANCE_SCAN(scan);
128 } else {
129 first_range = false;
130 }
131 // Range with three or more contiguous bits in the affinity mask
132 if (previous - start > 1) {
133 KMP_SNPRINTF(scan, end - scan + 1, "%d-%d", static_cast<int>(start),
134 static_cast<int>(previous));
135 } else {
136 // Range with one or two contiguous bits in the affinity mask
137 KMP_SNPRINTF(scan, end - scan + 1, "%d", static_cast<int>(start));
138 KMP_ADVANCE_SCAN(scan);
139 if (previous - start > 0) {
140 KMP_SNPRINTF(scan, end - scan + 1, ",%d", static_cast<int>(previous));
141 }
142 }
143 KMP_ADVANCE_SCAN(scan);
144 // Start over with new start point
145 start = finish;
146 if (start == mask->end())
147 break;
148 // Check for overflow
149 if (end - scan < 2)
150 break;
151 }
152
153 // Check for overflow
154 KMP_ASSERT(scan <= end);
155 return buf;
156 }
157 #undef KMP_ADVANCE_SCAN
158
159 // Print the affinity mask to the string buffer object in a pretty format
160 // The format is a comma separated list of non-negative integers or integer
161 // ranges: e.g., 1,2,3-5,7,9-15
162 // The format can also be the string "{<empty>}" if no bits are set in mask
__kmp_affinity_str_buf_mask(kmp_str_buf_t * buf,kmp_affin_mask_t * mask)163 kmp_str_buf_t *__kmp_affinity_str_buf_mask(kmp_str_buf_t *buf,
164 kmp_affin_mask_t *mask) {
165 int start = 0, finish = 0, previous = 0;
166 bool first_range;
167 KMP_ASSERT(buf);
168 KMP_ASSERT(mask);
169
170 __kmp_str_buf_clear(buf);
171
172 // Check for empty set.
173 if (mask->begin() == mask->end()) {
174 __kmp_str_buf_print(buf, "%s", "{<empty>}");
175 return buf;
176 }
177
178 first_range = true;
179 start = mask->begin();
180 while (1) {
181 // Find next range
182 // [start, previous] is inclusive range of contiguous bits in mask
183 for (finish = mask->next(start), previous = start;
184 finish == previous + 1 && finish != mask->end();
185 finish = mask->next(finish)) {
186 previous = finish;
187 }
188
189 // The first range does not need a comma printed before it, but the rest
190 // of the ranges do need a comma beforehand
191 if (!first_range) {
192 __kmp_str_buf_print(buf, "%s", ",");
193 } else {
194 first_range = false;
195 }
196 // Range with three or more contiguous bits in the affinity mask
197 if (previous - start > 1) {
198 __kmp_str_buf_print(buf, "%d-%d", static_cast<int>(start),
199 static_cast<int>(previous));
200 } else {
201 // Range with one or two contiguous bits in the affinity mask
202 __kmp_str_buf_print(buf, "%d", static_cast<int>(start));
203 if (previous - start > 0) {
204 __kmp_str_buf_print(buf, ",%d", static_cast<int>(previous));
205 }
206 }
207 // Start over with new start point
208 start = finish;
209 if (start == mask->end())
210 break;
211 }
212 return buf;
213 }
214
__kmp_affinity_entire_machine_mask(kmp_affin_mask_t * mask)215 void __kmp_affinity_entire_machine_mask(kmp_affin_mask_t *mask) {
216 KMP_CPU_ZERO(mask);
217
218 #if KMP_GROUP_AFFINITY
219
220 if (__kmp_num_proc_groups > 1) {
221 int group;
222 KMP_DEBUG_ASSERT(__kmp_GetActiveProcessorCount != NULL);
223 for (group = 0; group < __kmp_num_proc_groups; group++) {
224 int i;
225 int num = __kmp_GetActiveProcessorCount(group);
226 for (i = 0; i < num; i++) {
227 KMP_CPU_SET(i + group * (CHAR_BIT * sizeof(DWORD_PTR)), mask);
228 }
229 }
230 } else
231
232 #endif /* KMP_GROUP_AFFINITY */
233
234 {
235 int proc;
236 for (proc = 0; proc < __kmp_xproc; proc++) {
237 KMP_CPU_SET(proc, mask);
238 }
239 }
240 }
241
242 // When sorting by labels, __kmp_affinity_assign_child_nums() must first be
243 // called to renumber the labels from [0..n] and place them into the child_num
244 // vector of the address object. This is done in case the labels used for
245 // the children at one node of the hierarchy differ from those used for
246 // another node at the same level. Example: suppose the machine has 2 nodes
247 // with 2 packages each. The first node contains packages 601 and 602, and
248 // second node contains packages 603 and 604. If we try to sort the table
249 // for "scatter" affinity, the table will still be sorted 601, 602, 603, 604
250 // because we are paying attention to the labels themselves, not the ordinal
251 // child numbers. By using the child numbers in the sort, the result is
252 // {0,0}=601, {0,1}=603, {1,0}=602, {1,1}=604.
__kmp_affinity_assign_child_nums(AddrUnsPair * address2os,int numAddrs)253 static void __kmp_affinity_assign_child_nums(AddrUnsPair *address2os,
254 int numAddrs) {
255 KMP_DEBUG_ASSERT(numAddrs > 0);
256 int depth = address2os->first.depth;
257 unsigned *counts = (unsigned *)__kmp_allocate(depth * sizeof(unsigned));
258 unsigned *lastLabel = (unsigned *)__kmp_allocate(depth * sizeof(unsigned));
259 int labCt;
260 for (labCt = 0; labCt < depth; labCt++) {
261 address2os[0].first.childNums[labCt] = counts[labCt] = 0;
262 lastLabel[labCt] = address2os[0].first.labels[labCt];
263 }
264 int i;
265 for (i = 1; i < numAddrs; i++) {
266 for (labCt = 0; labCt < depth; labCt++) {
267 if (address2os[i].first.labels[labCt] != lastLabel[labCt]) {
268 int labCt2;
269 for (labCt2 = labCt + 1; labCt2 < depth; labCt2++) {
270 counts[labCt2] = 0;
271 lastLabel[labCt2] = address2os[i].first.labels[labCt2];
272 }
273 counts[labCt]++;
274 lastLabel[labCt] = address2os[i].first.labels[labCt];
275 break;
276 }
277 }
278 for (labCt = 0; labCt < depth; labCt++) {
279 address2os[i].first.childNums[labCt] = counts[labCt];
280 }
281 for (; labCt < (int)Address::maxDepth; labCt++) {
282 address2os[i].first.childNums[labCt] = 0;
283 }
284 }
285 __kmp_free(lastLabel);
286 __kmp_free(counts);
287 }
288
289 // All of the __kmp_affinity_create_*_map() routines should set
290 // __kmp_affinity_masks to a vector of affinity mask objects of length
291 // __kmp_affinity_num_masks, if __kmp_affinity_type != affinity_none, and return
292 // the number of levels in the machine topology tree (zero if
293 // __kmp_affinity_type == affinity_none).
294 //
295 // All of the __kmp_affinity_create_*_map() routines should set
296 // *__kmp_affin_fullMask to the affinity mask for the initialization thread.
297 // They need to save and restore the mask, and it could be needed later, so
298 // saving it is just an optimization to avoid calling kmp_get_system_affinity()
299 // again.
300 kmp_affin_mask_t *__kmp_affin_fullMask = NULL;
301
302 static int nCoresPerPkg, nPackages;
303 static int __kmp_nThreadsPerCore;
304 #ifndef KMP_DFLT_NTH_CORES
305 static int __kmp_ncores;
306 #endif
307 static int *__kmp_pu_os_idx = NULL;
308
309 // __kmp_affinity_uniform_topology() doesn't work when called from
310 // places which support arbitrarily many levels in the machine topology
311 // map, i.e. the non-default cases in __kmp_affinity_create_cpuinfo_map()
312 // __kmp_affinity_create_x2apicid_map().
__kmp_affinity_uniform_topology()313 inline static bool __kmp_affinity_uniform_topology() {
314 return __kmp_avail_proc == (__kmp_nThreadsPerCore * nCoresPerPkg * nPackages);
315 }
316
317 // Print out the detailed machine topology map, i.e. the physical locations
318 // of each OS proc.
__kmp_affinity_print_topology(AddrUnsPair * address2os,int len,int depth,int pkgLevel,int coreLevel,int threadLevel)319 static void __kmp_affinity_print_topology(AddrUnsPair *address2os, int len,
320 int depth, int pkgLevel,
321 int coreLevel, int threadLevel) {
322 int proc;
323
324 KMP_INFORM(OSProcToPhysicalThreadMap, "KMP_AFFINITY");
325 for (proc = 0; proc < len; proc++) {
326 int level;
327 kmp_str_buf_t buf;
328 __kmp_str_buf_init(&buf);
329 for (level = 0; level < depth; level++) {
330 if (level == threadLevel) {
331 __kmp_str_buf_print(&buf, "%s ", KMP_I18N_STR(Thread));
332 } else if (level == coreLevel) {
333 __kmp_str_buf_print(&buf, "%s ", KMP_I18N_STR(Core));
334 } else if (level == pkgLevel) {
335 __kmp_str_buf_print(&buf, "%s ", KMP_I18N_STR(Package));
336 } else if (level > pkgLevel) {
337 __kmp_str_buf_print(&buf, "%s_%d ", KMP_I18N_STR(Node),
338 level - pkgLevel - 1);
339 } else {
340 __kmp_str_buf_print(&buf, "L%d ", level);
341 }
342 __kmp_str_buf_print(&buf, "%d ", address2os[proc].first.labels[level]);
343 }
344 KMP_INFORM(OSProcMapToPack, "KMP_AFFINITY", address2os[proc].second,
345 buf.str);
346 __kmp_str_buf_free(&buf);
347 }
348 }
349
350 #if KMP_USE_HWLOC
351
__kmp_affinity_print_hwloc_tp(AddrUnsPair * addrP,int len,int depth,int * levels)352 static void __kmp_affinity_print_hwloc_tp(AddrUnsPair *addrP, int len,
353 int depth, int *levels) {
354 int proc;
355 kmp_str_buf_t buf;
356 __kmp_str_buf_init(&buf);
357 KMP_INFORM(OSProcToPhysicalThreadMap, "KMP_AFFINITY");
358 for (proc = 0; proc < len; proc++) {
359 __kmp_str_buf_print(&buf, "%s %d ", KMP_I18N_STR(Package),
360 addrP[proc].first.labels[0]);
361 if (depth > 1) {
362 int level = 1; // iterate over levels
363 int label = 1; // iterate over labels
364 if (__kmp_numa_detected)
365 // node level follows package
366 if (levels[level++] > 0)
367 __kmp_str_buf_print(&buf, "%s %d ", KMP_I18N_STR(Node),
368 addrP[proc].first.labels[label++]);
369 if (__kmp_tile_depth > 0)
370 // tile level follows node if any, or package
371 if (levels[level++] > 0)
372 __kmp_str_buf_print(&buf, "%s %d ", KMP_I18N_STR(Tile),
373 addrP[proc].first.labels[label++]);
374 if (levels[level++] > 0)
375 // core level follows
376 __kmp_str_buf_print(&buf, "%s %d ", KMP_I18N_STR(Core),
377 addrP[proc].first.labels[label++]);
378 if (levels[level++] > 0)
379 // thread level is the latest
380 __kmp_str_buf_print(&buf, "%s %d ", KMP_I18N_STR(Thread),
381 addrP[proc].first.labels[label++]);
382 KMP_DEBUG_ASSERT(label == depth);
383 }
384 KMP_INFORM(OSProcMapToPack, "KMP_AFFINITY", addrP[proc].second, buf.str);
385 __kmp_str_buf_clear(&buf);
386 }
387 __kmp_str_buf_free(&buf);
388 }
389
390 static int nNodePerPkg, nTilePerPkg, nTilePerNode, nCorePerNode, nCorePerTile;
391
392 // This function removes the topology levels that are radix 1 and don't offer
393 // further information about the topology. The most common example is when you
394 // have one thread context per core, we don't want the extra thread context
395 // level if it offers no unique labels. So they are removed.
396 // return value: the new depth of address2os
__kmp_affinity_remove_radix_one_levels(AddrUnsPair * addrP,int nTh,int depth,int * levels)397 static int __kmp_affinity_remove_radix_one_levels(AddrUnsPair *addrP, int nTh,
398 int depth, int *levels) {
399 int level;
400 int i;
401 int radix1_detected;
402 int new_depth = depth;
403 for (level = depth - 1; level > 0; --level) {
404 // Detect if this level is radix 1
405 radix1_detected = 1;
406 for (i = 1; i < nTh; ++i) {
407 if (addrP[0].first.labels[level] != addrP[i].first.labels[level]) {
408 // There are differing label values for this level so it stays
409 radix1_detected = 0;
410 break;
411 }
412 }
413 if (!radix1_detected)
414 continue;
415 // Radix 1 was detected
416 --new_depth;
417 levels[level] = -1; // mark level as not present in address2os array
418 if (level == new_depth) {
419 // "turn off" deepest level, just decrement the depth that removes
420 // the level from address2os array
421 for (i = 0; i < nTh; ++i) {
422 addrP[i].first.depth--;
423 }
424 } else {
425 // For other levels, we move labels over and also reduce the depth
426 int j;
427 for (j = level; j < new_depth; ++j) {
428 for (i = 0; i < nTh; ++i) {
429 addrP[i].first.labels[j] = addrP[i].first.labels[j + 1];
430 addrP[i].first.depth--;
431 }
432 levels[j + 1] -= 1;
433 }
434 }
435 }
436 return new_depth;
437 }
438
439 // Returns the number of objects of type 'type' below 'obj' within the topology
440 // tree structure. e.g., if obj is a HWLOC_OBJ_PACKAGE object, and type is
441 // HWLOC_OBJ_PU, then this will return the number of PU's under the SOCKET
442 // object.
__kmp_hwloc_get_nobjs_under_obj(hwloc_obj_t obj,hwloc_obj_type_t type)443 static int __kmp_hwloc_get_nobjs_under_obj(hwloc_obj_t obj,
444 hwloc_obj_type_t type) {
445 int retval = 0;
446 hwloc_obj_t first;
447 for (first = hwloc_get_obj_below_by_type(__kmp_hwloc_topology, obj->type,
448 obj->logical_index, type, 0);
449 first != NULL &&
450 hwloc_get_ancestor_obj_by_type(__kmp_hwloc_topology, obj->type, first) ==
451 obj;
452 first = hwloc_get_next_obj_by_type(__kmp_hwloc_topology, first->type,
453 first)) {
454 ++retval;
455 }
456 return retval;
457 }
458
__kmp_hwloc_count_children_by_depth(hwloc_topology_t t,hwloc_obj_t o,kmp_hwloc_depth_t depth,hwloc_obj_t * f)459 static int __kmp_hwloc_count_children_by_depth(hwloc_topology_t t,
460 hwloc_obj_t o,
461 kmp_hwloc_depth_t depth,
462 hwloc_obj_t *f) {
463 if (o->depth == depth) {
464 if (*f == NULL)
465 *f = o; // output first descendant found
466 return 1;
467 }
468 int sum = 0;
469 for (unsigned i = 0; i < o->arity; i++)
470 sum += __kmp_hwloc_count_children_by_depth(t, o->children[i], depth, f);
471 return sum; // will be 0 if no one found (as PU arity is 0)
472 }
473
__kmp_hwloc_count_children_by_type(hwloc_topology_t t,hwloc_obj_t o,hwloc_obj_type_t type,hwloc_obj_t * f)474 static int __kmp_hwloc_count_children_by_type(hwloc_topology_t t, hwloc_obj_t o,
475 hwloc_obj_type_t type,
476 hwloc_obj_t *f) {
477 if (!hwloc_compare_types(o->type, type)) {
478 if (*f == NULL)
479 *f = o; // output first descendant found
480 return 1;
481 }
482 int sum = 0;
483 for (unsigned i = 0; i < o->arity; i++)
484 sum += __kmp_hwloc_count_children_by_type(t, o->children[i], type, f);
485 return sum; // will be 0 if no one found (as PU arity is 0)
486 }
487
__kmp_hwloc_process_obj_core_pu(AddrUnsPair * addrPair,int & nActiveThreads,int & num_active_cores,hwloc_obj_t obj,int depth,int * labels)488 static int __kmp_hwloc_process_obj_core_pu(AddrUnsPair *addrPair,
489 int &nActiveThreads,
490 int &num_active_cores,
491 hwloc_obj_t obj, int depth,
492 int *labels) {
493 hwloc_obj_t core = NULL;
494 hwloc_topology_t &tp = __kmp_hwloc_topology;
495 int NC = __kmp_hwloc_count_children_by_type(tp, obj, HWLOC_OBJ_CORE, &core);
496 for (int core_id = 0; core_id < NC; ++core_id, core = core->next_cousin) {
497 hwloc_obj_t pu = NULL;
498 KMP_DEBUG_ASSERT(core != NULL);
499 int num_active_threads = 0;
500 int NT = __kmp_hwloc_count_children_by_type(tp, core, HWLOC_OBJ_PU, &pu);
501 // int NT = core->arity; pu = core->first_child; // faster?
502 for (int pu_id = 0; pu_id < NT; ++pu_id, pu = pu->next_cousin) {
503 KMP_DEBUG_ASSERT(pu != NULL);
504 if (!KMP_CPU_ISSET(pu->os_index, __kmp_affin_fullMask))
505 continue; // skip inactive (inaccessible) unit
506 Address addr(depth + 2);
507 KA_TRACE(20, ("Hwloc inserting %d (%d) %d (%d) %d (%d) into address2os\n",
508 obj->os_index, obj->logical_index, core->os_index,
509 core->logical_index, pu->os_index, pu->logical_index));
510 for (int i = 0; i < depth; ++i)
511 addr.labels[i] = labels[i]; // package, etc.
512 addr.labels[depth] = core_id; // core
513 addr.labels[depth + 1] = pu_id; // pu
514 addrPair[nActiveThreads] = AddrUnsPair(addr, pu->os_index);
515 __kmp_pu_os_idx[nActiveThreads] = pu->os_index;
516 nActiveThreads++;
517 ++num_active_threads; // count active threads per core
518 }
519 if (num_active_threads) { // were there any active threads on the core?
520 ++__kmp_ncores; // count total active cores
521 ++num_active_cores; // count active cores per socket
522 if (num_active_threads > __kmp_nThreadsPerCore)
523 __kmp_nThreadsPerCore = num_active_threads; // calc maximum
524 }
525 }
526 return 0;
527 }
528
529 // Check if NUMA node detected below the package,
530 // and if tile object is detected and return its depth
__kmp_hwloc_check_numa()531 static int __kmp_hwloc_check_numa() {
532 hwloc_topology_t &tp = __kmp_hwloc_topology;
533 hwloc_obj_t hT, hC, hL, hN, hS; // hwloc objects (pointers to)
534 int depth, l2cache_depth, package_depth;
535
536 // Get some PU
537 hT = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PU, 0);
538 if (hT == NULL) // something has gone wrong
539 return 1;
540
541 // check NUMA node below PACKAGE
542 hN = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hT);
543 hS = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hT);
544 KMP_DEBUG_ASSERT(hS != NULL);
545 if (hN != NULL && hN->depth > hS->depth) {
546 __kmp_numa_detected = TRUE; // socket includes node(s)
547 if (__kmp_affinity_gran == affinity_gran_node) {
548 __kmp_affinity_gran = affinity_gran_numa;
549 }
550 }
551
552 package_depth = hwloc_get_type_depth(tp, HWLOC_OBJ_PACKAGE);
553 l2cache_depth = hwloc_get_cache_type_depth(tp, 2, HWLOC_OBJ_CACHE_UNIFIED);
554 // check tile, get object by depth because of multiple caches possible
555 depth = (l2cache_depth < package_depth) ? package_depth : l2cache_depth;
556 hL = hwloc_get_ancestor_obj_by_depth(tp, depth, hT);
557 hC = NULL; // not used, but reset it here just in case
558 if (hL != NULL &&
559 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC) > 1)
560 __kmp_tile_depth = depth; // tile consists of multiple cores
561 return 0;
562 }
563
__kmp_affinity_create_hwloc_map(AddrUnsPair ** address2os,kmp_i18n_id_t * const msg_id)564 static int __kmp_affinity_create_hwloc_map(AddrUnsPair **address2os,
565 kmp_i18n_id_t *const msg_id) {
566 hwloc_topology_t &tp = __kmp_hwloc_topology; // shortcut of a long name
567 *address2os = NULL;
568 *msg_id = kmp_i18n_null;
569
570 // Save the affinity mask for the current thread.
571 kmp_affin_mask_t *oldMask;
572 KMP_CPU_ALLOC(oldMask);
573 __kmp_get_system_affinity(oldMask, TRUE);
574 __kmp_hwloc_check_numa();
575
576 if (!KMP_AFFINITY_CAPABLE()) {
577 // Hack to try and infer the machine topology using only the data
578 // available from cpuid on the current thread, and __kmp_xproc.
579 KMP_ASSERT(__kmp_affinity_type == affinity_none);
580
581 nCoresPerPkg = __kmp_hwloc_get_nobjs_under_obj(
582 hwloc_get_obj_by_type(tp, HWLOC_OBJ_PACKAGE, 0), HWLOC_OBJ_CORE);
583 __kmp_nThreadsPerCore = __kmp_hwloc_get_nobjs_under_obj(
584 hwloc_get_obj_by_type(tp, HWLOC_OBJ_CORE, 0), HWLOC_OBJ_PU);
585 __kmp_ncores = __kmp_xproc / __kmp_nThreadsPerCore;
586 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
587 if (__kmp_affinity_verbose) {
588 KMP_INFORM(AffNotCapableUseLocCpuidL11, "KMP_AFFINITY");
589 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
590 if (__kmp_affinity_uniform_topology()) {
591 KMP_INFORM(Uniform, "KMP_AFFINITY");
592 } else {
593 KMP_INFORM(NonUniform, "KMP_AFFINITY");
594 }
595 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
596 __kmp_nThreadsPerCore, __kmp_ncores);
597 }
598 KMP_CPU_FREE(oldMask);
599 return 0;
600 }
601
602 int depth = 3;
603 int levels[5] = {0, 1, 2, 3, 4}; // package, [node,] [tile,] core, thread
604 int labels[3] = {0}; // package [,node] [,tile] - head of lables array
605 if (__kmp_numa_detected)
606 ++depth;
607 if (__kmp_tile_depth)
608 ++depth;
609
610 // Allocate the data structure to be returned.
611 AddrUnsPair *retval =
612 (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair) * __kmp_avail_proc);
613 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
614 __kmp_pu_os_idx = (int *)__kmp_allocate(sizeof(int) * __kmp_avail_proc);
615
616 // When affinity is off, this routine will still be called to set
617 // __kmp_ncores, as well as __kmp_nThreadsPerCore,
618 // nCoresPerPkg, & nPackages. Make sure all these vars are set
619 // correctly, and return if affinity is not enabled.
620
621 hwloc_obj_t socket, node, tile;
622 int nActiveThreads = 0;
623 int socket_id = 0;
624 // re-calculate globals to count only accessible resources
625 __kmp_ncores = nPackages = nCoresPerPkg = __kmp_nThreadsPerCore = 0;
626 nNodePerPkg = nTilePerPkg = nTilePerNode = nCorePerNode = nCorePerTile = 0;
627 for (socket = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PACKAGE, 0); socket != NULL;
628 socket = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PACKAGE, socket),
629 socket_id++) {
630 labels[0] = socket_id;
631 if (__kmp_numa_detected) {
632 int NN;
633 int n_active_nodes = 0;
634 node = NULL;
635 NN = __kmp_hwloc_count_children_by_type(tp, socket, HWLOC_OBJ_NUMANODE,
636 &node);
637 for (int node_id = 0; node_id < NN; ++node_id, node = node->next_cousin) {
638 labels[1] = node_id;
639 if (__kmp_tile_depth) {
640 // NUMA + tiles
641 int NT;
642 int n_active_tiles = 0;
643 tile = NULL;
644 NT = __kmp_hwloc_count_children_by_depth(tp, node, __kmp_tile_depth,
645 &tile);
646 for (int tl_id = 0; tl_id < NT; ++tl_id, tile = tile->next_cousin) {
647 labels[2] = tl_id;
648 int n_active_cores = 0;
649 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
650 n_active_cores, tile, 3, labels);
651 if (n_active_cores) { // were there any active cores on the socket?
652 ++n_active_tiles; // count active tiles per node
653 if (n_active_cores > nCorePerTile)
654 nCorePerTile = n_active_cores; // calc maximum
655 }
656 }
657 if (n_active_tiles) { // were there any active tiles on the socket?
658 ++n_active_nodes; // count active nodes per package
659 if (n_active_tiles > nTilePerNode)
660 nTilePerNode = n_active_tiles; // calc maximum
661 }
662 } else {
663 // NUMA, no tiles
664 int n_active_cores = 0;
665 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
666 n_active_cores, node, 2, labels);
667 if (n_active_cores) { // were there any active cores on the socket?
668 ++n_active_nodes; // count active nodes per package
669 if (n_active_cores > nCorePerNode)
670 nCorePerNode = n_active_cores; // calc maximum
671 }
672 }
673 }
674 if (n_active_nodes) { // were there any active nodes on the socket?
675 ++nPackages; // count total active packages
676 if (n_active_nodes > nNodePerPkg)
677 nNodePerPkg = n_active_nodes; // calc maximum
678 }
679 } else {
680 if (__kmp_tile_depth) {
681 // no NUMA, tiles
682 int NT;
683 int n_active_tiles = 0;
684 tile = NULL;
685 NT = __kmp_hwloc_count_children_by_depth(tp, socket, __kmp_tile_depth,
686 &tile);
687 for (int tl_id = 0; tl_id < NT; ++tl_id, tile = tile->next_cousin) {
688 labels[1] = tl_id;
689 int n_active_cores = 0;
690 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
691 n_active_cores, tile, 2, labels);
692 if (n_active_cores) { // were there any active cores on the socket?
693 ++n_active_tiles; // count active tiles per package
694 if (n_active_cores > nCorePerTile)
695 nCorePerTile = n_active_cores; // calc maximum
696 }
697 }
698 if (n_active_tiles) { // were there any active tiles on the socket?
699 ++nPackages; // count total active packages
700 if (n_active_tiles > nTilePerPkg)
701 nTilePerPkg = n_active_tiles; // calc maximum
702 }
703 } else {
704 // no NUMA, no tiles
705 int n_active_cores = 0;
706 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads, n_active_cores,
707 socket, 1, labels);
708 if (n_active_cores) { // were there any active cores on the socket?
709 ++nPackages; // count total active packages
710 if (n_active_cores > nCoresPerPkg)
711 nCoresPerPkg = n_active_cores; // calc maximum
712 }
713 }
714 }
715 }
716
717 // If there's only one thread context to bind to, return now.
718 KMP_DEBUG_ASSERT(nActiveThreads == __kmp_avail_proc);
719 KMP_ASSERT(nActiveThreads > 0);
720 if (nActiveThreads == 1) {
721 __kmp_ncores = nPackages = 1;
722 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
723 if (__kmp_affinity_verbose) {
724 char buf[KMP_AFFIN_MASK_PRINT_LEN];
725 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
726
727 KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY");
728 if (__kmp_affinity_respect_mask) {
729 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
730 } else {
731 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
732 }
733 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
734 KMP_INFORM(Uniform, "KMP_AFFINITY");
735 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
736 __kmp_nThreadsPerCore, __kmp_ncores);
737 }
738
739 if (__kmp_affinity_type == affinity_none) {
740 __kmp_free(retval);
741 KMP_CPU_FREE(oldMask);
742 return 0;
743 }
744
745 // Form an Address object which only includes the package level.
746 Address addr(1);
747 addr.labels[0] = retval[0].first.labels[0];
748 retval[0].first = addr;
749
750 if (__kmp_affinity_gran_levels < 0) {
751 __kmp_affinity_gran_levels = 0;
752 }
753
754 if (__kmp_affinity_verbose) {
755 __kmp_affinity_print_topology(retval, 1, 1, 0, -1, -1);
756 }
757
758 *address2os = retval;
759 KMP_CPU_FREE(oldMask);
760 return 1;
761 }
762
763 // Sort the table by physical Id.
764 qsort(retval, nActiveThreads, sizeof(*retval),
765 __kmp_affinity_cmp_Address_labels);
766
767 // Check to see if the machine topology is uniform
768 int nPUs = nPackages * __kmp_nThreadsPerCore;
769 if (__kmp_numa_detected) {
770 if (__kmp_tile_depth) { // NUMA + tiles
771 nPUs *= (nNodePerPkg * nTilePerNode * nCorePerTile);
772 } else { // NUMA, no tiles
773 nPUs *= (nNodePerPkg * nCorePerNode);
774 }
775 } else {
776 if (__kmp_tile_depth) { // no NUMA, tiles
777 nPUs *= (nTilePerPkg * nCorePerTile);
778 } else { // no NUMA, no tiles
779 nPUs *= nCoresPerPkg;
780 }
781 }
782 unsigned uniform = (nPUs == nActiveThreads);
783
784 // Print the machine topology summary.
785 if (__kmp_affinity_verbose) {
786 char mask[KMP_AFFIN_MASK_PRINT_LEN];
787 __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
788 if (__kmp_affinity_respect_mask) {
789 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", mask);
790 } else {
791 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", mask);
792 }
793 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
794 if (uniform) {
795 KMP_INFORM(Uniform, "KMP_AFFINITY");
796 } else {
797 KMP_INFORM(NonUniform, "KMP_AFFINITY");
798 }
799 if (__kmp_numa_detected) {
800 if (__kmp_tile_depth) { // NUMA + tiles
801 KMP_INFORM(TopologyExtraNoTi, "KMP_AFFINITY", nPackages, nNodePerPkg,
802 nTilePerNode, nCorePerTile, __kmp_nThreadsPerCore,
803 __kmp_ncores);
804 } else { // NUMA, no tiles
805 KMP_INFORM(TopologyExtraNode, "KMP_AFFINITY", nPackages, nNodePerPkg,
806 nCorePerNode, __kmp_nThreadsPerCore, __kmp_ncores);
807 nPUs *= (nNodePerPkg * nCorePerNode);
808 }
809 } else {
810 if (__kmp_tile_depth) { // no NUMA, tiles
811 KMP_INFORM(TopologyExtraTile, "KMP_AFFINITY", nPackages, nTilePerPkg,
812 nCorePerTile, __kmp_nThreadsPerCore, __kmp_ncores);
813 } else { // no NUMA, no tiles
814 kmp_str_buf_t buf;
815 __kmp_str_buf_init(&buf);
816 __kmp_str_buf_print(&buf, "%d", nPackages);
817 KMP_INFORM(TopologyExtra, "KMP_AFFINITY", buf.str, nCoresPerPkg,
818 __kmp_nThreadsPerCore, __kmp_ncores);
819 __kmp_str_buf_free(&buf);
820 }
821 }
822 }
823
824 if (__kmp_affinity_type == affinity_none) {
825 __kmp_free(retval);
826 KMP_CPU_FREE(oldMask);
827 return 0;
828 }
829
830 int depth_full = depth; // number of levels before compressing
831 // Find any levels with radiix 1, and remove them from the map
832 // (except for the package level).
833 depth = __kmp_affinity_remove_radix_one_levels(retval, nActiveThreads, depth,
834 levels);
835 KMP_DEBUG_ASSERT(__kmp_affinity_gran != affinity_gran_default);
836 if (__kmp_affinity_gran_levels < 0) {
837 // Set the granularity level based on what levels are modeled
838 // in the machine topology map.
839 __kmp_affinity_gran_levels = 0; // lowest level (e.g. fine)
840 if (__kmp_affinity_gran > affinity_gran_thread) {
841 for (int i = 1; i <= depth_full; ++i) {
842 if (__kmp_affinity_gran <= i) // only count deeper levels
843 break;
844 if (levels[depth_full - i] > 0)
845 __kmp_affinity_gran_levels++;
846 }
847 }
848 if (__kmp_affinity_gran > affinity_gran_package)
849 __kmp_affinity_gran_levels++; // e.g. granularity = group
850 }
851
852 if (__kmp_affinity_verbose)
853 __kmp_affinity_print_hwloc_tp(retval, nActiveThreads, depth, levels);
854
855 KMP_CPU_FREE(oldMask);
856 *address2os = retval;
857 return depth;
858 }
859 #endif // KMP_USE_HWLOC
860
861 // If we don't know how to retrieve the machine's processor topology, or
862 // encounter an error in doing so, this routine is called to form a "flat"
863 // mapping of os thread id's <-> processor id's.
__kmp_affinity_create_flat_map(AddrUnsPair ** address2os,kmp_i18n_id_t * const msg_id)864 static int __kmp_affinity_create_flat_map(AddrUnsPair **address2os,
865 kmp_i18n_id_t *const msg_id) {
866 *address2os = NULL;
867 *msg_id = kmp_i18n_null;
868
869 // Even if __kmp_affinity_type == affinity_none, this routine might still
870 // called to set __kmp_ncores, as well as
871 // __kmp_nThreadsPerCore, nCoresPerPkg, & nPackages.
872 if (!KMP_AFFINITY_CAPABLE()) {
873 KMP_ASSERT(__kmp_affinity_type == affinity_none);
874 __kmp_ncores = nPackages = __kmp_xproc;
875 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
876 if (__kmp_affinity_verbose) {
877 KMP_INFORM(AffFlatTopology, "KMP_AFFINITY");
878 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
879 KMP_INFORM(Uniform, "KMP_AFFINITY");
880 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
881 __kmp_nThreadsPerCore, __kmp_ncores);
882 }
883 return 0;
884 }
885
886 // When affinity is off, this routine will still be called to set
887 // __kmp_ncores, as well as __kmp_nThreadsPerCore, nCoresPerPkg, & nPackages.
888 // Make sure all these vars are set correctly, and return now if affinity is
889 // not enabled.
890 __kmp_ncores = nPackages = __kmp_avail_proc;
891 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
892 if (__kmp_affinity_verbose) {
893 char buf[KMP_AFFIN_MASK_PRINT_LEN];
894 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
895 __kmp_affin_fullMask);
896
897 KMP_INFORM(AffCapableUseFlat, "KMP_AFFINITY");
898 if (__kmp_affinity_respect_mask) {
899 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
900 } else {
901 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
902 }
903 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
904 KMP_INFORM(Uniform, "KMP_AFFINITY");
905 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
906 __kmp_nThreadsPerCore, __kmp_ncores);
907 }
908 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
909 __kmp_pu_os_idx = (int *)__kmp_allocate(sizeof(int) * __kmp_avail_proc);
910 if (__kmp_affinity_type == affinity_none) {
911 int avail_ct = 0;
912 int i;
913 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
914 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask))
915 continue;
916 __kmp_pu_os_idx[avail_ct++] = i; // suppose indices are flat
917 }
918 return 0;
919 }
920
921 // Contruct the data structure to be returned.
922 *address2os =
923 (AddrUnsPair *)__kmp_allocate(sizeof(**address2os) * __kmp_avail_proc);
924 int avail_ct = 0;
925 int i;
926 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
927 // Skip this proc if it is not included in the machine model.
928 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
929 continue;
930 }
931 __kmp_pu_os_idx[avail_ct] = i; // suppose indices are flat
932 Address addr(1);
933 addr.labels[0] = i;
934 (*address2os)[avail_ct++] = AddrUnsPair(addr, i);
935 }
936 if (__kmp_affinity_verbose) {
937 KMP_INFORM(OSProcToPackage, "KMP_AFFINITY");
938 }
939
940 if (__kmp_affinity_gran_levels < 0) {
941 // Only the package level is modeled in the machine topology map,
942 // so the #levels of granularity is either 0 or 1.
943 if (__kmp_affinity_gran > affinity_gran_package) {
944 __kmp_affinity_gran_levels = 1;
945 } else {
946 __kmp_affinity_gran_levels = 0;
947 }
948 }
949 return 1;
950 }
951
952 #if KMP_GROUP_AFFINITY
953
954 // If multiple Windows* OS processor groups exist, we can create a 2-level
955 // topology map with the groups at level 0 and the individual procs at level 1.
956 // This facilitates letting the threads float among all procs in a group,
957 // if granularity=group (the default when there are multiple groups).
__kmp_affinity_create_proc_group_map(AddrUnsPair ** address2os,kmp_i18n_id_t * const msg_id)958 static int __kmp_affinity_create_proc_group_map(AddrUnsPair **address2os,
959 kmp_i18n_id_t *const msg_id) {
960 *address2os = NULL;
961 *msg_id = kmp_i18n_null;
962
963 // If we aren't affinity capable, then return now.
964 // The flat mapping will be used.
965 if (!KMP_AFFINITY_CAPABLE()) {
966 // FIXME set *msg_id
967 return -1;
968 }
969
970 // Contruct the data structure to be returned.
971 *address2os =
972 (AddrUnsPair *)__kmp_allocate(sizeof(**address2os) * __kmp_avail_proc);
973 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
974 __kmp_pu_os_idx = (int *)__kmp_allocate(sizeof(int) * __kmp_avail_proc);
975 int avail_ct = 0;
976 int i;
977 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
978 // Skip this proc if it is not included in the machine model.
979 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
980 continue;
981 }
982 __kmp_pu_os_idx[avail_ct] = i; // suppose indices are flat
983 Address addr(2);
984 addr.labels[0] = i / (CHAR_BIT * sizeof(DWORD_PTR));
985 addr.labels[1] = i % (CHAR_BIT * sizeof(DWORD_PTR));
986 (*address2os)[avail_ct++] = AddrUnsPair(addr, i);
987
988 if (__kmp_affinity_verbose) {
989 KMP_INFORM(AffOSProcToGroup, "KMP_AFFINITY", i, addr.labels[0],
990 addr.labels[1]);
991 }
992 }
993
994 if (__kmp_affinity_gran_levels < 0) {
995 if (__kmp_affinity_gran == affinity_gran_group) {
996 __kmp_affinity_gran_levels = 1;
997 } else if ((__kmp_affinity_gran == affinity_gran_fine) ||
998 (__kmp_affinity_gran == affinity_gran_thread)) {
999 __kmp_affinity_gran_levels = 0;
1000 } else {
1001 const char *gran_str = NULL;
1002 if (__kmp_affinity_gran == affinity_gran_core) {
1003 gran_str = "core";
1004 } else if (__kmp_affinity_gran == affinity_gran_package) {
1005 gran_str = "package";
1006 } else if (__kmp_affinity_gran == affinity_gran_node) {
1007 gran_str = "node";
1008 } else {
1009 KMP_ASSERT(0);
1010 }
1011
1012 // Warning: can't use affinity granularity \"gran\" with group topology
1013 // method, using "thread"
1014 __kmp_affinity_gran_levels = 0;
1015 }
1016 }
1017 return 2;
1018 }
1019
1020 #endif /* KMP_GROUP_AFFINITY */
1021
1022 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1023
__kmp_cpuid_mask_width(int count)1024 static int __kmp_cpuid_mask_width(int count) {
1025 int r = 0;
1026
1027 while ((1 << r) < count)
1028 ++r;
1029 return r;
1030 }
1031
1032 class apicThreadInfo {
1033 public:
1034 unsigned osId; // param to __kmp_affinity_bind_thread
1035 unsigned apicId; // from cpuid after binding
1036 unsigned maxCoresPerPkg; // ""
1037 unsigned maxThreadsPerPkg; // ""
1038 unsigned pkgId; // inferred from above values
1039 unsigned coreId; // ""
1040 unsigned threadId; // ""
1041 };
1042
__kmp_affinity_cmp_apicThreadInfo_phys_id(const void * a,const void * b)1043 static int __kmp_affinity_cmp_apicThreadInfo_phys_id(const void *a,
1044 const void *b) {
1045 const apicThreadInfo *aa = (const apicThreadInfo *)a;
1046 const apicThreadInfo *bb = (const apicThreadInfo *)b;
1047 if (aa->pkgId < bb->pkgId)
1048 return -1;
1049 if (aa->pkgId > bb->pkgId)
1050 return 1;
1051 if (aa->coreId < bb->coreId)
1052 return -1;
1053 if (aa->coreId > bb->coreId)
1054 return 1;
1055 if (aa->threadId < bb->threadId)
1056 return -1;
1057 if (aa->threadId > bb->threadId)
1058 return 1;
1059 return 0;
1060 }
1061
1062 // On IA-32 architecture and Intel(R) 64 architecture, we attempt to use
1063 // an algorithm which cycles through the available os threads, setting
1064 // the current thread's affinity mask to that thread, and then retrieves
1065 // the Apic Id for each thread context using the cpuid instruction.
__kmp_affinity_create_apicid_map(AddrUnsPair ** address2os,kmp_i18n_id_t * const msg_id)1066 static int __kmp_affinity_create_apicid_map(AddrUnsPair **address2os,
1067 kmp_i18n_id_t *const msg_id) {
1068 kmp_cpuid buf;
1069 *address2os = NULL;
1070 *msg_id = kmp_i18n_null;
1071
1072 // Check if cpuid leaf 4 is supported.
1073 __kmp_x86_cpuid(0, 0, &buf);
1074 if (buf.eax < 4) {
1075 *msg_id = kmp_i18n_str_NoLeaf4Support;
1076 return -1;
1077 }
1078
1079 // The algorithm used starts by setting the affinity to each available thread
1080 // and retrieving info from the cpuid instruction, so if we are not capable of
1081 // calling __kmp_get_system_affinity() and _kmp_get_system_affinity(), then we
1082 // need to do something else - use the defaults that we calculated from
1083 // issuing cpuid without binding to each proc.
1084 if (!KMP_AFFINITY_CAPABLE()) {
1085 // Hack to try and infer the machine topology using only the data
1086 // available from cpuid on the current thread, and __kmp_xproc.
1087 KMP_ASSERT(__kmp_affinity_type == affinity_none);
1088
1089 // Get an upper bound on the number of threads per package using cpuid(1).
1090 // On some OS/chps combinations where HT is supported by the chip but is
1091 // disabled, this value will be 2 on a single core chip. Usually, it will be
1092 // 2 if HT is enabled and 1 if HT is disabled.
1093 __kmp_x86_cpuid(1, 0, &buf);
1094 int maxThreadsPerPkg = (buf.ebx >> 16) & 0xff;
1095 if (maxThreadsPerPkg == 0) {
1096 maxThreadsPerPkg = 1;
1097 }
1098
1099 // The num cores per pkg comes from cpuid(4). 1 must be added to the encoded
1100 // value.
1101 //
1102 // The author of cpu_count.cpp treated this only an upper bound on the
1103 // number of cores, but I haven't seen any cases where it was greater than
1104 // the actual number of cores, so we will treat it as exact in this block of
1105 // code.
1106 //
1107 // First, we need to check if cpuid(4) is supported on this chip. To see if
1108 // cpuid(n) is supported, issue cpuid(0) and check if eax has the value n or
1109 // greater.
1110 __kmp_x86_cpuid(0, 0, &buf);
1111 if (buf.eax >= 4) {
1112 __kmp_x86_cpuid(4, 0, &buf);
1113 nCoresPerPkg = ((buf.eax >> 26) & 0x3f) + 1;
1114 } else {
1115 nCoresPerPkg = 1;
1116 }
1117
1118 // There is no way to reliably tell if HT is enabled without issuing the
1119 // cpuid instruction from every thread, can correlating the cpuid info, so
1120 // if the machine is not affinity capable, we assume that HT is off. We have
1121 // seen quite a few machines where maxThreadsPerPkg is 2, yet the machine
1122 // does not support HT.
1123 //
1124 // - Older OSes are usually found on machines with older chips, which do not
1125 // support HT.
1126 // - The performance penalty for mistakenly identifying a machine as HT when
1127 // it isn't (which results in blocktime being incorrectly set to 0) is
1128 // greater than the penalty when for mistakenly identifying a machine as
1129 // being 1 thread/core when it is really HT enabled (which results in
1130 // blocktime being incorrectly set to a positive value).
1131 __kmp_ncores = __kmp_xproc;
1132 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
1133 __kmp_nThreadsPerCore = 1;
1134 if (__kmp_affinity_verbose) {
1135 KMP_INFORM(AffNotCapableUseLocCpuid, "KMP_AFFINITY");
1136 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
1137 if (__kmp_affinity_uniform_topology()) {
1138 KMP_INFORM(Uniform, "KMP_AFFINITY");
1139 } else {
1140 KMP_INFORM(NonUniform, "KMP_AFFINITY");
1141 }
1142 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
1143 __kmp_nThreadsPerCore, __kmp_ncores);
1144 }
1145 return 0;
1146 }
1147
1148 // From here on, we can assume that it is safe to call
1149 // __kmp_get_system_affinity() and __kmp_set_system_affinity(), even if
1150 // __kmp_affinity_type = affinity_none.
1151
1152 // Save the affinity mask for the current thread.
1153 kmp_affin_mask_t *oldMask;
1154 KMP_CPU_ALLOC(oldMask);
1155 KMP_ASSERT(oldMask != NULL);
1156 __kmp_get_system_affinity(oldMask, TRUE);
1157
1158 // Run through each of the available contexts, binding the current thread
1159 // to it, and obtaining the pertinent information using the cpuid instr.
1160 //
1161 // The relevant information is:
1162 // - Apic Id: Bits 24:31 of ebx after issuing cpuid(1) - each thread context
1163 // has a uniqie Apic Id, which is of the form pkg# : core# : thread#.
1164 // - Max Threads Per Pkg: Bits 16:23 of ebx after issuing cpuid(1). The value
1165 // of this field determines the width of the core# + thread# fields in the
1166 // Apic Id. It is also an upper bound on the number of threads per
1167 // package, but it has been verified that situations happen were it is not
1168 // exact. In particular, on certain OS/chip combinations where Intel(R)
1169 // Hyper-Threading Technology is supported by the chip but has been
1170 // disabled, the value of this field will be 2 (for a single core chip).
1171 // On other OS/chip combinations supporting Intel(R) Hyper-Threading
1172 // Technology, the value of this field will be 1 when Intel(R)
1173 // Hyper-Threading Technology is disabled and 2 when it is enabled.
1174 // - Max Cores Per Pkg: Bits 26:31 of eax after issuing cpuid(4). The value
1175 // of this field (+1) determines the width of the core# field in the Apic
1176 // Id. The comments in "cpucount.cpp" say that this value is an upper
1177 // bound, but the IA-32 architecture manual says that it is exactly the
1178 // number of cores per package, and I haven't seen any case where it
1179 // wasn't.
1180 //
1181 // From this information, deduce the package Id, core Id, and thread Id,
1182 // and set the corresponding fields in the apicThreadInfo struct.
1183 unsigned i;
1184 apicThreadInfo *threadInfo = (apicThreadInfo *)__kmp_allocate(
1185 __kmp_avail_proc * sizeof(apicThreadInfo));
1186 unsigned nApics = 0;
1187 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
1188 // Skip this proc if it is not included in the machine model.
1189 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
1190 continue;
1191 }
1192 KMP_DEBUG_ASSERT((int)nApics < __kmp_avail_proc);
1193
1194 __kmp_affinity_dispatch->bind_thread(i);
1195 threadInfo[nApics].osId = i;
1196
1197 // The apic id and max threads per pkg come from cpuid(1).
1198 __kmp_x86_cpuid(1, 0, &buf);
1199 if (((buf.edx >> 9) & 1) == 0) {
1200 __kmp_set_system_affinity(oldMask, TRUE);
1201 __kmp_free(threadInfo);
1202 KMP_CPU_FREE(oldMask);
1203 *msg_id = kmp_i18n_str_ApicNotPresent;
1204 return -1;
1205 }
1206 threadInfo[nApics].apicId = (buf.ebx >> 24) & 0xff;
1207 threadInfo[nApics].maxThreadsPerPkg = (buf.ebx >> 16) & 0xff;
1208 if (threadInfo[nApics].maxThreadsPerPkg == 0) {
1209 threadInfo[nApics].maxThreadsPerPkg = 1;
1210 }
1211
1212 // Max cores per pkg comes from cpuid(4). 1 must be added to the encoded
1213 // value.
1214 //
1215 // First, we need to check if cpuid(4) is supported on this chip. To see if
1216 // cpuid(n) is supported, issue cpuid(0) and check if eax has the value n
1217 // or greater.
1218 __kmp_x86_cpuid(0, 0, &buf);
1219 if (buf.eax >= 4) {
1220 __kmp_x86_cpuid(4, 0, &buf);
1221 threadInfo[nApics].maxCoresPerPkg = ((buf.eax >> 26) & 0x3f) + 1;
1222 } else {
1223 threadInfo[nApics].maxCoresPerPkg = 1;
1224 }
1225
1226 // Infer the pkgId / coreId / threadId using only the info obtained locally.
1227 int widthCT = __kmp_cpuid_mask_width(threadInfo[nApics].maxThreadsPerPkg);
1228 threadInfo[nApics].pkgId = threadInfo[nApics].apicId >> widthCT;
1229
1230 int widthC = __kmp_cpuid_mask_width(threadInfo[nApics].maxCoresPerPkg);
1231 int widthT = widthCT - widthC;
1232 if (widthT < 0) {
1233 // I've never seen this one happen, but I suppose it could, if the cpuid
1234 // instruction on a chip was really screwed up. Make sure to restore the
1235 // affinity mask before the tail call.
1236 __kmp_set_system_affinity(oldMask, TRUE);
1237 __kmp_free(threadInfo);
1238 KMP_CPU_FREE(oldMask);
1239 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1240 return -1;
1241 }
1242
1243 int maskC = (1 << widthC) - 1;
1244 threadInfo[nApics].coreId = (threadInfo[nApics].apicId >> widthT) & maskC;
1245
1246 int maskT = (1 << widthT) - 1;
1247 threadInfo[nApics].threadId = threadInfo[nApics].apicId & maskT;
1248
1249 nApics++;
1250 }
1251
1252 // We've collected all the info we need.
1253 // Restore the old affinity mask for this thread.
1254 __kmp_set_system_affinity(oldMask, TRUE);
1255
1256 // If there's only one thread context to bind to, form an Address object
1257 // with depth 1 and return immediately (or, if affinity is off, set
1258 // address2os to NULL and return).
1259 //
1260 // If it is configured to omit the package level when there is only a single
1261 // package, the logic at the end of this routine won't work if there is only
1262 // a single thread - it would try to form an Address object with depth 0.
1263 KMP_ASSERT(nApics > 0);
1264 if (nApics == 1) {
1265 __kmp_ncores = nPackages = 1;
1266 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
1267 if (__kmp_affinity_verbose) {
1268 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1269 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1270
1271 KMP_INFORM(AffUseGlobCpuid, "KMP_AFFINITY");
1272 if (__kmp_affinity_respect_mask) {
1273 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
1274 } else {
1275 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
1276 }
1277 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
1278 KMP_INFORM(Uniform, "KMP_AFFINITY");
1279 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
1280 __kmp_nThreadsPerCore, __kmp_ncores);
1281 }
1282
1283 if (__kmp_affinity_type == affinity_none) {
1284 __kmp_free(threadInfo);
1285 KMP_CPU_FREE(oldMask);
1286 return 0;
1287 }
1288
1289 *address2os = (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair));
1290 Address addr(1);
1291 addr.labels[0] = threadInfo[0].pkgId;
1292 (*address2os)[0] = AddrUnsPair(addr, threadInfo[0].osId);
1293
1294 if (__kmp_affinity_gran_levels < 0) {
1295 __kmp_affinity_gran_levels = 0;
1296 }
1297
1298 if (__kmp_affinity_verbose) {
1299 __kmp_affinity_print_topology(*address2os, 1, 1, 0, -1, -1);
1300 }
1301
1302 __kmp_free(threadInfo);
1303 KMP_CPU_FREE(oldMask);
1304 return 1;
1305 }
1306
1307 // Sort the threadInfo table by physical Id.
1308 qsort(threadInfo, nApics, sizeof(*threadInfo),
1309 __kmp_affinity_cmp_apicThreadInfo_phys_id);
1310
1311 // The table is now sorted by pkgId / coreId / threadId, but we really don't
1312 // know the radix of any of the fields. pkgId's may be sparsely assigned among
1313 // the chips on a system. Although coreId's are usually assigned
1314 // [0 .. coresPerPkg-1] and threadId's are usually assigned
1315 // [0..threadsPerCore-1], we don't want to make any such assumptions.
1316 //
1317 // For that matter, we don't know what coresPerPkg and threadsPerCore (or the
1318 // total # packages) are at this point - we want to determine that now. We
1319 // only have an upper bound on the first two figures.
1320 //
1321 // We also perform a consistency check at this point: the values returned by
1322 // the cpuid instruction for any thread bound to a given package had better
1323 // return the same info for maxThreadsPerPkg and maxCoresPerPkg.
1324 nPackages = 1;
1325 nCoresPerPkg = 1;
1326 __kmp_nThreadsPerCore = 1;
1327 unsigned nCores = 1;
1328
1329 unsigned pkgCt = 1; // to determine radii
1330 unsigned lastPkgId = threadInfo[0].pkgId;
1331 unsigned coreCt = 1;
1332 unsigned lastCoreId = threadInfo[0].coreId;
1333 unsigned threadCt = 1;
1334 unsigned lastThreadId = threadInfo[0].threadId;
1335
1336 // intra-pkg consist checks
1337 unsigned prevMaxCoresPerPkg = threadInfo[0].maxCoresPerPkg;
1338 unsigned prevMaxThreadsPerPkg = threadInfo[0].maxThreadsPerPkg;
1339
1340 for (i = 1; i < nApics; i++) {
1341 if (threadInfo[i].pkgId != lastPkgId) {
1342 nCores++;
1343 pkgCt++;
1344 lastPkgId = threadInfo[i].pkgId;
1345 if ((int)coreCt > nCoresPerPkg)
1346 nCoresPerPkg = coreCt;
1347 coreCt = 1;
1348 lastCoreId = threadInfo[i].coreId;
1349 if ((int)threadCt > __kmp_nThreadsPerCore)
1350 __kmp_nThreadsPerCore = threadCt;
1351 threadCt = 1;
1352 lastThreadId = threadInfo[i].threadId;
1353
1354 // This is a different package, so go on to the next iteration without
1355 // doing any consistency checks. Reset the consistency check vars, though.
1356 prevMaxCoresPerPkg = threadInfo[i].maxCoresPerPkg;
1357 prevMaxThreadsPerPkg = threadInfo[i].maxThreadsPerPkg;
1358 continue;
1359 }
1360
1361 if (threadInfo[i].coreId != lastCoreId) {
1362 nCores++;
1363 coreCt++;
1364 lastCoreId = threadInfo[i].coreId;
1365 if ((int)threadCt > __kmp_nThreadsPerCore)
1366 __kmp_nThreadsPerCore = threadCt;
1367 threadCt = 1;
1368 lastThreadId = threadInfo[i].threadId;
1369 } else if (threadInfo[i].threadId != lastThreadId) {
1370 threadCt++;
1371 lastThreadId = threadInfo[i].threadId;
1372 } else {
1373 __kmp_free(threadInfo);
1374 KMP_CPU_FREE(oldMask);
1375 *msg_id = kmp_i18n_str_LegacyApicIDsNotUnique;
1376 return -1;
1377 }
1378
1379 // Check to make certain that the maxCoresPerPkg and maxThreadsPerPkg
1380 // fields agree between all the threads bounds to a given package.
1381 if ((prevMaxCoresPerPkg != threadInfo[i].maxCoresPerPkg) ||
1382 (prevMaxThreadsPerPkg != threadInfo[i].maxThreadsPerPkg)) {
1383 __kmp_free(threadInfo);
1384 KMP_CPU_FREE(oldMask);
1385 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1386 return -1;
1387 }
1388 }
1389 nPackages = pkgCt;
1390 if ((int)coreCt > nCoresPerPkg)
1391 nCoresPerPkg = coreCt;
1392 if ((int)threadCt > __kmp_nThreadsPerCore)
1393 __kmp_nThreadsPerCore = threadCt;
1394
1395 // When affinity is off, this routine will still be called to set
1396 // __kmp_ncores, as well as __kmp_nThreadsPerCore, nCoresPerPkg, & nPackages.
1397 // Make sure all these vars are set correctly, and return now if affinity is
1398 // not enabled.
1399 __kmp_ncores = nCores;
1400 if (__kmp_affinity_verbose) {
1401 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1402 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1403
1404 KMP_INFORM(AffUseGlobCpuid, "KMP_AFFINITY");
1405 if (__kmp_affinity_respect_mask) {
1406 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
1407 } else {
1408 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
1409 }
1410 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
1411 if (__kmp_affinity_uniform_topology()) {
1412 KMP_INFORM(Uniform, "KMP_AFFINITY");
1413 } else {
1414 KMP_INFORM(NonUniform, "KMP_AFFINITY");
1415 }
1416 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
1417 __kmp_nThreadsPerCore, __kmp_ncores);
1418 }
1419 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
1420 KMP_DEBUG_ASSERT(nApics == (unsigned)__kmp_avail_proc);
1421 __kmp_pu_os_idx = (int *)__kmp_allocate(sizeof(int) * __kmp_avail_proc);
1422 for (i = 0; i < nApics; ++i) {
1423 __kmp_pu_os_idx[i] = threadInfo[i].osId;
1424 }
1425 if (__kmp_affinity_type == affinity_none) {
1426 __kmp_free(threadInfo);
1427 KMP_CPU_FREE(oldMask);
1428 return 0;
1429 }
1430
1431 // Now that we've determined the number of packages, the number of cores per
1432 // package, and the number of threads per core, we can construct the data
1433 // structure that is to be returned.
1434 int pkgLevel = 0;
1435 int coreLevel = (nCoresPerPkg <= 1) ? -1 : 1;
1436 int threadLevel =
1437 (__kmp_nThreadsPerCore <= 1) ? -1 : ((coreLevel >= 0) ? 2 : 1);
1438 unsigned depth = (pkgLevel >= 0) + (coreLevel >= 0) + (threadLevel >= 0);
1439
1440 KMP_ASSERT(depth > 0);
1441 *address2os = (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair) * nApics);
1442
1443 for (i = 0; i < nApics; ++i) {
1444 Address addr(depth);
1445 unsigned os = threadInfo[i].osId;
1446 int d = 0;
1447
1448 if (pkgLevel >= 0) {
1449 addr.labels[d++] = threadInfo[i].pkgId;
1450 }
1451 if (coreLevel >= 0) {
1452 addr.labels[d++] = threadInfo[i].coreId;
1453 }
1454 if (threadLevel >= 0) {
1455 addr.labels[d++] = threadInfo[i].threadId;
1456 }
1457 (*address2os)[i] = AddrUnsPair(addr, os);
1458 }
1459
1460 if (__kmp_affinity_gran_levels < 0) {
1461 // Set the granularity level based on what levels are modeled in the machine
1462 // topology map.
1463 __kmp_affinity_gran_levels = 0;
1464 if ((threadLevel >= 0) && (__kmp_affinity_gran > affinity_gran_thread)) {
1465 __kmp_affinity_gran_levels++;
1466 }
1467 if ((coreLevel >= 0) && (__kmp_affinity_gran > affinity_gran_core)) {
1468 __kmp_affinity_gran_levels++;
1469 }
1470 if ((pkgLevel >= 0) && (__kmp_affinity_gran > affinity_gran_package)) {
1471 __kmp_affinity_gran_levels++;
1472 }
1473 }
1474
1475 if (__kmp_affinity_verbose) {
1476 __kmp_affinity_print_topology(*address2os, nApics, depth, pkgLevel,
1477 coreLevel, threadLevel);
1478 }
1479
1480 __kmp_free(threadInfo);
1481 KMP_CPU_FREE(oldMask);
1482 return depth;
1483 }
1484
1485 // Intel(R) microarchitecture code name Nehalem, Dunnington and later
1486 // architectures support a newer interface for specifying the x2APIC Ids,
1487 // based on cpuid leaf 11.
__kmp_affinity_create_x2apicid_map(AddrUnsPair ** address2os,kmp_i18n_id_t * const msg_id)1488 static int __kmp_affinity_create_x2apicid_map(AddrUnsPair **address2os,
1489 kmp_i18n_id_t *const msg_id) {
1490 kmp_cpuid buf;
1491 *address2os = NULL;
1492 *msg_id = kmp_i18n_null;
1493
1494 // Check to see if cpuid leaf 11 is supported.
1495 __kmp_x86_cpuid(0, 0, &buf);
1496 if (buf.eax < 11) {
1497 *msg_id = kmp_i18n_str_NoLeaf11Support;
1498 return -1;
1499 }
1500 __kmp_x86_cpuid(11, 0, &buf);
1501 if (buf.ebx == 0) {
1502 *msg_id = kmp_i18n_str_NoLeaf11Support;
1503 return -1;
1504 }
1505
1506 // Find the number of levels in the machine topology. While we're at it, get
1507 // the default values for __kmp_nThreadsPerCore & nCoresPerPkg. We will try to
1508 // get more accurate values later by explicitly counting them, but get
1509 // reasonable defaults now, in case we return early.
1510 int level;
1511 int threadLevel = -1;
1512 int coreLevel = -1;
1513 int pkgLevel = -1;
1514 __kmp_nThreadsPerCore = nCoresPerPkg = nPackages = 1;
1515
1516 for (level = 0;; level++) {
1517 if (level > 31) {
1518 // FIXME: Hack for DPD200163180
1519 //
1520 // If level is big then something went wrong -> exiting
1521 //
1522 // There could actually be 32 valid levels in the machine topology, but so
1523 // far, the only machine we have seen which does not exit this loop before
1524 // iteration 32 has fubar x2APIC settings.
1525 //
1526 // For now, just reject this case based upon loop trip count.
1527 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1528 return -1;
1529 }
1530 __kmp_x86_cpuid(11, level, &buf);
1531 if (buf.ebx == 0) {
1532 if (pkgLevel < 0) {
1533 // Will infer nPackages from __kmp_xproc
1534 pkgLevel = level;
1535 level++;
1536 }
1537 break;
1538 }
1539 int kind = (buf.ecx >> 8) & 0xff;
1540 if (kind == 1) {
1541 // SMT level
1542 threadLevel = level;
1543 coreLevel = -1;
1544 pkgLevel = -1;
1545 __kmp_nThreadsPerCore = buf.ebx & 0xffff;
1546 if (__kmp_nThreadsPerCore == 0) {
1547 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1548 return -1;
1549 }
1550 } else if (kind == 2) {
1551 // core level
1552 coreLevel = level;
1553 pkgLevel = -1;
1554 nCoresPerPkg = buf.ebx & 0xffff;
1555 if (nCoresPerPkg == 0) {
1556 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1557 return -1;
1558 }
1559 } else {
1560 if (level <= 0) {
1561 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1562 return -1;
1563 }
1564 if (pkgLevel >= 0) {
1565 continue;
1566 }
1567 pkgLevel = level;
1568 nPackages = buf.ebx & 0xffff;
1569 if (nPackages == 0) {
1570 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1571 return -1;
1572 }
1573 }
1574 }
1575 int depth = level;
1576
1577 // In the above loop, "level" was counted from the finest level (usually
1578 // thread) to the coarsest. The caller expects that we will place the labels
1579 // in (*address2os)[].first.labels[] in the inverse order, so we need to
1580 // invert the vars saying which level means what.
1581 if (threadLevel >= 0) {
1582 threadLevel = depth - threadLevel - 1;
1583 }
1584 if (coreLevel >= 0) {
1585 coreLevel = depth - coreLevel - 1;
1586 }
1587 KMP_DEBUG_ASSERT(pkgLevel >= 0);
1588 pkgLevel = depth - pkgLevel - 1;
1589
1590 // The algorithm used starts by setting the affinity to each available thread
1591 // and retrieving info from the cpuid instruction, so if we are not capable of
1592 // calling __kmp_get_system_affinity() and _kmp_get_system_affinity(), then we
1593 // need to do something else - use the defaults that we calculated from
1594 // issuing cpuid without binding to each proc.
1595 if (!KMP_AFFINITY_CAPABLE()) {
1596 // Hack to try and infer the machine topology using only the data
1597 // available from cpuid on the current thread, and __kmp_xproc.
1598 KMP_ASSERT(__kmp_affinity_type == affinity_none);
1599
1600 __kmp_ncores = __kmp_xproc / __kmp_nThreadsPerCore;
1601 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
1602 if (__kmp_affinity_verbose) {
1603 KMP_INFORM(AffNotCapableUseLocCpuidL11, "KMP_AFFINITY");
1604 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
1605 if (__kmp_affinity_uniform_topology()) {
1606 KMP_INFORM(Uniform, "KMP_AFFINITY");
1607 } else {
1608 KMP_INFORM(NonUniform, "KMP_AFFINITY");
1609 }
1610 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
1611 __kmp_nThreadsPerCore, __kmp_ncores);
1612 }
1613 return 0;
1614 }
1615
1616 // From here on, we can assume that it is safe to call
1617 // __kmp_get_system_affinity() and __kmp_set_system_affinity(), even if
1618 // __kmp_affinity_type = affinity_none.
1619
1620 // Save the affinity mask for the current thread.
1621 kmp_affin_mask_t *oldMask;
1622 KMP_CPU_ALLOC(oldMask);
1623 __kmp_get_system_affinity(oldMask, TRUE);
1624
1625 // Allocate the data structure to be returned.
1626 AddrUnsPair *retval =
1627 (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair) * __kmp_avail_proc);
1628
1629 // Run through each of the available contexts, binding the current thread
1630 // to it, and obtaining the pertinent information using the cpuid instr.
1631 unsigned int proc;
1632 int nApics = 0;
1633 KMP_CPU_SET_ITERATE(proc, __kmp_affin_fullMask) {
1634 // Skip this proc if it is not included in the machine model.
1635 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
1636 continue;
1637 }
1638 KMP_DEBUG_ASSERT(nApics < __kmp_avail_proc);
1639
1640 __kmp_affinity_dispatch->bind_thread(proc);
1641
1642 // Extract labels for each level in the machine topology map from Apic ID.
1643 Address addr(depth);
1644 int prev_shift = 0;
1645
1646 for (level = 0; level < depth; level++) {
1647 __kmp_x86_cpuid(11, level, &buf);
1648 unsigned apicId = buf.edx;
1649 if (buf.ebx == 0) {
1650 if (level != depth - 1) {
1651 KMP_CPU_FREE(oldMask);
1652 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1653 return -1;
1654 }
1655 addr.labels[depth - level - 1] = apicId >> prev_shift;
1656 level++;
1657 break;
1658 }
1659 int shift = buf.eax & 0x1f;
1660 int mask = (1 << shift) - 1;
1661 addr.labels[depth - level - 1] = (apicId & mask) >> prev_shift;
1662 prev_shift = shift;
1663 }
1664 if (level != depth) {
1665 KMP_CPU_FREE(oldMask);
1666 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1667 return -1;
1668 }
1669
1670 retval[nApics] = AddrUnsPair(addr, proc);
1671 nApics++;
1672 }
1673
1674 // We've collected all the info we need.
1675 // Restore the old affinity mask for this thread.
1676 __kmp_set_system_affinity(oldMask, TRUE);
1677
1678 // If there's only one thread context to bind to, return now.
1679 KMP_ASSERT(nApics > 0);
1680 if (nApics == 1) {
1681 __kmp_ncores = nPackages = 1;
1682 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
1683 if (__kmp_affinity_verbose) {
1684 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1685 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1686
1687 KMP_INFORM(AffUseGlobCpuidL11, "KMP_AFFINITY");
1688 if (__kmp_affinity_respect_mask) {
1689 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
1690 } else {
1691 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
1692 }
1693 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
1694 KMP_INFORM(Uniform, "KMP_AFFINITY");
1695 KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
1696 __kmp_nThreadsPerCore, __kmp_ncores);
1697 }
1698
1699 if (__kmp_affinity_type == affinity_none) {
1700 __kmp_free(retval);
1701 KMP_CPU_FREE(oldMask);
1702 return 0;
1703 }
1704
1705 // Form an Address object which only includes the package level.
1706 Address addr(1);
1707 addr.labels[0] = retval[0].first.labels[pkgLevel];
1708 retval[0].first = addr;
1709
1710 if (__kmp_affinity_gran_levels < 0) {
1711 __kmp_affinity_gran_levels = 0;
1712 }
1713
1714 if (__kmp_affinity_verbose) {
1715 __kmp_affinity_print_topology(retval, 1, 1, 0, -1, -1);
1716 }
1717
1718 *address2os = retval;
1719 KMP_CPU_FREE(oldMask);
1720 return 1;
1721 }
1722
1723 // Sort the table by physical Id.
1724 qsort(retval, nApics, sizeof(*retval), __kmp_affinity_cmp_Address_labels);
1725
1726 // Find the radix at each of the levels.
1727 unsigned *totals = (unsigned *)__kmp_allocate(depth * sizeof(unsigned));
1728 unsigned *counts = (unsigned *)__kmp_allocate(depth * sizeof(unsigned));
1729 unsigned *maxCt = (unsigned *)__kmp_allocate(depth * sizeof(unsigned));
1730 unsigned *last = (unsigned *)__kmp_allocate(depth * sizeof(unsigned));
1731 for (level = 0; level < depth; level++) {
1732 totals[level] = 1;
1733 maxCt[level] = 1;
1734 counts[level] = 1;
1735 last[level] = retval[0].first.labels[level];
1736 }
1737
1738 // From here on, the iteration variable "level" runs from the finest level to
1739 // the coarsest, i.e. we iterate forward through
1740 // (*address2os)[].first.labels[] - in the previous loops, we iterated
1741 // backwards.
1742 for (proc = 1; (int)proc < nApics; proc++) {
1743 int level;
1744 for (level = 0; level < depth; level++) {
1745 if (retval[proc].first.labels[level] != last[level]) {
1746 int j;
1747 for (j = level + 1; j < depth; j++) {
1748 totals[j]++;
1749 counts[j] = 1;
1750 // The line below causes printing incorrect topology information in
1751 // case the max value for some level (maxCt[level]) is encountered
1752 // earlier than some less value while going through the array. For
1753 // example, let pkg0 has 4 cores and pkg1 has 2 cores. Then
1754 // maxCt[1] == 2
1755 // whereas it must be 4.
1756 // TODO!!! Check if it can be commented safely
1757 // maxCt[j] = 1;
1758 last[j] = retval[proc].first.labels[j];
1759 }
1760 totals[level]++;
1761 counts[level]++;
1762 if (counts[level] > maxCt[level]) {
1763 maxCt[level] = counts[level];
1764 }
1765 last[level] = retval[proc].first.labels[level];
1766 break;
1767 } else if (level == depth - 1) {
1768 __kmp_free(last);
1769 __kmp_free(maxCt);
1770 __kmp_free(counts);
1771 __kmp_free(totals);
1772 __kmp_free(retval);
1773 KMP_CPU_FREE(oldMask);
1774 *msg_id = kmp_i18n_str_x2ApicIDsNotUnique;
1775 return -1;
1776 }
1777 }
1778 }
1779
1780 // When affinity is off, this routine will still be called to set
1781 // __kmp_ncores, as well as __kmp_nThreadsPerCore, nCoresPerPkg, & nPackages.
1782 // Make sure all these vars are set correctly, and return if affinity is not
1783 // enabled.
1784 if (threadLevel >= 0) {
1785 __kmp_nThreadsPerCore = maxCt[threadLevel];
1786 } else {
1787 __kmp_nThreadsPerCore = 1;
1788 }
1789 nPackages = totals[pkgLevel];
1790
1791 if (coreLevel >= 0) {
1792 __kmp_ncores = totals[coreLevel];
1793 nCoresPerPkg = maxCt[coreLevel];
1794 } else {
1795 __kmp_ncores = nPackages;
1796 nCoresPerPkg = 1;
1797 }
1798
1799 // Check to see if the machine topology is uniform
1800 unsigned prod = maxCt[0];
1801 for (level = 1; level < depth; level++) {
1802 prod *= maxCt[level];
1803 }
1804 bool uniform = (prod == totals[level - 1]);
1805
1806 // Print the machine topology summary.
1807 if (__kmp_affinity_verbose) {
1808 char mask[KMP_AFFIN_MASK_PRINT_LEN];
1809 __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1810
1811 KMP_INFORM(AffUseGlobCpuidL11, "KMP_AFFINITY");
1812 if (__kmp_affinity_respect_mask) {
1813 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", mask);
1814 } else {
1815 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", mask);
1816 }
1817 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
1818 if (uniform) {
1819 KMP_INFORM(Uniform, "KMP_AFFINITY");
1820 } else {
1821 KMP_INFORM(NonUniform, "KMP_AFFINITY");
1822 }
1823
1824 kmp_str_buf_t buf;
1825 __kmp_str_buf_init(&buf);
1826
1827 __kmp_str_buf_print(&buf, "%d", totals[0]);
1828 for (level = 1; level <= pkgLevel; level++) {
1829 __kmp_str_buf_print(&buf, " x %d", maxCt[level]);
1830 }
1831 KMP_INFORM(TopologyExtra, "KMP_AFFINITY", buf.str, nCoresPerPkg,
1832 __kmp_nThreadsPerCore, __kmp_ncores);
1833
1834 __kmp_str_buf_free(&buf);
1835 }
1836 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
1837 KMP_DEBUG_ASSERT(nApics == __kmp_avail_proc);
1838 __kmp_pu_os_idx = (int *)__kmp_allocate(sizeof(int) * __kmp_avail_proc);
1839 for (proc = 0; (int)proc < nApics; ++proc) {
1840 __kmp_pu_os_idx[proc] = retval[proc].second;
1841 }
1842 if (__kmp_affinity_type == affinity_none) {
1843 __kmp_free(last);
1844 __kmp_free(maxCt);
1845 __kmp_free(counts);
1846 __kmp_free(totals);
1847 __kmp_free(retval);
1848 KMP_CPU_FREE(oldMask);
1849 return 0;
1850 }
1851
1852 // Find any levels with radiix 1, and remove them from the map
1853 // (except for the package level).
1854 int new_depth = 0;
1855 for (level = 0; level < depth; level++) {
1856 if ((maxCt[level] == 1) && (level != pkgLevel)) {
1857 continue;
1858 }
1859 new_depth++;
1860 }
1861
1862 // If we are removing any levels, allocate a new vector to return,
1863 // and copy the relevant information to it.
1864 if (new_depth != depth) {
1865 AddrUnsPair *new_retval =
1866 (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair) * nApics);
1867 for (proc = 0; (int)proc < nApics; proc++) {
1868 Address addr(new_depth);
1869 new_retval[proc] = AddrUnsPair(addr, retval[proc].second);
1870 }
1871 int new_level = 0;
1872 int newPkgLevel = -1;
1873 int newCoreLevel = -1;
1874 int newThreadLevel = -1;
1875 for (level = 0; level < depth; level++) {
1876 if ((maxCt[level] == 1) && (level != pkgLevel)) {
1877 // Remove this level. Never remove the package level
1878 continue;
1879 }
1880 if (level == pkgLevel) {
1881 newPkgLevel = new_level;
1882 }
1883 if (level == coreLevel) {
1884 newCoreLevel = new_level;
1885 }
1886 if (level == threadLevel) {
1887 newThreadLevel = new_level;
1888 }
1889 for (proc = 0; (int)proc < nApics; proc++) {
1890 new_retval[proc].first.labels[new_level] =
1891 retval[proc].first.labels[level];
1892 }
1893 new_level++;
1894 }
1895
1896 __kmp_free(retval);
1897 retval = new_retval;
1898 depth = new_depth;
1899 pkgLevel = newPkgLevel;
1900 coreLevel = newCoreLevel;
1901 threadLevel = newThreadLevel;
1902 }
1903
1904 if (__kmp_affinity_gran_levels < 0) {
1905 // Set the granularity level based on what levels are modeled
1906 // in the machine topology map.
1907 __kmp_affinity_gran_levels = 0;
1908 if ((threadLevel >= 0) && (__kmp_affinity_gran > affinity_gran_thread)) {
1909 __kmp_affinity_gran_levels++;
1910 }
1911 if ((coreLevel >= 0) && (__kmp_affinity_gran > affinity_gran_core)) {
1912 __kmp_affinity_gran_levels++;
1913 }
1914 if (__kmp_affinity_gran > affinity_gran_package) {
1915 __kmp_affinity_gran_levels++;
1916 }
1917 }
1918
1919 if (__kmp_affinity_verbose) {
1920 __kmp_affinity_print_topology(retval, nApics, depth, pkgLevel, coreLevel,
1921 threadLevel);
1922 }
1923
1924 __kmp_free(last);
1925 __kmp_free(maxCt);
1926 __kmp_free(counts);
1927 __kmp_free(totals);
1928 KMP_CPU_FREE(oldMask);
1929 *address2os = retval;
1930 return depth;
1931 }
1932
1933 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1934
1935 #define osIdIndex 0
1936 #define threadIdIndex 1
1937 #define coreIdIndex 2
1938 #define pkgIdIndex 3
1939 #define nodeIdIndex 4
1940
1941 typedef unsigned *ProcCpuInfo;
1942 static unsigned maxIndex = pkgIdIndex;
1943
__kmp_affinity_cmp_ProcCpuInfo_phys_id(const void * a,const void * b)1944 static int __kmp_affinity_cmp_ProcCpuInfo_phys_id(const void *a,
1945 const void *b) {
1946 unsigned i;
1947 const unsigned *aa = *(unsigned *const *)a;
1948 const unsigned *bb = *(unsigned *const *)b;
1949 for (i = maxIndex;; i--) {
1950 if (aa[i] < bb[i])
1951 return -1;
1952 if (aa[i] > bb[i])
1953 return 1;
1954 if (i == osIdIndex)
1955 break;
1956 }
1957 return 0;
1958 }
1959
1960 #if KMP_USE_HIER_SCHED
1961 // Set the array sizes for the hierarchy layers
__kmp_dispatch_set_hierarchy_values()1962 static void __kmp_dispatch_set_hierarchy_values() {
1963 // Set the maximum number of L1's to number of cores
1964 // Set the maximum number of L2's to to either number of cores / 2 for
1965 // Intel(R) Xeon Phi(TM) coprocessor formally codenamed Knights Landing
1966 // Or the number of cores for Intel(R) Xeon(R) processors
1967 // Set the maximum number of NUMA nodes and L3's to number of packages
1968 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1] =
1969 nPackages * nCoresPerPkg * __kmp_nThreadsPerCore;
1970 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L1 + 1] = __kmp_ncores;
1971 #if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS)
1972 if (__kmp_mic_type >= mic3)
1973 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L2 + 1] = __kmp_ncores / 2;
1974 else
1975 #endif // KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
1976 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L2 + 1] = __kmp_ncores;
1977 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L3 + 1] = nPackages;
1978 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_NUMA + 1] = nPackages;
1979 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_LOOP + 1] = 1;
1980 // Set the number of threads per unit
1981 // Number of hardware threads per L1/L2/L3/NUMA/LOOP
1982 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_THREAD + 1] = 1;
1983 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L1 + 1] =
1984 __kmp_nThreadsPerCore;
1985 #if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS)
1986 if (__kmp_mic_type >= mic3)
1987 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L2 + 1] =
1988 2 * __kmp_nThreadsPerCore;
1989 else
1990 #endif // KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
1991 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L2 + 1] =
1992 __kmp_nThreadsPerCore;
1993 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L3 + 1] =
1994 nCoresPerPkg * __kmp_nThreadsPerCore;
1995 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_NUMA + 1] =
1996 nCoresPerPkg * __kmp_nThreadsPerCore;
1997 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_LOOP + 1] =
1998 nPackages * nCoresPerPkg * __kmp_nThreadsPerCore;
1999 }
2000
2001 // Return the index into the hierarchy for this tid and layer type (L1, L2, etc)
2002 // i.e., this thread's L1 or this thread's L2, etc.
__kmp_dispatch_get_index(int tid,kmp_hier_layer_e type)2003 int __kmp_dispatch_get_index(int tid, kmp_hier_layer_e type) {
2004 int index = type + 1;
2005 int num_hw_threads = __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1];
2006 KMP_DEBUG_ASSERT(type != kmp_hier_layer_e::LAYER_LAST);
2007 if (type == kmp_hier_layer_e::LAYER_THREAD)
2008 return tid;
2009 else if (type == kmp_hier_layer_e::LAYER_LOOP)
2010 return 0;
2011 KMP_DEBUG_ASSERT(__kmp_hier_max_units[index] != 0);
2012 if (tid >= num_hw_threads)
2013 tid = tid % num_hw_threads;
2014 return (tid / __kmp_hier_threads_per[index]) % __kmp_hier_max_units[index];
2015 }
2016
2017 // Return the number of t1's per t2
__kmp_dispatch_get_t1_per_t2(kmp_hier_layer_e t1,kmp_hier_layer_e t2)2018 int __kmp_dispatch_get_t1_per_t2(kmp_hier_layer_e t1, kmp_hier_layer_e t2) {
2019 int i1 = t1 + 1;
2020 int i2 = t2 + 1;
2021 KMP_DEBUG_ASSERT(i1 <= i2);
2022 KMP_DEBUG_ASSERT(t1 != kmp_hier_layer_e::LAYER_LAST);
2023 KMP_DEBUG_ASSERT(t2 != kmp_hier_layer_e::LAYER_LAST);
2024 KMP_DEBUG_ASSERT(__kmp_hier_threads_per[i1] != 0);
2025 // (nthreads/t2) / (nthreads/t1) = t1 / t2
2026 return __kmp_hier_threads_per[i2] / __kmp_hier_threads_per[i1];
2027 }
2028 #endif // KMP_USE_HIER_SCHED
2029
2030 // Parse /proc/cpuinfo (or an alternate file in the same format) to obtain the
2031 // affinity map.
__kmp_affinity_create_cpuinfo_map(AddrUnsPair ** address2os,int * line,kmp_i18n_id_t * const msg_id,FILE * f)2032 static int __kmp_affinity_create_cpuinfo_map(AddrUnsPair **address2os,
2033 int *line,
2034 kmp_i18n_id_t *const msg_id,
2035 FILE *f) {
2036 *address2os = NULL;
2037 *msg_id = kmp_i18n_null;
2038
2039 // Scan of the file, and count the number of "processor" (osId) fields,
2040 // and find the highest value of <n> for a node_<n> field.
2041 char buf[256];
2042 unsigned num_records = 0;
2043 while (!feof(f)) {
2044 buf[sizeof(buf) - 1] = 1;
2045 if (!fgets(buf, sizeof(buf), f)) {
2046 // Read errors presumably because of EOF
2047 break;
2048 }
2049
2050 char s1[] = "processor";
2051 if (strncmp(buf, s1, sizeof(s1) - 1) == 0) {
2052 num_records++;
2053 continue;
2054 }
2055
2056 // FIXME - this will match "node_<n> <garbage>"
2057 unsigned level;
2058 if (KMP_SSCANF(buf, "node_%u id", &level) == 1) {
2059 if (nodeIdIndex + level >= maxIndex) {
2060 maxIndex = nodeIdIndex + level;
2061 }
2062 continue;
2063 }
2064 }
2065
2066 // Check for empty file / no valid processor records, or too many. The number
2067 // of records can't exceed the number of valid bits in the affinity mask.
2068 if (num_records == 0) {
2069 *line = 0;
2070 *msg_id = kmp_i18n_str_NoProcRecords;
2071 return -1;
2072 }
2073 if (num_records > (unsigned)__kmp_xproc) {
2074 *line = 0;
2075 *msg_id = kmp_i18n_str_TooManyProcRecords;
2076 return -1;
2077 }
2078
2079 // Set the file pointer back to the beginning, so that we can scan the file
2080 // again, this time performing a full parse of the data. Allocate a vector of
2081 // ProcCpuInfo object, where we will place the data. Adding an extra element
2082 // at the end allows us to remove a lot of extra checks for termination
2083 // conditions.
2084 if (fseek(f, 0, SEEK_SET) != 0) {
2085 *line = 0;
2086 *msg_id = kmp_i18n_str_CantRewindCpuinfo;
2087 return -1;
2088 }
2089
2090 // Allocate the array of records to store the proc info in. The dummy
2091 // element at the end makes the logic in filling them out easier to code.
2092 unsigned **threadInfo =
2093 (unsigned **)__kmp_allocate((num_records + 1) * sizeof(unsigned *));
2094 unsigned i;
2095 for (i = 0; i <= num_records; i++) {
2096 threadInfo[i] =
2097 (unsigned *)__kmp_allocate((maxIndex + 1) * sizeof(unsigned));
2098 }
2099
2100 #define CLEANUP_THREAD_INFO \
2101 for (i = 0; i <= num_records; i++) { \
2102 __kmp_free(threadInfo[i]); \
2103 } \
2104 __kmp_free(threadInfo);
2105
2106 // A value of UINT_MAX means that we didn't find the field
2107 unsigned __index;
2108
2109 #define INIT_PROC_INFO(p) \
2110 for (__index = 0; __index <= maxIndex; __index++) { \
2111 (p)[__index] = UINT_MAX; \
2112 }
2113
2114 for (i = 0; i <= num_records; i++) {
2115 INIT_PROC_INFO(threadInfo[i]);
2116 }
2117
2118 unsigned num_avail = 0;
2119 *line = 0;
2120 while (!feof(f)) {
2121 // Create an inner scoping level, so that all the goto targets at the end of
2122 // the loop appear in an outer scoping level. This avoids warnings about
2123 // jumping past an initialization to a target in the same block.
2124 {
2125 buf[sizeof(buf) - 1] = 1;
2126 bool long_line = false;
2127 if (!fgets(buf, sizeof(buf), f)) {
2128 // Read errors presumably because of EOF
2129 // If there is valid data in threadInfo[num_avail], then fake
2130 // a blank line in ensure that the last address gets parsed.
2131 bool valid = false;
2132 for (i = 0; i <= maxIndex; i++) {
2133 if (threadInfo[num_avail][i] != UINT_MAX) {
2134 valid = true;
2135 }
2136 }
2137 if (!valid) {
2138 break;
2139 }
2140 buf[0] = 0;
2141 } else if (!buf[sizeof(buf) - 1]) {
2142 // The line is longer than the buffer. Set a flag and don't
2143 // emit an error if we were going to ignore the line, anyway.
2144 long_line = true;
2145
2146 #define CHECK_LINE \
2147 if (long_line) { \
2148 CLEANUP_THREAD_INFO; \
2149 *msg_id = kmp_i18n_str_LongLineCpuinfo; \
2150 return -1; \
2151 }
2152 }
2153 (*line)++;
2154
2155 char s1[] = "processor";
2156 if (strncmp(buf, s1, sizeof(s1) - 1) == 0) {
2157 CHECK_LINE;
2158 char *p = strchr(buf + sizeof(s1) - 1, ':');
2159 unsigned val;
2160 if ((p == NULL) || (KMP_SSCANF(p + 1, "%u\n", &val) != 1))
2161 goto no_val;
2162 if (threadInfo[num_avail][osIdIndex] != UINT_MAX)
2163 #if KMP_ARCH_AARCH64
2164 // Handle the old AArch64 /proc/cpuinfo layout differently,
2165 // it contains all of the 'processor' entries listed in a
2166 // single 'Processor' section, therefore the normal looking
2167 // for duplicates in that section will always fail.
2168 num_avail++;
2169 #else
2170 goto dup_field;
2171 #endif
2172 threadInfo[num_avail][osIdIndex] = val;
2173 #if KMP_OS_LINUX && !(KMP_ARCH_X86 || KMP_ARCH_X86_64)
2174 char path[256];
2175 KMP_SNPRINTF(
2176 path, sizeof(path),
2177 "/sys/devices/system/cpu/cpu%u/topology/physical_package_id",
2178 threadInfo[num_avail][osIdIndex]);
2179 __kmp_read_from_file(path, "%u", &threadInfo[num_avail][pkgIdIndex]);
2180
2181 KMP_SNPRINTF(path, sizeof(path),
2182 "/sys/devices/system/cpu/cpu%u/topology/core_id",
2183 threadInfo[num_avail][osIdIndex]);
2184 __kmp_read_from_file(path, "%u", &threadInfo[num_avail][coreIdIndex]);
2185 continue;
2186 #else
2187 }
2188 char s2[] = "physical id";
2189 if (strncmp(buf, s2, sizeof(s2) - 1) == 0) {
2190 CHECK_LINE;
2191 char *p = strchr(buf + sizeof(s2) - 1, ':');
2192 unsigned val;
2193 if ((p == NULL) || (KMP_SSCANF(p + 1, "%u\n", &val) != 1))
2194 goto no_val;
2195 if (threadInfo[num_avail][pkgIdIndex] != UINT_MAX)
2196 goto dup_field;
2197 threadInfo[num_avail][pkgIdIndex] = val;
2198 continue;
2199 }
2200 char s3[] = "core id";
2201 if (strncmp(buf, s3, sizeof(s3) - 1) == 0) {
2202 CHECK_LINE;
2203 char *p = strchr(buf + sizeof(s3) - 1, ':');
2204 unsigned val;
2205 if ((p == NULL) || (KMP_SSCANF(p + 1, "%u\n", &val) != 1))
2206 goto no_val;
2207 if (threadInfo[num_avail][coreIdIndex] != UINT_MAX)
2208 goto dup_field;
2209 threadInfo[num_avail][coreIdIndex] = val;
2210 continue;
2211 #endif // KMP_OS_LINUX && USE_SYSFS_INFO
2212 }
2213 char s4[] = "thread id";
2214 if (strncmp(buf, s4, sizeof(s4) - 1) == 0) {
2215 CHECK_LINE;
2216 char *p = strchr(buf + sizeof(s4) - 1, ':');
2217 unsigned val;
2218 if ((p == NULL) || (KMP_SSCANF(p + 1, "%u\n", &val) != 1))
2219 goto no_val;
2220 if (threadInfo[num_avail][threadIdIndex] != UINT_MAX)
2221 goto dup_field;
2222 threadInfo[num_avail][threadIdIndex] = val;
2223 continue;
2224 }
2225 unsigned level;
2226 if (KMP_SSCANF(buf, "node_%u id", &level) == 1) {
2227 CHECK_LINE;
2228 char *p = strchr(buf + sizeof(s4) - 1, ':');
2229 unsigned val;
2230 if ((p == NULL) || (KMP_SSCANF(p + 1, "%u\n", &val) != 1))
2231 goto no_val;
2232 KMP_ASSERT(nodeIdIndex + level <= maxIndex);
2233 if (threadInfo[num_avail][nodeIdIndex + level] != UINT_MAX)
2234 goto dup_field;
2235 threadInfo[num_avail][nodeIdIndex + level] = val;
2236 continue;
2237 }
2238
2239 // We didn't recognize the leading token on the line. There are lots of
2240 // leading tokens that we don't recognize - if the line isn't empty, go on
2241 // to the next line.
2242 if ((*buf != 0) && (*buf != '\n')) {
2243 // If the line is longer than the buffer, read characters
2244 // until we find a newline.
2245 if (long_line) {
2246 int ch;
2247 while (((ch = fgetc(f)) != EOF) && (ch != '\n'))
2248 ;
2249 }
2250 continue;
2251 }
2252
2253 // A newline has signalled the end of the processor record.
2254 // Check that there aren't too many procs specified.
2255 if ((int)num_avail == __kmp_xproc) {
2256 CLEANUP_THREAD_INFO;
2257 *msg_id = kmp_i18n_str_TooManyEntries;
2258 return -1;
2259 }
2260
2261 // Check for missing fields. The osId field must be there, and we
2262 // currently require that the physical id field is specified, also.
2263 if (threadInfo[num_avail][osIdIndex] == UINT_MAX) {
2264 CLEANUP_THREAD_INFO;
2265 *msg_id = kmp_i18n_str_MissingProcField;
2266 return -1;
2267 }
2268 if (threadInfo[0][pkgIdIndex] == UINT_MAX) {
2269 CLEANUP_THREAD_INFO;
2270 *msg_id = kmp_i18n_str_MissingPhysicalIDField;
2271 return -1;
2272 }
2273
2274 // Skip this proc if it is not included in the machine model.
2275 if (!KMP_CPU_ISSET(threadInfo[num_avail][osIdIndex],
2276 __kmp_affin_fullMask)) {
2277 INIT_PROC_INFO(threadInfo[num_avail]);
2278 continue;
2279 }
2280
2281 // We have a successful parse of this proc's info.
2282 // Increment the counter, and prepare for the next proc.
2283 num_avail++;
2284 KMP_ASSERT(num_avail <= num_records);
2285 INIT_PROC_INFO(threadInfo[num_avail]);
2286 }
2287 continue;
2288
2289 no_val:
2290 CLEANUP_THREAD_INFO;
2291 *msg_id = kmp_i18n_str_MissingValCpuinfo;
2292 return -1;
2293
2294 dup_field:
2295 CLEANUP_THREAD_INFO;
2296 *msg_id = kmp_i18n_str_DuplicateFieldCpuinfo;
2297 return -1;
2298 }
2299 *line = 0;
2300
2301 #if KMP_MIC && REDUCE_TEAM_SIZE
2302 unsigned teamSize = 0;
2303 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2304
2305 // check for num_records == __kmp_xproc ???
2306
2307 // If there's only one thread context to bind to, form an Address object with
2308 // depth 1 and return immediately (or, if affinity is off, set address2os to
2309 // NULL and return).
2310 //
2311 // If it is configured to omit the package level when there is only a single
2312 // package, the logic at the end of this routine won't work if there is only a
2313 // single thread - it would try to form an Address object with depth 0.
2314 KMP_ASSERT(num_avail > 0);
2315 KMP_ASSERT(num_avail <= num_records);
2316 if (num_avail == 1) {
2317 __kmp_ncores = 1;
2318 __kmp_nThreadsPerCore = nCoresPerPkg = nPackages = 1;
2319 if (__kmp_affinity_verbose) {
2320 if (!KMP_AFFINITY_CAPABLE()) {
2321 KMP_INFORM(AffNotCapableUseCpuinfo, "KMP_AFFINITY");
2322 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
2323 KMP_INFORM(Uniform, "KMP_AFFINITY");
2324 } else {
2325 char buf[KMP_AFFIN_MASK_PRINT_LEN];
2326 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
2327 __kmp_affin_fullMask);
2328 KMP_INFORM(AffCapableUseCpuinfo, "KMP_AFFINITY");
2329 if (__kmp_affinity_respect_mask) {
2330 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
2331 } else {
2332 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
2333 }
2334 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
2335 KMP_INFORM(Uniform, "KMP_AFFINITY");
2336 }
2337 int index;
2338 kmp_str_buf_t buf;
2339 __kmp_str_buf_init(&buf);
2340 __kmp_str_buf_print(&buf, "1");
2341 for (index = maxIndex - 1; index > pkgIdIndex; index--) {
2342 __kmp_str_buf_print(&buf, " x 1");
2343 }
2344 KMP_INFORM(TopologyExtra, "KMP_AFFINITY", buf.str, 1, 1, 1);
2345 __kmp_str_buf_free(&buf);
2346 }
2347
2348 if (__kmp_affinity_type == affinity_none) {
2349 CLEANUP_THREAD_INFO;
2350 return 0;
2351 }
2352
2353 *address2os = (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair));
2354 Address addr(1);
2355 addr.labels[0] = threadInfo[0][pkgIdIndex];
2356 (*address2os)[0] = AddrUnsPair(addr, threadInfo[0][osIdIndex]);
2357
2358 if (__kmp_affinity_gran_levels < 0) {
2359 __kmp_affinity_gran_levels = 0;
2360 }
2361
2362 if (__kmp_affinity_verbose) {
2363 __kmp_affinity_print_topology(*address2os, 1, 1, 0, -1, -1);
2364 }
2365
2366 CLEANUP_THREAD_INFO;
2367 return 1;
2368 }
2369
2370 // Sort the threadInfo table by physical Id.
2371 qsort(threadInfo, num_avail, sizeof(*threadInfo),
2372 __kmp_affinity_cmp_ProcCpuInfo_phys_id);
2373
2374 // The table is now sorted by pkgId / coreId / threadId, but we really don't
2375 // know the radix of any of the fields. pkgId's may be sparsely assigned among
2376 // the chips on a system. Although coreId's are usually assigned
2377 // [0 .. coresPerPkg-1] and threadId's are usually assigned
2378 // [0..threadsPerCore-1], we don't want to make any such assumptions.
2379 //
2380 // For that matter, we don't know what coresPerPkg and threadsPerCore (or the
2381 // total # packages) are at this point - we want to determine that now. We
2382 // only have an upper bound on the first two figures.
2383 unsigned *counts =
2384 (unsigned *)__kmp_allocate((maxIndex + 1) * sizeof(unsigned));
2385 unsigned *maxCt =
2386 (unsigned *)__kmp_allocate((maxIndex + 1) * sizeof(unsigned));
2387 unsigned *totals =
2388 (unsigned *)__kmp_allocate((maxIndex + 1) * sizeof(unsigned));
2389 unsigned *lastId =
2390 (unsigned *)__kmp_allocate((maxIndex + 1) * sizeof(unsigned));
2391
2392 bool assign_thread_ids = false;
2393 unsigned threadIdCt;
2394 unsigned index;
2395
2396 restart_radix_check:
2397 threadIdCt = 0;
2398
2399 // Initialize the counter arrays with data from threadInfo[0].
2400 if (assign_thread_ids) {
2401 if (threadInfo[0][threadIdIndex] == UINT_MAX) {
2402 threadInfo[0][threadIdIndex] = threadIdCt++;
2403 } else if (threadIdCt <= threadInfo[0][threadIdIndex]) {
2404 threadIdCt = threadInfo[0][threadIdIndex] + 1;
2405 }
2406 }
2407 for (index = 0; index <= maxIndex; index++) {
2408 counts[index] = 1;
2409 maxCt[index] = 1;
2410 totals[index] = 1;
2411 lastId[index] = threadInfo[0][index];
2412 ;
2413 }
2414
2415 // Run through the rest of the OS procs.
2416 for (i = 1; i < num_avail; i++) {
2417 // Find the most significant index whose id differs from the id for the
2418 // previous OS proc.
2419 for (index = maxIndex; index >= threadIdIndex; index--) {
2420 if (assign_thread_ids && (index == threadIdIndex)) {
2421 // Auto-assign the thread id field if it wasn't specified.
2422 if (threadInfo[i][threadIdIndex] == UINT_MAX) {
2423 threadInfo[i][threadIdIndex] = threadIdCt++;
2424 }
2425 // Apparently the thread id field was specified for some entries and not
2426 // others. Start the thread id counter off at the next higher thread id.
2427 else if (threadIdCt <= threadInfo[i][threadIdIndex]) {
2428 threadIdCt = threadInfo[i][threadIdIndex] + 1;
2429 }
2430 }
2431 if (threadInfo[i][index] != lastId[index]) {
2432 // Run through all indices which are less significant, and reset the
2433 // counts to 1. At all levels up to and including index, we need to
2434 // increment the totals and record the last id.
2435 unsigned index2;
2436 for (index2 = threadIdIndex; index2 < index; index2++) {
2437 totals[index2]++;
2438 if (counts[index2] > maxCt[index2]) {
2439 maxCt[index2] = counts[index2];
2440 }
2441 counts[index2] = 1;
2442 lastId[index2] = threadInfo[i][index2];
2443 }
2444 counts[index]++;
2445 totals[index]++;
2446 lastId[index] = threadInfo[i][index];
2447
2448 if (assign_thread_ids && (index > threadIdIndex)) {
2449
2450 #if KMP_MIC && REDUCE_TEAM_SIZE
2451 // The default team size is the total #threads in the machine
2452 // minus 1 thread for every core that has 3 or more threads.
2453 teamSize += (threadIdCt <= 2) ? (threadIdCt) : (threadIdCt - 1);
2454 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2455
2456 // Restart the thread counter, as we are on a new core.
2457 threadIdCt = 0;
2458
2459 // Auto-assign the thread id field if it wasn't specified.
2460 if (threadInfo[i][threadIdIndex] == UINT_MAX) {
2461 threadInfo[i][threadIdIndex] = threadIdCt++;
2462 }
2463
2464 // Apparently the thread id field was specified for some entries and
2465 // not others. Start the thread id counter off at the next higher
2466 // thread id.
2467 else if (threadIdCt <= threadInfo[i][threadIdIndex]) {
2468 threadIdCt = threadInfo[i][threadIdIndex] + 1;
2469 }
2470 }
2471 break;
2472 }
2473 }
2474 if (index < threadIdIndex) {
2475 // If thread ids were specified, it is an error if they are not unique.
2476 // Also, check that we waven't already restarted the loop (to be safe -
2477 // shouldn't need to).
2478 if ((threadInfo[i][threadIdIndex] != UINT_MAX) || assign_thread_ids) {
2479 __kmp_free(lastId);
2480 __kmp_free(totals);
2481 __kmp_free(maxCt);
2482 __kmp_free(counts);
2483 CLEANUP_THREAD_INFO;
2484 *msg_id = kmp_i18n_str_PhysicalIDsNotUnique;
2485 return -1;
2486 }
2487
2488 // If the thread ids were not specified and we see entries entries that
2489 // are duplicates, start the loop over and assign the thread ids manually.
2490 assign_thread_ids = true;
2491 goto restart_radix_check;
2492 }
2493 }
2494
2495 #if KMP_MIC && REDUCE_TEAM_SIZE
2496 // The default team size is the total #threads in the machine
2497 // minus 1 thread for every core that has 3 or more threads.
2498 teamSize += (threadIdCt <= 2) ? (threadIdCt) : (threadIdCt - 1);
2499 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2500
2501 for (index = threadIdIndex; index <= maxIndex; index++) {
2502 if (counts[index] > maxCt[index]) {
2503 maxCt[index] = counts[index];
2504 }
2505 }
2506
2507 __kmp_nThreadsPerCore = maxCt[threadIdIndex];
2508 nCoresPerPkg = maxCt[coreIdIndex];
2509 nPackages = totals[pkgIdIndex];
2510
2511 // Check to see if the machine topology is uniform
2512 unsigned prod = totals[maxIndex];
2513 for (index = threadIdIndex; index < maxIndex; index++) {
2514 prod *= maxCt[index];
2515 }
2516 bool uniform = (prod == totals[threadIdIndex]);
2517
2518 // When affinity is off, this routine will still be called to set
2519 // __kmp_ncores, as well as __kmp_nThreadsPerCore, nCoresPerPkg, & nPackages.
2520 // Make sure all these vars are set correctly, and return now if affinity is
2521 // not enabled.
2522 __kmp_ncores = totals[coreIdIndex];
2523
2524 if (__kmp_affinity_verbose) {
2525 if (!KMP_AFFINITY_CAPABLE()) {
2526 KMP_INFORM(AffNotCapableUseCpuinfo, "KMP_AFFINITY");
2527 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
2528 if (uniform) {
2529 KMP_INFORM(Uniform, "KMP_AFFINITY");
2530 } else {
2531 KMP_INFORM(NonUniform, "KMP_AFFINITY");
2532 }
2533 } else {
2534 char buf[KMP_AFFIN_MASK_PRINT_LEN];
2535 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
2536 __kmp_affin_fullMask);
2537 KMP_INFORM(AffCapableUseCpuinfo, "KMP_AFFINITY");
2538 if (__kmp_affinity_respect_mask) {
2539 KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
2540 } else {
2541 KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
2542 }
2543 KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
2544 if (uniform) {
2545 KMP_INFORM(Uniform, "KMP_AFFINITY");
2546 } else {
2547 KMP_INFORM(NonUniform, "KMP_AFFINITY");
2548 }
2549 }
2550 kmp_str_buf_t buf;
2551 __kmp_str_buf_init(&buf);
2552
2553 __kmp_str_buf_print(&buf, "%d", totals[maxIndex]);
2554 for (index = maxIndex - 1; index >= pkgIdIndex; index--) {
2555 __kmp_str_buf_print(&buf, " x %d", maxCt[index]);
2556 }
2557 KMP_INFORM(TopologyExtra, "KMP_AFFINITY", buf.str, maxCt[coreIdIndex],
2558 maxCt[threadIdIndex], __kmp_ncores);
2559
2560 __kmp_str_buf_free(&buf);
2561 }
2562
2563 #if KMP_MIC && REDUCE_TEAM_SIZE
2564 // Set the default team size.
2565 if ((__kmp_dflt_team_nth == 0) && (teamSize > 0)) {
2566 __kmp_dflt_team_nth = teamSize;
2567 KA_TRACE(20, ("__kmp_affinity_create_cpuinfo_map: setting "
2568 "__kmp_dflt_team_nth = %d\n",
2569 __kmp_dflt_team_nth));
2570 }
2571 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2572
2573 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
2574 KMP_DEBUG_ASSERT(num_avail == (unsigned)__kmp_avail_proc);
2575 __kmp_pu_os_idx = (int *)__kmp_allocate(sizeof(int) * __kmp_avail_proc);
2576 for (i = 0; i < num_avail; ++i) { // fill the os indices
2577 __kmp_pu_os_idx[i] = threadInfo[i][osIdIndex];
2578 }
2579
2580 if (__kmp_affinity_type == affinity_none) {
2581 __kmp_free(lastId);
2582 __kmp_free(totals);
2583 __kmp_free(maxCt);
2584 __kmp_free(counts);
2585 CLEANUP_THREAD_INFO;
2586 return 0;
2587 }
2588
2589 // Count the number of levels which have more nodes at that level than at the
2590 // parent's level (with there being an implicit root node of the top level).
2591 // This is equivalent to saying that there is at least one node at this level
2592 // which has a sibling. These levels are in the map, and the package level is
2593 // always in the map.
2594 bool *inMap = (bool *)__kmp_allocate((maxIndex + 1) * sizeof(bool));
2595 for (index = threadIdIndex; index < maxIndex; index++) {
2596 KMP_ASSERT(totals[index] >= totals[index + 1]);
2597 inMap[index] = (totals[index] > totals[index + 1]);
2598 }
2599 inMap[maxIndex] = (totals[maxIndex] > 1);
2600 inMap[pkgIdIndex] = true;
2601
2602 int depth = 0;
2603 for (index = threadIdIndex; index <= maxIndex; index++) {
2604 if (inMap[index]) {
2605 depth++;
2606 }
2607 }
2608 KMP_ASSERT(depth > 0);
2609
2610 // Construct the data structure that is to be returned.
2611 *address2os = (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair) * num_avail);
2612 int pkgLevel = -1;
2613 int coreLevel = -1;
2614 int threadLevel = -1;
2615
2616 for (i = 0; i < num_avail; ++i) {
2617 Address addr(depth);
2618 unsigned os = threadInfo[i][osIdIndex];
2619 int src_index;
2620 int dst_index = 0;
2621
2622 for (src_index = maxIndex; src_index >= threadIdIndex; src_index--) {
2623 if (!inMap[src_index]) {
2624 continue;
2625 }
2626 addr.labels[dst_index] = threadInfo[i][src_index];
2627 if (src_index == pkgIdIndex) {
2628 pkgLevel = dst_index;
2629 } else if (src_index == coreIdIndex) {
2630 coreLevel = dst_index;
2631 } else if (src_index == threadIdIndex) {
2632 threadLevel = dst_index;
2633 }
2634 dst_index++;
2635 }
2636 (*address2os)[i] = AddrUnsPair(addr, os);
2637 }
2638
2639 if (__kmp_affinity_gran_levels < 0) {
2640 // Set the granularity level based on what levels are modeled
2641 // in the machine topology map.
2642 unsigned src_index;
2643 __kmp_affinity_gran_levels = 0;
2644 for (src_index = threadIdIndex; src_index <= maxIndex; src_index++) {
2645 if (!inMap[src_index]) {
2646 continue;
2647 }
2648 switch (src_index) {
2649 case threadIdIndex:
2650 if (__kmp_affinity_gran > affinity_gran_thread) {
2651 __kmp_affinity_gran_levels++;
2652 }
2653
2654 break;
2655 case coreIdIndex:
2656 if (__kmp_affinity_gran > affinity_gran_core) {
2657 __kmp_affinity_gran_levels++;
2658 }
2659 break;
2660
2661 case pkgIdIndex:
2662 if (__kmp_affinity_gran > affinity_gran_package) {
2663 __kmp_affinity_gran_levels++;
2664 }
2665 break;
2666 }
2667 }
2668 }
2669
2670 if (__kmp_affinity_verbose) {
2671 __kmp_affinity_print_topology(*address2os, num_avail, depth, pkgLevel,
2672 coreLevel, threadLevel);
2673 }
2674
2675 __kmp_free(inMap);
2676 __kmp_free(lastId);
2677 __kmp_free(totals);
2678 __kmp_free(maxCt);
2679 __kmp_free(counts);
2680 CLEANUP_THREAD_INFO;
2681 return depth;
2682 }
2683
2684 // Create and return a table of affinity masks, indexed by OS thread ID.
2685 // This routine handles OR'ing together all the affinity masks of threads
2686 // that are sufficiently close, if granularity > fine.
__kmp_create_masks(unsigned * maxIndex,unsigned * numUnique,AddrUnsPair * address2os,unsigned numAddrs)2687 static kmp_affin_mask_t *__kmp_create_masks(unsigned *maxIndex,
2688 unsigned *numUnique,
2689 AddrUnsPair *address2os,
2690 unsigned numAddrs) {
2691 // First form a table of affinity masks in order of OS thread id.
2692 unsigned depth;
2693 unsigned maxOsId;
2694 unsigned i;
2695
2696 KMP_ASSERT(numAddrs > 0);
2697 depth = address2os[0].first.depth;
2698
2699 maxOsId = 0;
2700 for (i = numAddrs - 1;; --i) {
2701 unsigned osId = address2os[i].second;
2702 if (osId > maxOsId) {
2703 maxOsId = osId;
2704 }
2705 if (i == 0)
2706 break;
2707 }
2708 kmp_affin_mask_t *osId2Mask;
2709 KMP_CPU_ALLOC_ARRAY(osId2Mask, (maxOsId + 1));
2710
2711 // Sort the address2os table according to physical order. Doing so will put
2712 // all threads on the same core/package/node in consecutive locations.
2713 qsort(address2os, numAddrs, sizeof(*address2os),
2714 __kmp_affinity_cmp_Address_labels);
2715
2716 KMP_ASSERT(__kmp_affinity_gran_levels >= 0);
2717 if (__kmp_affinity_verbose && (__kmp_affinity_gran_levels > 0)) {
2718 KMP_INFORM(ThreadsMigrate, "KMP_AFFINITY", __kmp_affinity_gran_levels);
2719 }
2720 if (__kmp_affinity_gran_levels >= (int)depth) {
2721 if (__kmp_affinity_verbose ||
2722 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
2723 KMP_WARNING(AffThreadsMayMigrate);
2724 }
2725 }
2726
2727 // Run through the table, forming the masks for all threads on each core.
2728 // Threads on the same core will have identical "Address" objects, not
2729 // considering the last level, which must be the thread id. All threads on a
2730 // core will appear consecutively.
2731 unsigned unique = 0;
2732 unsigned j = 0; // index of 1st thread on core
2733 unsigned leader = 0;
2734 Address *leaderAddr = &(address2os[0].first);
2735 kmp_affin_mask_t *sum;
2736 KMP_CPU_ALLOC_ON_STACK(sum);
2737 KMP_CPU_ZERO(sum);
2738 KMP_CPU_SET(address2os[0].second, sum);
2739 for (i = 1; i < numAddrs; i++) {
2740 // If this thread is sufficiently close to the leader (within the
2741 // granularity setting), then set the bit for this os thread in the
2742 // affinity mask for this group, and go on to the next thread.
2743 if (leaderAddr->isClose(address2os[i].first, __kmp_affinity_gran_levels)) {
2744 KMP_CPU_SET(address2os[i].second, sum);
2745 continue;
2746 }
2747
2748 // For every thread in this group, copy the mask to the thread's entry in
2749 // the osId2Mask table. Mark the first address as a leader.
2750 for (; j < i; j++) {
2751 unsigned osId = address2os[j].second;
2752 KMP_DEBUG_ASSERT(osId <= maxOsId);
2753 kmp_affin_mask_t *mask = KMP_CPU_INDEX(osId2Mask, osId);
2754 KMP_CPU_COPY(mask, sum);
2755 address2os[j].first.leader = (j == leader);
2756 }
2757 unique++;
2758
2759 // Start a new mask.
2760 leader = i;
2761 leaderAddr = &(address2os[i].first);
2762 KMP_CPU_ZERO(sum);
2763 KMP_CPU_SET(address2os[i].second, sum);
2764 }
2765
2766 // For every thread in last group, copy the mask to the thread's
2767 // entry in the osId2Mask table.
2768 for (; j < i; j++) {
2769 unsigned osId = address2os[j].second;
2770 KMP_DEBUG_ASSERT(osId <= maxOsId);
2771 kmp_affin_mask_t *mask = KMP_CPU_INDEX(osId2Mask, osId);
2772 KMP_CPU_COPY(mask, sum);
2773 address2os[j].first.leader = (j == leader);
2774 }
2775 unique++;
2776 KMP_CPU_FREE_FROM_STACK(sum);
2777
2778 *maxIndex = maxOsId;
2779 *numUnique = unique;
2780 return osId2Mask;
2781 }
2782
2783 // Stuff for the affinity proclist parsers. It's easier to declare these vars
2784 // as file-static than to try and pass them through the calling sequence of
2785 // the recursive-descent OMP_PLACES parser.
2786 static kmp_affin_mask_t *newMasks;
2787 static int numNewMasks;
2788 static int nextNewMask;
2789
2790 #define ADD_MASK(_mask) \
2791 { \
2792 if (nextNewMask >= numNewMasks) { \
2793 int i; \
2794 numNewMasks *= 2; \
2795 kmp_affin_mask_t *temp; \
2796 KMP_CPU_INTERNAL_ALLOC_ARRAY(temp, numNewMasks); \
2797 for (i = 0; i < numNewMasks / 2; i++) { \
2798 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i); \
2799 kmp_affin_mask_t *dest = KMP_CPU_INDEX(temp, i); \
2800 KMP_CPU_COPY(dest, src); \
2801 } \
2802 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks / 2); \
2803 newMasks = temp; \
2804 } \
2805 KMP_CPU_COPY(KMP_CPU_INDEX(newMasks, nextNewMask), (_mask)); \
2806 nextNewMask++; \
2807 }
2808
2809 #define ADD_MASK_OSID(_osId, _osId2Mask, _maxOsId) \
2810 { \
2811 if (((_osId) > _maxOsId) || \
2812 (!KMP_CPU_ISSET((_osId), KMP_CPU_INDEX((_osId2Mask), (_osId))))) { \
2813 if (__kmp_affinity_verbose || \
2814 (__kmp_affinity_warnings && \
2815 (__kmp_affinity_type != affinity_none))) { \
2816 KMP_WARNING(AffIgnoreInvalidProcID, _osId); \
2817 } \
2818 } else { \
2819 ADD_MASK(KMP_CPU_INDEX(_osId2Mask, (_osId))); \
2820 } \
2821 }
2822
2823 // Re-parse the proclist (for the explicit affinity type), and form the list
2824 // of affinity newMasks indexed by gtid.
__kmp_affinity_process_proclist(kmp_affin_mask_t ** out_masks,unsigned int * out_numMasks,const char * proclist,kmp_affin_mask_t * osId2Mask,int maxOsId)2825 static void __kmp_affinity_process_proclist(kmp_affin_mask_t **out_masks,
2826 unsigned int *out_numMasks,
2827 const char *proclist,
2828 kmp_affin_mask_t *osId2Mask,
2829 int maxOsId) {
2830 int i;
2831 const char *scan = proclist;
2832 const char *next = proclist;
2833
2834 // We use malloc() for the temporary mask vector, so that we can use
2835 // realloc() to extend it.
2836 numNewMasks = 2;
2837 KMP_CPU_INTERNAL_ALLOC_ARRAY(newMasks, numNewMasks);
2838 nextNewMask = 0;
2839 kmp_affin_mask_t *sumMask;
2840 KMP_CPU_ALLOC(sumMask);
2841 int setSize = 0;
2842
2843 for (;;) {
2844 int start, end, stride;
2845
2846 SKIP_WS(scan);
2847 next = scan;
2848 if (*next == '\0') {
2849 break;
2850 }
2851
2852 if (*next == '{') {
2853 int num;
2854 setSize = 0;
2855 next++; // skip '{'
2856 SKIP_WS(next);
2857 scan = next;
2858
2859 // Read the first integer in the set.
2860 KMP_ASSERT2((*next >= '0') && (*next <= '9'), "bad proclist");
2861 SKIP_DIGITS(next);
2862 num = __kmp_str_to_int(scan, *next);
2863 KMP_ASSERT2(num >= 0, "bad explicit proc list");
2864
2865 // Copy the mask for that osId to the sum (union) mask.
2866 if ((num > maxOsId) ||
2867 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
2868 if (__kmp_affinity_verbose ||
2869 (__kmp_affinity_warnings &&
2870 (__kmp_affinity_type != affinity_none))) {
2871 KMP_WARNING(AffIgnoreInvalidProcID, num);
2872 }
2873 KMP_CPU_ZERO(sumMask);
2874 } else {
2875 KMP_CPU_COPY(sumMask, KMP_CPU_INDEX(osId2Mask, num));
2876 setSize = 1;
2877 }
2878
2879 for (;;) {
2880 // Check for end of set.
2881 SKIP_WS(next);
2882 if (*next == '}') {
2883 next++; // skip '}'
2884 break;
2885 }
2886
2887 // Skip optional comma.
2888 if (*next == ',') {
2889 next++;
2890 }
2891 SKIP_WS(next);
2892
2893 // Read the next integer in the set.
2894 scan = next;
2895 KMP_ASSERT2((*next >= '0') && (*next <= '9'), "bad explicit proc list");
2896
2897 SKIP_DIGITS(next);
2898 num = __kmp_str_to_int(scan, *next);
2899 KMP_ASSERT2(num >= 0, "bad explicit proc list");
2900
2901 // Add the mask for that osId to the sum mask.
2902 if ((num > maxOsId) ||
2903 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
2904 if (__kmp_affinity_verbose ||
2905 (__kmp_affinity_warnings &&
2906 (__kmp_affinity_type != affinity_none))) {
2907 KMP_WARNING(AffIgnoreInvalidProcID, num);
2908 }
2909 } else {
2910 KMP_CPU_UNION(sumMask, KMP_CPU_INDEX(osId2Mask, num));
2911 setSize++;
2912 }
2913 }
2914 if (setSize > 0) {
2915 ADD_MASK(sumMask);
2916 }
2917
2918 SKIP_WS(next);
2919 if (*next == ',') {
2920 next++;
2921 }
2922 scan = next;
2923 continue;
2924 }
2925
2926 // Read the first integer.
2927 KMP_ASSERT2((*next >= '0') && (*next <= '9'), "bad explicit proc list");
2928 SKIP_DIGITS(next);
2929 start = __kmp_str_to_int(scan, *next);
2930 KMP_ASSERT2(start >= 0, "bad explicit proc list");
2931 SKIP_WS(next);
2932
2933 // If this isn't a range, then add a mask to the list and go on.
2934 if (*next != '-') {
2935 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2936
2937 // Skip optional comma.
2938 if (*next == ',') {
2939 next++;
2940 }
2941 scan = next;
2942 continue;
2943 }
2944
2945 // This is a range. Skip over the '-' and read in the 2nd int.
2946 next++; // skip '-'
2947 SKIP_WS(next);
2948 scan = next;
2949 KMP_ASSERT2((*next >= '0') && (*next <= '9'), "bad explicit proc list");
2950 SKIP_DIGITS(next);
2951 end = __kmp_str_to_int(scan, *next);
2952 KMP_ASSERT2(end >= 0, "bad explicit proc list");
2953
2954 // Check for a stride parameter
2955 stride = 1;
2956 SKIP_WS(next);
2957 if (*next == ':') {
2958 // A stride is specified. Skip over the ':" and read the 3rd int.
2959 int sign = +1;
2960 next++; // skip ':'
2961 SKIP_WS(next);
2962 scan = next;
2963 if (*next == '-') {
2964 sign = -1;
2965 next++;
2966 SKIP_WS(next);
2967 scan = next;
2968 }
2969 KMP_ASSERT2((*next >= '0') && (*next <= '9'), "bad explicit proc list");
2970 SKIP_DIGITS(next);
2971 stride = __kmp_str_to_int(scan, *next);
2972 KMP_ASSERT2(stride >= 0, "bad explicit proc list");
2973 stride *= sign;
2974 }
2975
2976 // Do some range checks.
2977 KMP_ASSERT2(stride != 0, "bad explicit proc list");
2978 if (stride > 0) {
2979 KMP_ASSERT2(start <= end, "bad explicit proc list");
2980 } else {
2981 KMP_ASSERT2(start >= end, "bad explicit proc list");
2982 }
2983 KMP_ASSERT2((end - start) / stride <= 65536, "bad explicit proc list");
2984
2985 // Add the mask for each OS proc # to the list.
2986 if (stride > 0) {
2987 do {
2988 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2989 start += stride;
2990 } while (start <= end);
2991 } else {
2992 do {
2993 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2994 start += stride;
2995 } while (start >= end);
2996 }
2997
2998 // Skip optional comma.
2999 SKIP_WS(next);
3000 if (*next == ',') {
3001 next++;
3002 }
3003 scan = next;
3004 }
3005
3006 *out_numMasks = nextNewMask;
3007 if (nextNewMask == 0) {
3008 *out_masks = NULL;
3009 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3010 return;
3011 }
3012 KMP_CPU_ALLOC_ARRAY((*out_masks), nextNewMask);
3013 for (i = 0; i < nextNewMask; i++) {
3014 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i);
3015 kmp_affin_mask_t *dest = KMP_CPU_INDEX((*out_masks), i);
3016 KMP_CPU_COPY(dest, src);
3017 }
3018 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3019 KMP_CPU_FREE(sumMask);
3020 }
3021
3022 /*-----------------------------------------------------------------------------
3023 Re-parse the OMP_PLACES proc id list, forming the newMasks for the different
3024 places. Again, Here is the grammar:
3025
3026 place_list := place
3027 place_list := place , place_list
3028 place := num
3029 place := place : num
3030 place := place : num : signed
3031 place := { subplacelist }
3032 place := ! place // (lowest priority)
3033 subplace_list := subplace
3034 subplace_list := subplace , subplace_list
3035 subplace := num
3036 subplace := num : num
3037 subplace := num : num : signed
3038 signed := num
3039 signed := + signed
3040 signed := - signed
3041 -----------------------------------------------------------------------------*/
__kmp_process_subplace_list(const char ** scan,kmp_affin_mask_t * osId2Mask,int maxOsId,kmp_affin_mask_t * tempMask,int * setSize)3042 static void __kmp_process_subplace_list(const char **scan,
3043 kmp_affin_mask_t *osId2Mask,
3044 int maxOsId, kmp_affin_mask_t *tempMask,
3045 int *setSize) {
3046 const char *next;
3047
3048 for (;;) {
3049 int start, count, stride, i;
3050
3051 // Read in the starting proc id
3052 SKIP_WS(*scan);
3053 KMP_ASSERT2((**scan >= '0') && (**scan <= '9'), "bad explicit places list");
3054 next = *scan;
3055 SKIP_DIGITS(next);
3056 start = __kmp_str_to_int(*scan, *next);
3057 KMP_ASSERT(start >= 0);
3058 *scan = next;
3059
3060 // valid follow sets are ',' ':' and '}'
3061 SKIP_WS(*scan);
3062 if (**scan == '}' || **scan == ',') {
3063 if ((start > maxOsId) ||
3064 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
3065 if (__kmp_affinity_verbose ||
3066 (__kmp_affinity_warnings &&
3067 (__kmp_affinity_type != affinity_none))) {
3068 KMP_WARNING(AffIgnoreInvalidProcID, start);
3069 }
3070 } else {
3071 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
3072 (*setSize)++;
3073 }
3074 if (**scan == '}') {
3075 break;
3076 }
3077 (*scan)++; // skip ','
3078 continue;
3079 }
3080 KMP_ASSERT2(**scan == ':', "bad explicit places list");
3081 (*scan)++; // skip ':'
3082
3083 // Read count parameter
3084 SKIP_WS(*scan);
3085 KMP_ASSERT2((**scan >= '0') && (**scan <= '9'), "bad explicit places list");
3086 next = *scan;
3087 SKIP_DIGITS(next);
3088 count = __kmp_str_to_int(*scan, *next);
3089 KMP_ASSERT(count >= 0);
3090 *scan = next;
3091
3092 // valid follow sets are ',' ':' and '}'
3093 SKIP_WS(*scan);
3094 if (**scan == '}' || **scan == ',') {
3095 for (i = 0; i < count; i++) {
3096 if ((start > maxOsId) ||
3097 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
3098 if (__kmp_affinity_verbose ||
3099 (__kmp_affinity_warnings &&
3100 (__kmp_affinity_type != affinity_none))) {
3101 KMP_WARNING(AffIgnoreInvalidProcID, start);
3102 }
3103 break; // don't proliferate warnings for large count
3104 } else {
3105 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
3106 start++;
3107 (*setSize)++;
3108 }
3109 }
3110 if (**scan == '}') {
3111 break;
3112 }
3113 (*scan)++; // skip ','
3114 continue;
3115 }
3116 KMP_ASSERT2(**scan == ':', "bad explicit places list");
3117 (*scan)++; // skip ':'
3118
3119 // Read stride parameter
3120 int sign = +1;
3121 for (;;) {
3122 SKIP_WS(*scan);
3123 if (**scan == '+') {
3124 (*scan)++; // skip '+'
3125 continue;
3126 }
3127 if (**scan == '-') {
3128 sign *= -1;
3129 (*scan)++; // skip '-'
3130 continue;
3131 }
3132 break;
3133 }
3134 SKIP_WS(*scan);
3135 KMP_ASSERT2((**scan >= '0') && (**scan <= '9'), "bad explicit places list");
3136 next = *scan;
3137 SKIP_DIGITS(next);
3138 stride = __kmp_str_to_int(*scan, *next);
3139 KMP_ASSERT(stride >= 0);
3140 *scan = next;
3141 stride *= sign;
3142
3143 // valid follow sets are ',' and '}'
3144 SKIP_WS(*scan);
3145 if (**scan == '}' || **scan == ',') {
3146 for (i = 0; i < count; i++) {
3147 if ((start > maxOsId) ||
3148 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
3149 if (__kmp_affinity_verbose ||
3150 (__kmp_affinity_warnings &&
3151 (__kmp_affinity_type != affinity_none))) {
3152 KMP_WARNING(AffIgnoreInvalidProcID, start);
3153 }
3154 break; // don't proliferate warnings for large count
3155 } else {
3156 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
3157 start += stride;
3158 (*setSize)++;
3159 }
3160 }
3161 if (**scan == '}') {
3162 break;
3163 }
3164 (*scan)++; // skip ','
3165 continue;
3166 }
3167
3168 KMP_ASSERT2(0, "bad explicit places list");
3169 }
3170 }
3171
__kmp_process_place(const char ** scan,kmp_affin_mask_t * osId2Mask,int maxOsId,kmp_affin_mask_t * tempMask,int * setSize)3172 static void __kmp_process_place(const char **scan, kmp_affin_mask_t *osId2Mask,
3173 int maxOsId, kmp_affin_mask_t *tempMask,
3174 int *setSize) {
3175 const char *next;
3176
3177 // valid follow sets are '{' '!' and num
3178 SKIP_WS(*scan);
3179 if (**scan == '{') {
3180 (*scan)++; // skip '{'
3181 __kmp_process_subplace_list(scan, osId2Mask, maxOsId, tempMask, setSize);
3182 KMP_ASSERT2(**scan == '}', "bad explicit places list");
3183 (*scan)++; // skip '}'
3184 } else if (**scan == '!') {
3185 (*scan)++; // skip '!'
3186 __kmp_process_place(scan, osId2Mask, maxOsId, tempMask, setSize);
3187 KMP_CPU_COMPLEMENT(maxOsId, tempMask);
3188 } else if ((**scan >= '0') && (**scan <= '9')) {
3189 next = *scan;
3190 SKIP_DIGITS(next);
3191 int num = __kmp_str_to_int(*scan, *next);
3192 KMP_ASSERT(num >= 0);
3193 if ((num > maxOsId) ||
3194 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
3195 if (__kmp_affinity_verbose ||
3196 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
3197 KMP_WARNING(AffIgnoreInvalidProcID, num);
3198 }
3199 } else {
3200 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, num));
3201 (*setSize)++;
3202 }
3203 *scan = next; // skip num
3204 } else {
3205 KMP_ASSERT2(0, "bad explicit places list");
3206 }
3207 }
3208
3209 // static void
__kmp_affinity_process_placelist(kmp_affin_mask_t ** out_masks,unsigned int * out_numMasks,const char * placelist,kmp_affin_mask_t * osId2Mask,int maxOsId)3210 void __kmp_affinity_process_placelist(kmp_affin_mask_t **out_masks,
3211 unsigned int *out_numMasks,
3212 const char *placelist,
3213 kmp_affin_mask_t *osId2Mask,
3214 int maxOsId) {
3215 int i, j, count, stride, sign;
3216 const char *scan = placelist;
3217 const char *next = placelist;
3218
3219 numNewMasks = 2;
3220 KMP_CPU_INTERNAL_ALLOC_ARRAY(newMasks, numNewMasks);
3221 nextNewMask = 0;
3222
3223 // tempMask is modified based on the previous or initial
3224 // place to form the current place
3225 // previousMask contains the previous place
3226 kmp_affin_mask_t *tempMask;
3227 kmp_affin_mask_t *previousMask;
3228 KMP_CPU_ALLOC(tempMask);
3229 KMP_CPU_ZERO(tempMask);
3230 KMP_CPU_ALLOC(previousMask);
3231 KMP_CPU_ZERO(previousMask);
3232 int setSize = 0;
3233
3234 for (;;) {
3235 __kmp_process_place(&scan, osId2Mask, maxOsId, tempMask, &setSize);
3236
3237 // valid follow sets are ',' ':' and EOL
3238 SKIP_WS(scan);
3239 if (*scan == '\0' || *scan == ',') {
3240 if (setSize > 0) {
3241 ADD_MASK(tempMask);
3242 }
3243 KMP_CPU_ZERO(tempMask);
3244 setSize = 0;
3245 if (*scan == '\0') {
3246 break;
3247 }
3248 scan++; // skip ','
3249 continue;
3250 }
3251
3252 KMP_ASSERT2(*scan == ':', "bad explicit places list");
3253 scan++; // skip ':'
3254
3255 // Read count parameter
3256 SKIP_WS(scan);
3257 KMP_ASSERT2((*scan >= '0') && (*scan <= '9'), "bad explicit places list");
3258 next = scan;
3259 SKIP_DIGITS(next);
3260 count = __kmp_str_to_int(scan, *next);
3261 KMP_ASSERT(count >= 0);
3262 scan = next;
3263
3264 // valid follow sets are ',' ':' and EOL
3265 SKIP_WS(scan);
3266 if (*scan == '\0' || *scan == ',') {
3267 stride = +1;
3268 } else {
3269 KMP_ASSERT2(*scan == ':', "bad explicit places list");
3270 scan++; // skip ':'
3271
3272 // Read stride parameter
3273 sign = +1;
3274 for (;;) {
3275 SKIP_WS(scan);
3276 if (*scan == '+') {
3277 scan++; // skip '+'
3278 continue;
3279 }
3280 if (*scan == '-') {
3281 sign *= -1;
3282 scan++; // skip '-'
3283 continue;
3284 }
3285 break;
3286 }
3287 SKIP_WS(scan);
3288 KMP_ASSERT2((*scan >= '0') && (*scan <= '9'), "bad explicit places list");
3289 next = scan;
3290 SKIP_DIGITS(next);
3291 stride = __kmp_str_to_int(scan, *next);
3292 KMP_DEBUG_ASSERT(stride >= 0);
3293 scan = next;
3294 stride *= sign;
3295 }
3296
3297 // Add places determined by initial_place : count : stride
3298 for (i = 0; i < count; i++) {
3299 if (setSize == 0) {
3300 break;
3301 }
3302 // Add the current place, then build the next place (tempMask) from that
3303 KMP_CPU_COPY(previousMask, tempMask);
3304 ADD_MASK(previousMask);
3305 KMP_CPU_ZERO(tempMask);
3306 setSize = 0;
3307 KMP_CPU_SET_ITERATE(j, previousMask) {
3308 if (!KMP_CPU_ISSET(j, previousMask)) {
3309 continue;
3310 }
3311 if ((j + stride > maxOsId) || (j + stride < 0) ||
3312 (!KMP_CPU_ISSET(j, __kmp_affin_fullMask)) ||
3313 (!KMP_CPU_ISSET(j + stride,
3314 KMP_CPU_INDEX(osId2Mask, j + stride)))) {
3315 if ((__kmp_affinity_verbose ||
3316 (__kmp_affinity_warnings &&
3317 (__kmp_affinity_type != affinity_none))) &&
3318 i < count - 1) {
3319 KMP_WARNING(AffIgnoreInvalidProcID, j + stride);
3320 }
3321 continue;
3322 }
3323 KMP_CPU_SET(j + stride, tempMask);
3324 setSize++;
3325 }
3326 }
3327 KMP_CPU_ZERO(tempMask);
3328 setSize = 0;
3329
3330 // valid follow sets are ',' and EOL
3331 SKIP_WS(scan);
3332 if (*scan == '\0') {
3333 break;
3334 }
3335 if (*scan == ',') {
3336 scan++; // skip ','
3337 continue;
3338 }
3339
3340 KMP_ASSERT2(0, "bad explicit places list");
3341 }
3342
3343 *out_numMasks = nextNewMask;
3344 if (nextNewMask == 0) {
3345 *out_masks = NULL;
3346 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3347 return;
3348 }
3349 KMP_CPU_ALLOC_ARRAY((*out_masks), nextNewMask);
3350 KMP_CPU_FREE(tempMask);
3351 KMP_CPU_FREE(previousMask);
3352 for (i = 0; i < nextNewMask; i++) {
3353 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i);
3354 kmp_affin_mask_t *dest = KMP_CPU_INDEX((*out_masks), i);
3355 KMP_CPU_COPY(dest, src);
3356 }
3357 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3358 }
3359
3360 #undef ADD_MASK
3361 #undef ADD_MASK_OSID
3362
3363 #if KMP_USE_HWLOC
__kmp_hwloc_skip_PUs_obj(hwloc_topology_t t,hwloc_obj_t o)3364 static int __kmp_hwloc_skip_PUs_obj(hwloc_topology_t t, hwloc_obj_t o) {
3365 // skip PUs descendants of the object o
3366 int skipped = 0;
3367 hwloc_obj_t hT = NULL;
3368 int N = __kmp_hwloc_count_children_by_type(t, o, HWLOC_OBJ_PU, &hT);
3369 for (int i = 0; i < N; ++i) {
3370 KMP_DEBUG_ASSERT(hT);
3371 unsigned idx = hT->os_index;
3372 if (KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3373 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3374 KC_TRACE(200, ("KMP_HW_SUBSET: skipped proc %d\n", idx));
3375 ++skipped;
3376 }
3377 hT = hwloc_get_next_obj_by_type(t, HWLOC_OBJ_PU, hT);
3378 }
3379 return skipped; // count number of skipped units
3380 }
3381
__kmp_hwloc_obj_has_PUs(hwloc_topology_t t,hwloc_obj_t o)3382 static int __kmp_hwloc_obj_has_PUs(hwloc_topology_t t, hwloc_obj_t o) {
3383 // check if obj has PUs present in fullMask
3384 hwloc_obj_t hT = NULL;
3385 int N = __kmp_hwloc_count_children_by_type(t, o, HWLOC_OBJ_PU, &hT);
3386 for (int i = 0; i < N; ++i) {
3387 KMP_DEBUG_ASSERT(hT);
3388 unsigned idx = hT->os_index;
3389 if (KMP_CPU_ISSET(idx, __kmp_affin_fullMask))
3390 return 1; // found PU
3391 hT = hwloc_get_next_obj_by_type(t, HWLOC_OBJ_PU, hT);
3392 }
3393 return 0; // no PUs found
3394 }
3395 #endif // KMP_USE_HWLOC
3396
__kmp_apply_thread_places(AddrUnsPair ** pAddr,int depth)3397 static void __kmp_apply_thread_places(AddrUnsPair **pAddr, int depth) {
3398 AddrUnsPair *newAddr;
3399 if (__kmp_hws_requested == 0)
3400 goto _exit; // no topology limiting actions requested, exit
3401 #if KMP_USE_HWLOC
3402 if (__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) {
3403 // Number of subobjects calculated dynamically, this works fine for
3404 // any non-uniform topology.
3405 // L2 cache objects are determined by depth, other objects - by type.
3406 hwloc_topology_t tp = __kmp_hwloc_topology;
3407 int nS = 0, nN = 0, nL = 0, nC = 0,
3408 nT = 0; // logical index including skipped
3409 int nCr = 0, nTr = 0; // number of requested units
3410 int nPkg = 0, nCo = 0, n_new = 0, n_old = 0, nCpP = 0, nTpC = 0; // counters
3411 hwloc_obj_t hT, hC, hL, hN, hS; // hwloc objects (pointers to)
3412 int L2depth, idx;
3413
3414 // check support of extensions ----------------------------------
3415 int numa_support = 0, tile_support = 0;
3416 if (__kmp_pu_os_idx)
3417 hT = hwloc_get_pu_obj_by_os_index(tp,
3418 __kmp_pu_os_idx[__kmp_avail_proc - 1]);
3419 else
3420 hT = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PU, __kmp_avail_proc - 1);
3421 if (hT == NULL) { // something's gone wrong
3422 KMP_WARNING(AffHWSubsetUnsupported);
3423 goto _exit;
3424 }
3425 // check NUMA node
3426 hN = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hT);
3427 hS = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hT);
3428 if (hN != NULL && hN->depth > hS->depth) {
3429 numa_support = 1; // 1 in case socket includes node(s)
3430 } else if (__kmp_hws_node.num > 0) {
3431 // don't support sockets inside NUMA node (no such HW found for testing)
3432 KMP_WARNING(AffHWSubsetUnsupported);
3433 goto _exit;
3434 }
3435 // check L2 cahce, get object by depth because of multiple caches
3436 L2depth = hwloc_get_cache_type_depth(tp, 2, HWLOC_OBJ_CACHE_UNIFIED);
3437 hL = hwloc_get_ancestor_obj_by_depth(tp, L2depth, hT);
3438 if (hL != NULL &&
3439 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC) > 1) {
3440 tile_support = 1; // no sense to count L2 if it includes single core
3441 } else if (__kmp_hws_tile.num > 0) {
3442 if (__kmp_hws_core.num == 0) {
3443 __kmp_hws_core = __kmp_hws_tile; // replace L2 with core
3444 __kmp_hws_tile.num = 0;
3445 } else {
3446 // L2 and core are both requested, but represent same object
3447 KMP_WARNING(AffHWSubsetInvalid);
3448 goto _exit;
3449 }
3450 }
3451 // end of check of extensions -----------------------------------
3452
3453 // fill in unset items, validate settings -----------------------
3454 if (__kmp_hws_socket.num == 0)
3455 __kmp_hws_socket.num = nPackages; // use all available sockets
3456 if (__kmp_hws_socket.offset >= nPackages) {
3457 KMP_WARNING(AffHWSubsetManySockets);
3458 goto _exit;
3459 }
3460 if (numa_support) {
3461 hN = NULL;
3462 int NN = __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_NUMANODE,
3463 &hN); // num nodes in socket
3464 if (__kmp_hws_node.num == 0)
3465 __kmp_hws_node.num = NN; // use all available nodes
3466 if (__kmp_hws_node.offset >= NN) {
3467 KMP_WARNING(AffHWSubsetManyNodes);
3468 goto _exit;
3469 }
3470 if (tile_support) {
3471 // get num tiles in node
3472 int NL = __kmp_hwloc_count_children_by_depth(tp, hN, L2depth, &hL);
3473 if (__kmp_hws_tile.num == 0) {
3474 __kmp_hws_tile.num = NL + 1;
3475 } // use all available tiles, some node may have more tiles, thus +1
3476 if (__kmp_hws_tile.offset >= NL) {
3477 KMP_WARNING(AffHWSubsetManyTiles);
3478 goto _exit;
3479 }
3480 int NC = __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE,
3481 &hC); // num cores in tile
3482 if (__kmp_hws_core.num == 0)
3483 __kmp_hws_core.num = NC; // use all available cores
3484 if (__kmp_hws_core.offset >= NC) {
3485 KMP_WARNING(AffHWSubsetManyCores);
3486 goto _exit;
3487 }
3488 } else { // tile_support
3489 int NC = __kmp_hwloc_count_children_by_type(tp, hN, HWLOC_OBJ_CORE,
3490 &hC); // num cores in node
3491 if (__kmp_hws_core.num == 0)
3492 __kmp_hws_core.num = NC; // use all available cores
3493 if (__kmp_hws_core.offset >= NC) {
3494 KMP_WARNING(AffHWSubsetManyCores);
3495 goto _exit;
3496 }
3497 } // tile_support
3498 } else { // numa_support
3499 if (tile_support) {
3500 // get num tiles in socket
3501 int NL = __kmp_hwloc_count_children_by_depth(tp, hS, L2depth, &hL);
3502 if (__kmp_hws_tile.num == 0)
3503 __kmp_hws_tile.num = NL; // use all available tiles
3504 if (__kmp_hws_tile.offset >= NL) {
3505 KMP_WARNING(AffHWSubsetManyTiles);
3506 goto _exit;
3507 }
3508 int NC = __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE,
3509 &hC); // num cores in tile
3510 if (__kmp_hws_core.num == 0)
3511 __kmp_hws_core.num = NC; // use all available cores
3512 if (__kmp_hws_core.offset >= NC) {
3513 KMP_WARNING(AffHWSubsetManyCores);
3514 goto _exit;
3515 }
3516 } else { // tile_support
3517 int NC = __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_CORE,
3518 &hC); // num cores in socket
3519 if (__kmp_hws_core.num == 0)
3520 __kmp_hws_core.num = NC; // use all available cores
3521 if (__kmp_hws_core.offset >= NC) {
3522 KMP_WARNING(AffHWSubsetManyCores);
3523 goto _exit;
3524 }
3525 } // tile_support
3526 }
3527 if (__kmp_hws_proc.num == 0)
3528 __kmp_hws_proc.num = __kmp_nThreadsPerCore; // use all available procs
3529 if (__kmp_hws_proc.offset >= __kmp_nThreadsPerCore) {
3530 KMP_WARNING(AffHWSubsetManyProcs);
3531 goto _exit;
3532 }
3533 // end of validation --------------------------------------------
3534
3535 if (pAddr) // pAddr is NULL in case of affinity_none
3536 newAddr = (AddrUnsPair *)__kmp_allocate(sizeof(AddrUnsPair) *
3537 __kmp_avail_proc); // max size
3538 // main loop to form HW subset ----------------------------------
3539 hS = NULL;
3540 int NP = hwloc_get_nbobjs_by_type(tp, HWLOC_OBJ_PACKAGE);
3541 for (int s = 0; s < NP; ++s) {
3542 // Check Socket -----------------------------------------------
3543 hS = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hS);
3544 if (!__kmp_hwloc_obj_has_PUs(tp, hS))
3545 continue; // skip socket if all PUs are out of fullMask
3546 ++nS; // only count objects those have PUs in affinity mask
3547 if (nS <= __kmp_hws_socket.offset ||
3548 nS > __kmp_hws_socket.num + __kmp_hws_socket.offset) {
3549 n_old += __kmp_hwloc_skip_PUs_obj(tp, hS); // skip socket
3550 continue; // move to next socket
3551 }
3552 nCr = 0; // count number of cores per socket
3553 // socket requested, go down the topology tree
3554 // check 4 cases: (+NUMA+Tile), (+NUMA-Tile), (-NUMA+Tile), (-NUMA-Tile)
3555 if (numa_support) {
3556 nN = 0;
3557 hN = NULL;
3558 // num nodes in current socket
3559 int NN =
3560 __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_NUMANODE, &hN);
3561 for (int n = 0; n < NN; ++n) {
3562 // Check NUMA Node ----------------------------------------
3563 if (!__kmp_hwloc_obj_has_PUs(tp, hN)) {
3564 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3565 continue; // skip node if all PUs are out of fullMask
3566 }
3567 ++nN;
3568 if (nN <= __kmp_hws_node.offset ||
3569 nN > __kmp_hws_node.num + __kmp_hws_node.offset) {
3570 // skip node as not requested
3571 n_old += __kmp_hwloc_skip_PUs_obj(tp, hN); // skip node
3572 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3573 continue; // move to next node
3574 }
3575 // node requested, go down the topology tree
3576 if (tile_support) {
3577 nL = 0;
3578 hL = NULL;
3579 int NL = __kmp_hwloc_count_children_by_depth(tp, hN, L2depth, &hL);
3580 for (int l = 0; l < NL; ++l) {
3581 // Check L2 (tile) ------------------------------------
3582 if (!__kmp_hwloc_obj_has_PUs(tp, hL)) {
3583 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3584 continue; // skip tile if all PUs are out of fullMask
3585 }
3586 ++nL;
3587 if (nL <= __kmp_hws_tile.offset ||
3588 nL > __kmp_hws_tile.num + __kmp_hws_tile.offset) {
3589 // skip tile as not requested
3590 n_old += __kmp_hwloc_skip_PUs_obj(tp, hL); // skip tile
3591 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3592 continue; // move to next tile
3593 }
3594 // tile requested, go down the topology tree
3595 nC = 0;
3596 hC = NULL;
3597 // num cores in current tile
3598 int NC = __kmp_hwloc_count_children_by_type(tp, hL,
3599 HWLOC_OBJ_CORE, &hC);
3600 for (int c = 0; c < NC; ++c) {
3601 // Check Core ---------------------------------------
3602 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3603 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3604 continue; // skip core if all PUs are out of fullMask
3605 }
3606 ++nC;
3607 if (nC <= __kmp_hws_core.offset ||
3608 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3609 // skip node as not requested
3610 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC); // skip core
3611 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3612 continue; // move to next node
3613 }
3614 // core requested, go down to PUs
3615 nT = 0;
3616 nTr = 0;
3617 hT = NULL;
3618 // num procs in current core
3619 int NT = __kmp_hwloc_count_children_by_type(tp, hC,
3620 HWLOC_OBJ_PU, &hT);
3621 for (int t = 0; t < NT; ++t) {
3622 // Check PU ---------------------------------------
3623 idx = hT->os_index;
3624 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3625 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3626 continue; // skip PU if not in fullMask
3627 }
3628 ++nT;
3629 if (nT <= __kmp_hws_proc.offset ||
3630 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3631 // skip PU
3632 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3633 ++n_old;
3634 KC_TRACE(200, ("KMP_HW_SUBSET: skipped proc %d\n", idx));
3635 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3636 continue; // move to next node
3637 }
3638 ++nTr;
3639 if (pAddr) // collect requested thread's data
3640 newAddr[n_new] = (*pAddr)[n_old];
3641 ++n_new;
3642 ++n_old;
3643 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3644 } // threads loop
3645 if (nTr > 0) {
3646 ++nCr; // num cores per socket
3647 ++nCo; // total num cores
3648 if (nTr > nTpC)
3649 nTpC = nTr; // calc max threads per core
3650 }
3651 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3652 } // cores loop
3653 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3654 } // tiles loop
3655 } else { // tile_support
3656 // no tiles, check cores
3657 nC = 0;
3658 hC = NULL;
3659 // num cores in current node
3660 int NC =
3661 __kmp_hwloc_count_children_by_type(tp, hN, HWLOC_OBJ_CORE, &hC);
3662 for (int c = 0; c < NC; ++c) {
3663 // Check Core ---------------------------------------
3664 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3665 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3666 continue; // skip core if all PUs are out of fullMask
3667 }
3668 ++nC;
3669 if (nC <= __kmp_hws_core.offset ||
3670 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3671 // skip node as not requested
3672 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC); // skip core
3673 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3674 continue; // move to next node
3675 }
3676 // core requested, go down to PUs
3677 nT = 0;
3678 nTr = 0;
3679 hT = NULL;
3680 int NT =
3681 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3682 for (int t = 0; t < NT; ++t) {
3683 // Check PU ---------------------------------------
3684 idx = hT->os_index;
3685 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3686 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3687 continue; // skip PU if not in fullMask
3688 }
3689 ++nT;
3690 if (nT <= __kmp_hws_proc.offset ||
3691 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3692 // skip PU
3693 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3694 ++n_old;
3695 KC_TRACE(200, ("KMP_HW_SUBSET: skipped proc %d\n", idx));
3696 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3697 continue; // move to next node
3698 }
3699 ++nTr;
3700 if (pAddr) // collect requested thread's data
3701 newAddr[n_new] = (*pAddr)[n_old];
3702 ++n_new;
3703 ++n_old;
3704 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3705 } // threads loop
3706 if (nTr > 0) {
3707 ++nCr; // num cores per socket
3708 ++nCo; // total num cores
3709 if (nTr > nTpC)
3710 nTpC = nTr; // calc max threads per core
3711 }
3712 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3713 } // cores loop
3714 } // tiles support
3715 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3716 } // nodes loop
3717 } else { // numa_support
3718 // no NUMA support
3719 if (tile_support) {
3720 nL = 0;
3721 hL = NULL;
3722 // num tiles in current socket
3723 int NL = __kmp_hwloc_count_children_by_depth(tp, hS, L2depth, &hL);
3724 for (int l = 0; l < NL; ++l) {
3725 // Check L2 (tile) ------------------------------------
3726 if (!__kmp_hwloc_obj_has_PUs(tp, hL)) {
3727 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3728 continue; // skip tile if all PUs are out of fullMask
3729 }
3730 ++nL;
3731 if (nL <= __kmp_hws_tile.offset ||
3732 nL > __kmp_hws_tile.num + __kmp_hws_tile.offset) {
3733 // skip tile as not requested
3734 n_old += __kmp_hwloc_skip_PUs_obj(tp, hL); // skip tile
3735 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3736 continue; // move to next tile
3737 }
3738 // tile requested, go down the topology tree
3739 nC = 0;
3740 hC = NULL;
3741 // num cores per tile
3742 int NC =
3743 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC);
3744 for (int c = 0; c < NC; ++c) {
3745 // Check Core ---------------------------------------
3746 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3747 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3748 continue; // skip core if all PUs are out of fullMask
3749 }
3750 ++nC;
3751 if (nC <= __kmp_hws_core.offset ||
3752 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3753 // skip node as not requested
3754 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC); // skip core
3755 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3756 continue; // move to next node
3757 }
3758 // core requested, go down to PUs
3759 nT = 0;
3760 nTr = 0;
3761 hT = NULL;
3762 // num procs per core
3763 int NT =
3764 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3765 for (int t = 0; t < NT; ++t) {
3766 // Check PU ---------------------------------------
3767 idx = hT->os_index;
3768 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3769 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3770 continue; // skip PU if not in fullMask
3771 }
3772 ++nT;
3773 if (nT <= __kmp_hws_proc.offset ||
3774 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3775 // skip PU
3776 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3777 ++n_old;
3778 KC_TRACE(200, ("KMP_HW_SUBSET: skipped proc %d\n", idx));
3779 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3780 continue; // move to next node
3781 }
3782 ++nTr;
3783 if (pAddr) // collect requested thread's data
3784 newAddr[n_new] = (*pAddr)[n_old];
3785 ++n_new;
3786 ++n_old;
3787 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3788 } // threads loop
3789 if (nTr > 0) {
3790 ++nCr; // num cores per socket
3791 ++nCo; // total num cores
3792 if (nTr > nTpC)
3793 nTpC = nTr; // calc max threads per core
3794 }
3795 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3796 } // cores loop
3797 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3798 } // tiles loop
3799 } else { // tile_support
3800 // no tiles, check cores
3801 nC = 0;
3802 hC = NULL;
3803 // num cores in socket
3804 int NC =
3805 __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_CORE, &hC);
3806 for (int c = 0; c < NC; ++c) {
3807 // Check Core -------------------------------------------
3808 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3809 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3810 continue; // skip core if all PUs are out of fullMask
3811 }
3812 ++nC;
3813 if (nC <= __kmp_hws_core.offset ||
3814 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3815 // skip node as not requested
3816 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC); // skip core
3817 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3818 continue; // move to next node
3819 }
3820 // core requested, go down to PUs
3821 nT = 0;
3822 nTr = 0;
3823 hT = NULL;
3824 // num procs per core
3825 int NT =
3826 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3827 for (int t = 0; t < NT; ++t) {
3828 // Check PU ---------------------------------------
3829 idx = hT->os_index;
3830 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3831 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3832 continue; // skip PU if not in fullMask
3833 }
3834 ++nT;
3835 if (nT <= __kmp_hws_proc.offset ||
3836 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3837 // skip PU
3838 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3839 ++n_old;
3840 KC_TRACE(200, ("KMP_HW_SUBSET: skipped proc %d\n", idx));
3841 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3842 continue; // move to next node
3843 }
3844 ++nTr;
3845 if (pAddr) // collect requested thread's data
3846 newAddr[n_new] = (*pAddr)[n_old];
3847 ++n_new;
3848 ++n_old;
3849 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3850 } // threads loop
3851 if (nTr > 0) {
3852 ++nCr; // num cores per socket
3853 ++nCo; // total num cores
3854 if (nTr > nTpC)
3855 nTpC = nTr; // calc max threads per core
3856 }
3857 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3858 } // cores loop
3859 } // tiles support
3860 } // numa_support
3861 if (nCr > 0) { // found cores?
3862 ++nPkg; // num sockets
3863 if (nCr > nCpP)
3864 nCpP = nCr; // calc max cores per socket
3865 }
3866 } // sockets loop
3867
3868 // check the subset is valid
3869 KMP_DEBUG_ASSERT(n_old == __kmp_avail_proc);
3870 KMP_DEBUG_ASSERT(nPkg > 0);
3871 KMP_DEBUG_ASSERT(nCpP > 0);
3872 KMP_DEBUG_ASSERT(nTpC > 0);
3873 KMP_DEBUG_ASSERT(nCo > 0);
3874 KMP_DEBUG_ASSERT(nPkg <= nPackages);
3875 KMP_DEBUG_ASSERT(nCpP <= nCoresPerPkg);
3876 KMP_DEBUG_ASSERT(nTpC <= __kmp_nThreadsPerCore);
3877 KMP_DEBUG_ASSERT(nCo <= __kmp_ncores);
3878
3879 nPackages = nPkg; // correct num sockets
3880 nCoresPerPkg = nCpP; // correct num cores per socket
3881 __kmp_nThreadsPerCore = nTpC; // correct num threads per core
3882 __kmp_avail_proc = n_new; // correct num procs
3883 __kmp_ncores = nCo; // correct num cores
3884 // hwloc topology method end
3885 } else
3886 #endif // KMP_USE_HWLOC
3887 {
3888 int n_old = 0, n_new = 0, proc_num = 0;
3889 if (__kmp_hws_node.num > 0 || __kmp_hws_tile.num > 0) {
3890 KMP_WARNING(AffHWSubsetNoHWLOC);
3891 goto _exit;
3892 }
3893 if (__kmp_hws_socket.num == 0)
3894 __kmp_hws_socket.num = nPackages; // use all available sockets
3895 if (__kmp_hws_core.num == 0)
3896 __kmp_hws_core.num = nCoresPerPkg; // use all available cores
3897 if (__kmp_hws_proc.num == 0 || __kmp_hws_proc.num > __kmp_nThreadsPerCore)
3898 __kmp_hws_proc.num = __kmp_nThreadsPerCore; // use all HW contexts
3899 if (!__kmp_affinity_uniform_topology()) {
3900 KMP_WARNING(AffHWSubsetNonUniform);
3901 goto _exit; // don't support non-uniform topology
3902 }
3903 if (depth > 3) {
3904 KMP_WARNING(AffHWSubsetNonThreeLevel);
3905 goto _exit; // don't support not-3-level topology
3906 }
3907 if (__kmp_hws_socket.offset + __kmp_hws_socket.num > nPackages) {
3908 KMP_WARNING(AffHWSubsetManySockets);
3909 goto _exit;
3910 }
3911 if (__kmp_hws_core.offset + __kmp_hws_core.num > nCoresPerPkg) {
3912 KMP_WARNING(AffHWSubsetManyCores);
3913 goto _exit;
3914 }
3915 // Form the requested subset
3916 if (pAddr) // pAddr is NULL in case of affinity_none
3917 newAddr = (AddrUnsPair *)__kmp_allocate(
3918 sizeof(AddrUnsPair) * __kmp_hws_socket.num * __kmp_hws_core.num *
3919 __kmp_hws_proc.num);
3920 for (int i = 0; i < nPackages; ++i) {
3921 if (i < __kmp_hws_socket.offset ||
3922 i >= __kmp_hws_socket.offset + __kmp_hws_socket.num) {
3923 // skip not-requested socket
3924 n_old += nCoresPerPkg * __kmp_nThreadsPerCore;
3925 if (__kmp_pu_os_idx != NULL) {
3926 // walk through skipped socket
3927 for (int j = 0; j < nCoresPerPkg; ++j) {
3928 for (int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3929 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3930 ++proc_num;
3931 }
3932 }
3933 }
3934 } else {
3935 // walk through requested socket
3936 for (int j = 0; j < nCoresPerPkg; ++j) {
3937 if (j < __kmp_hws_core.offset ||
3938 j >= __kmp_hws_core.offset +
3939 __kmp_hws_core.num) { // skip not-requested core
3940 n_old += __kmp_nThreadsPerCore;
3941 if (__kmp_pu_os_idx != NULL) {
3942 for (int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3943 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3944 ++proc_num;
3945 }
3946 }
3947 } else {
3948 // walk through requested core
3949 for (int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3950 if (k < __kmp_hws_proc.num) {
3951 if (pAddr) // collect requested thread's data
3952 newAddr[n_new] = (*pAddr)[n_old];
3953 n_new++;
3954 } else {
3955 if (__kmp_pu_os_idx != NULL)
3956 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3957 }
3958 n_old++;
3959 ++proc_num;
3960 }
3961 }
3962 }
3963 }
3964 }
3965 KMP_DEBUG_ASSERT(n_old == nPackages * nCoresPerPkg * __kmp_nThreadsPerCore);
3966 KMP_DEBUG_ASSERT(n_new ==
3967 __kmp_hws_socket.num * __kmp_hws_core.num *
3968 __kmp_hws_proc.num);
3969 nPackages = __kmp_hws_socket.num; // correct nPackages
3970 nCoresPerPkg = __kmp_hws_core.num; // correct nCoresPerPkg
3971 __kmp_nThreadsPerCore = __kmp_hws_proc.num; // correct __kmp_nThreadsPerCore
3972 __kmp_avail_proc = n_new; // correct avail_proc
3973 __kmp_ncores = nPackages * __kmp_hws_core.num; // correct ncores
3974 } // non-hwloc topology method
3975 if (pAddr) {
3976 __kmp_free(*pAddr);
3977 *pAddr = newAddr; // replace old topology with new one
3978 }
3979 if (__kmp_affinity_verbose) {
3980 char m[KMP_AFFIN_MASK_PRINT_LEN];
3981 __kmp_affinity_print_mask(m, KMP_AFFIN_MASK_PRINT_LEN,
3982 __kmp_affin_fullMask);
3983 if (__kmp_affinity_respect_mask) {
3984 KMP_INFORM(InitOSProcSetRespect, "KMP_HW_SUBSET", m);
3985 } else {
3986 KMP_INFORM(InitOSProcSetNotRespect, "KMP_HW_SUBSET", m);
3987 }
3988 KMP_INFORM(AvailableOSProc, "KMP_HW_SUBSET", __kmp_avail_proc);
3989 kmp_str_buf_t buf;
3990 __kmp_str_buf_init(&buf);
3991 __kmp_str_buf_print(&buf, "%d", nPackages);
3992 KMP_INFORM(TopologyExtra, "KMP_HW_SUBSET", buf.str, nCoresPerPkg,
3993 __kmp_nThreadsPerCore, __kmp_ncores);
3994 __kmp_str_buf_free(&buf);
3995 }
3996 _exit:
3997 if (__kmp_pu_os_idx != NULL) {
3998 __kmp_free(__kmp_pu_os_idx);
3999 __kmp_pu_os_idx = NULL;
4000 }
4001 }
4002
4003 // This function figures out the deepest level at which there is at least one
4004 // cluster/core with more than one processing unit bound to it.
__kmp_affinity_find_core_level(const AddrUnsPair * address2os,int nprocs,int bottom_level)4005 static int __kmp_affinity_find_core_level(const AddrUnsPair *address2os,
4006 int nprocs, int bottom_level) {
4007 int core_level = 0;
4008
4009 for (int i = 0; i < nprocs; i++) {
4010 for (int j = bottom_level; j > 0; j--) {
4011 if (address2os[i].first.labels[j] > 0) {
4012 if (core_level < (j - 1)) {
4013 core_level = j - 1;
4014 }
4015 }
4016 }
4017 }
4018 return core_level;
4019 }
4020
4021 // This function counts number of clusters/cores at given level.
__kmp_affinity_compute_ncores(const AddrUnsPair * address2os,int nprocs,int bottom_level,int core_level)4022 static int __kmp_affinity_compute_ncores(const AddrUnsPair *address2os,
4023 int nprocs, int bottom_level,
4024 int core_level) {
4025 int ncores = 0;
4026 int i, j;
4027
4028 j = bottom_level;
4029 for (i = 0; i < nprocs; i++) {
4030 for (j = bottom_level; j > core_level; j--) {
4031 if ((i + 1) < nprocs) {
4032 if (address2os[i + 1].first.labels[j] > 0) {
4033 break;
4034 }
4035 }
4036 }
4037 if (j == core_level) {
4038 ncores++;
4039 }
4040 }
4041 if (j > core_level) {
4042 // In case of ( nprocs < __kmp_avail_proc ) we may end too deep and miss one
4043 // core. May occur when called from __kmp_affinity_find_core().
4044 ncores++;
4045 }
4046 return ncores;
4047 }
4048
4049 // This function finds to which cluster/core given processing unit is bound.
__kmp_affinity_find_core(const AddrUnsPair * address2os,int proc,int bottom_level,int core_level)4050 static int __kmp_affinity_find_core(const AddrUnsPair *address2os, int proc,
4051 int bottom_level, int core_level) {
4052 return __kmp_affinity_compute_ncores(address2os, proc + 1, bottom_level,
4053 core_level) -
4054 1;
4055 }
4056
4057 // This function finds maximal number of processing units bound to a
4058 // cluster/core at given level.
__kmp_affinity_max_proc_per_core(const AddrUnsPair * address2os,int nprocs,int bottom_level,int core_level)4059 static int __kmp_affinity_max_proc_per_core(const AddrUnsPair *address2os,
4060 int nprocs, int bottom_level,
4061 int core_level) {
4062 int maxprocpercore = 0;
4063
4064 if (core_level < bottom_level) {
4065 for (int i = 0; i < nprocs; i++) {
4066 int percore = address2os[i].first.labels[core_level + 1] + 1;
4067
4068 if (percore > maxprocpercore) {
4069 maxprocpercore = percore;
4070 }
4071 }
4072 } else {
4073 maxprocpercore = 1;
4074 }
4075 return maxprocpercore;
4076 }
4077
4078 static AddrUnsPair *address2os = NULL;
4079 static int *procarr = NULL;
4080 static int __kmp_aff_depth = 0;
4081
4082 #if KMP_USE_HIER_SCHED
4083 #define KMP_EXIT_AFF_NONE \
4084 KMP_ASSERT(__kmp_affinity_type == affinity_none); \
4085 KMP_ASSERT(address2os == NULL); \
4086 __kmp_apply_thread_places(NULL, 0); \
4087 __kmp_create_affinity_none_places(); \
4088 __kmp_dispatch_set_hierarchy_values(); \
4089 return;
4090 #else
4091 #define KMP_EXIT_AFF_NONE \
4092 KMP_ASSERT(__kmp_affinity_type == affinity_none); \
4093 KMP_ASSERT(address2os == NULL); \
4094 __kmp_apply_thread_places(NULL, 0); \
4095 __kmp_create_affinity_none_places(); \
4096 return;
4097 #endif
4098
4099 // Create a one element mask array (set of places) which only contains the
4100 // initial process's affinity mask
__kmp_create_affinity_none_places()4101 static void __kmp_create_affinity_none_places() {
4102 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4103 KMP_ASSERT(__kmp_affinity_type == affinity_none);
4104 __kmp_affinity_num_masks = 1;
4105 KMP_CPU_ALLOC_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4106 kmp_affin_mask_t *dest = KMP_CPU_INDEX(__kmp_affinity_masks, 0);
4107 KMP_CPU_COPY(dest, __kmp_affin_fullMask);
4108 }
4109
__kmp_affinity_cmp_Address_child_num(const void * a,const void * b)4110 static int __kmp_affinity_cmp_Address_child_num(const void *a, const void *b) {
4111 const Address *aa = &(((const AddrUnsPair *)a)->first);
4112 const Address *bb = &(((const AddrUnsPair *)b)->first);
4113 unsigned depth = aa->depth;
4114 unsigned i;
4115 KMP_DEBUG_ASSERT(depth == bb->depth);
4116 KMP_DEBUG_ASSERT((unsigned)__kmp_affinity_compact <= depth);
4117 KMP_DEBUG_ASSERT(__kmp_affinity_compact >= 0);
4118 for (i = 0; i < (unsigned)__kmp_affinity_compact; i++) {
4119 int j = depth - i - 1;
4120 if (aa->childNums[j] < bb->childNums[j])
4121 return -1;
4122 if (aa->childNums[j] > bb->childNums[j])
4123 return 1;
4124 }
4125 for (; i < depth; i++) {
4126 int j = i - __kmp_affinity_compact;
4127 if (aa->childNums[j] < bb->childNums[j])
4128 return -1;
4129 if (aa->childNums[j] > bb->childNums[j])
4130 return 1;
4131 }
4132 return 0;
4133 }
4134
__kmp_aux_affinity_initialize(void)4135 static void __kmp_aux_affinity_initialize(void) {
4136 if (__kmp_affinity_masks != NULL) {
4137 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4138 return;
4139 }
4140
4141 // Create the "full" mask - this defines all of the processors that we
4142 // consider to be in the machine model. If respect is set, then it is the
4143 // initialization thread's affinity mask. Otherwise, it is all processors that
4144 // we know about on the machine.
4145 if (__kmp_affin_fullMask == NULL) {
4146 KMP_CPU_ALLOC(__kmp_affin_fullMask);
4147 }
4148 if (KMP_AFFINITY_CAPABLE()) {
4149 if (__kmp_affinity_respect_mask) {
4150 __kmp_get_system_affinity(__kmp_affin_fullMask, TRUE);
4151
4152 // Count the number of available processors.
4153 unsigned i;
4154 __kmp_avail_proc = 0;
4155 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
4156 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
4157 continue;
4158 }
4159 __kmp_avail_proc++;
4160 }
4161 if (__kmp_avail_proc > __kmp_xproc) {
4162 if (__kmp_affinity_verbose ||
4163 (__kmp_affinity_warnings &&
4164 (__kmp_affinity_type != affinity_none))) {
4165 KMP_WARNING(ErrorInitializeAffinity);
4166 }
4167 __kmp_affinity_type = affinity_none;
4168 KMP_AFFINITY_DISABLE();
4169 return;
4170 }
4171 } else {
4172 __kmp_affinity_entire_machine_mask(__kmp_affin_fullMask);
4173 __kmp_avail_proc = __kmp_xproc;
4174 }
4175 }
4176
4177 if (__kmp_affinity_gran == affinity_gran_tile &&
4178 // check if user's request is valid
4179 __kmp_affinity_dispatch->get_api_type() == KMPAffinity::NATIVE_OS) {
4180 KMP_WARNING(AffTilesNoHWLOC, "KMP_AFFINITY");
4181 __kmp_affinity_gran = affinity_gran_package;
4182 }
4183
4184 int depth = -1;
4185 kmp_i18n_id_t msg_id = kmp_i18n_null;
4186
4187 // For backward compatibility, setting KMP_CPUINFO_FILE =>
4188 // KMP_TOPOLOGY_METHOD=cpuinfo
4189 if ((__kmp_cpuinfo_file != NULL) &&
4190 (__kmp_affinity_top_method == affinity_top_method_all)) {
4191 __kmp_affinity_top_method = affinity_top_method_cpuinfo;
4192 }
4193
4194 if (__kmp_affinity_top_method == affinity_top_method_all) {
4195 // In the default code path, errors are not fatal - we just try using
4196 // another method. We only emit a warning message if affinity is on, or the
4197 // verbose flag is set, and the nowarnings flag was not set.
4198 const char *file_name = NULL;
4199 int line = 0;
4200 #if KMP_USE_HWLOC
4201 if (depth < 0 &&
4202 __kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) {
4203 if (__kmp_affinity_verbose) {
4204 KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY");
4205 }
4206 if (!__kmp_hwloc_error) {
4207 depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
4208 if (depth == 0) {
4209 KMP_EXIT_AFF_NONE;
4210 } else if (depth < 0 && __kmp_affinity_verbose) {
4211 KMP_INFORM(AffIgnoringHwloc, "KMP_AFFINITY");
4212 }
4213 } else if (__kmp_affinity_verbose) {
4214 KMP_INFORM(AffIgnoringHwloc, "KMP_AFFINITY");
4215 }
4216 }
4217 #endif
4218
4219 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
4220
4221 if (depth < 0) {
4222 if (__kmp_affinity_verbose) {
4223 KMP_INFORM(AffInfoStr, "KMP_AFFINITY", KMP_I18N_STR(Decodingx2APIC));
4224 }
4225
4226 file_name = NULL;
4227 depth = __kmp_affinity_create_x2apicid_map(&address2os, &msg_id);
4228 if (depth == 0) {
4229 KMP_EXIT_AFF_NONE;
4230 }
4231
4232 if (depth < 0) {
4233 if (__kmp_affinity_verbose) {
4234 if (msg_id != kmp_i18n_null) {
4235 KMP_INFORM(AffInfoStrStr, "KMP_AFFINITY",
4236 __kmp_i18n_catgets(msg_id),
4237 KMP_I18N_STR(DecodingLegacyAPIC));
4238 } else {
4239 KMP_INFORM(AffInfoStr, "KMP_AFFINITY",
4240 KMP_I18N_STR(DecodingLegacyAPIC));
4241 }
4242 }
4243
4244 file_name = NULL;
4245 depth = __kmp_affinity_create_apicid_map(&address2os, &msg_id);
4246 if (depth == 0) {
4247 KMP_EXIT_AFF_NONE;
4248 }
4249 }
4250 }
4251
4252 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4253
4254 #if KMP_OS_LINUX
4255
4256 if (depth < 0) {
4257 if (__kmp_affinity_verbose) {
4258 if (msg_id != kmp_i18n_null) {
4259 KMP_INFORM(AffStrParseFilename, "KMP_AFFINITY",
4260 __kmp_i18n_catgets(msg_id), "/proc/cpuinfo");
4261 } else {
4262 KMP_INFORM(AffParseFilename, "KMP_AFFINITY", "/proc/cpuinfo");
4263 }
4264 }
4265
4266 FILE *f = fopen("/proc/cpuinfo", "r");
4267 if (f == NULL) {
4268 msg_id = kmp_i18n_str_CantOpenCpuinfo;
4269 } else {
4270 file_name = "/proc/cpuinfo";
4271 depth =
4272 __kmp_affinity_create_cpuinfo_map(&address2os, &line, &msg_id, f);
4273 fclose(f);
4274 if (depth == 0) {
4275 KMP_EXIT_AFF_NONE;
4276 }
4277 }
4278 }
4279
4280 #endif /* KMP_OS_LINUX */
4281
4282 #if KMP_GROUP_AFFINITY
4283
4284 if ((depth < 0) && (__kmp_num_proc_groups > 1)) {
4285 if (__kmp_affinity_verbose) {
4286 KMP_INFORM(AffWindowsProcGroupMap, "KMP_AFFINITY");
4287 }
4288
4289 depth = __kmp_affinity_create_proc_group_map(&address2os, &msg_id);
4290 KMP_ASSERT(depth != 0);
4291 }
4292
4293 #endif /* KMP_GROUP_AFFINITY */
4294
4295 if (depth < 0) {
4296 if (__kmp_affinity_verbose && (msg_id != kmp_i18n_null)) {
4297 if (file_name == NULL) {
4298 KMP_INFORM(UsingFlatOS, __kmp_i18n_catgets(msg_id));
4299 } else if (line == 0) {
4300 KMP_INFORM(UsingFlatOSFile, file_name, __kmp_i18n_catgets(msg_id));
4301 } else {
4302 KMP_INFORM(UsingFlatOSFileLine, file_name, line,
4303 __kmp_i18n_catgets(msg_id));
4304 }
4305 }
4306 // FIXME - print msg if msg_id = kmp_i18n_null ???
4307
4308 file_name = "";
4309 depth = __kmp_affinity_create_flat_map(&address2os, &msg_id);
4310 if (depth == 0) {
4311 KMP_EXIT_AFF_NONE;
4312 }
4313 KMP_ASSERT(depth > 0);
4314 KMP_ASSERT(address2os != NULL);
4315 }
4316 }
4317
4318 #if KMP_USE_HWLOC
4319 else if (__kmp_affinity_top_method == affinity_top_method_hwloc) {
4320 KMP_ASSERT(__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC);
4321 if (__kmp_affinity_verbose) {
4322 KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY");
4323 }
4324 depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
4325 if (depth == 0) {
4326 KMP_EXIT_AFF_NONE;
4327 }
4328 }
4329 #endif // KMP_USE_HWLOC
4330
4331 // If the user has specified that a paricular topology discovery method is to be
4332 // used, then we abort if that method fails. The exception is group affinity,
4333 // which might have been implicitly set.
4334
4335 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
4336
4337 else if (__kmp_affinity_top_method == affinity_top_method_x2apicid) {
4338 if (__kmp_affinity_verbose) {
4339 KMP_INFORM(AffInfoStr, "KMP_AFFINITY", KMP_I18N_STR(Decodingx2APIC));
4340 }
4341
4342 depth = __kmp_affinity_create_x2apicid_map(&address2os, &msg_id);
4343 if (depth == 0) {
4344 KMP_EXIT_AFF_NONE;
4345 }
4346 if (depth < 0) {
4347 KMP_ASSERT(msg_id != kmp_i18n_null);
4348 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4349 }
4350 } else if (__kmp_affinity_top_method == affinity_top_method_apicid) {
4351 if (__kmp_affinity_verbose) {
4352 KMP_INFORM(AffInfoStr, "KMP_AFFINITY", KMP_I18N_STR(DecodingLegacyAPIC));
4353 }
4354
4355 depth = __kmp_affinity_create_apicid_map(&address2os, &msg_id);
4356 if (depth == 0) {
4357 KMP_EXIT_AFF_NONE;
4358 }
4359 if (depth < 0) {
4360 KMP_ASSERT(msg_id != kmp_i18n_null);
4361 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4362 }
4363 }
4364
4365 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4366
4367 else if (__kmp_affinity_top_method == affinity_top_method_cpuinfo) {
4368 const char *filename;
4369 if (__kmp_cpuinfo_file != NULL) {
4370 filename = __kmp_cpuinfo_file;
4371 } else {
4372 filename = "/proc/cpuinfo";
4373 }
4374
4375 if (__kmp_affinity_verbose) {
4376 KMP_INFORM(AffParseFilename, "KMP_AFFINITY", filename);
4377 }
4378
4379 FILE *f = fopen(filename, "r");
4380 if (f == NULL) {
4381 int code = errno;
4382 if (__kmp_cpuinfo_file != NULL) {
4383 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4384 KMP_HNT(NameComesFrom_CPUINFO_FILE), __kmp_msg_null);
4385 } else {
4386 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4387 __kmp_msg_null);
4388 }
4389 }
4390 int line = 0;
4391 depth = __kmp_affinity_create_cpuinfo_map(&address2os, &line, &msg_id, f);
4392 fclose(f);
4393 if (depth < 0) {
4394 KMP_ASSERT(msg_id != kmp_i18n_null);
4395 if (line > 0) {
4396 KMP_FATAL(FileLineMsgExiting, filename, line,
4397 __kmp_i18n_catgets(msg_id));
4398 } else {
4399 KMP_FATAL(FileMsgExiting, filename, __kmp_i18n_catgets(msg_id));
4400 }
4401 }
4402 if (__kmp_affinity_type == affinity_none) {
4403 KMP_ASSERT(depth == 0);
4404 KMP_EXIT_AFF_NONE;
4405 }
4406 }
4407
4408 #if KMP_GROUP_AFFINITY
4409
4410 else if (__kmp_affinity_top_method == affinity_top_method_group) {
4411 if (__kmp_affinity_verbose) {
4412 KMP_INFORM(AffWindowsProcGroupMap, "KMP_AFFINITY");
4413 }
4414
4415 depth = __kmp_affinity_create_proc_group_map(&address2os, &msg_id);
4416 KMP_ASSERT(depth != 0);
4417 if (depth < 0) {
4418 KMP_ASSERT(msg_id != kmp_i18n_null);
4419 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4420 }
4421 }
4422
4423 #endif /* KMP_GROUP_AFFINITY */
4424
4425 else if (__kmp_affinity_top_method == affinity_top_method_flat) {
4426 if (__kmp_affinity_verbose) {
4427 KMP_INFORM(AffUsingFlatOS, "KMP_AFFINITY");
4428 }
4429
4430 depth = __kmp_affinity_create_flat_map(&address2os, &msg_id);
4431 if (depth == 0) {
4432 KMP_EXIT_AFF_NONE;
4433 }
4434 // should not fail
4435 KMP_ASSERT(depth > 0);
4436 KMP_ASSERT(address2os != NULL);
4437 }
4438
4439 #if KMP_USE_HIER_SCHED
4440 __kmp_dispatch_set_hierarchy_values();
4441 #endif
4442
4443 if (address2os == NULL) {
4444 if (KMP_AFFINITY_CAPABLE() &&
4445 (__kmp_affinity_verbose ||
4446 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none)))) {
4447 KMP_WARNING(ErrorInitializeAffinity);
4448 }
4449 __kmp_affinity_type = affinity_none;
4450 __kmp_create_affinity_none_places();
4451 KMP_AFFINITY_DISABLE();
4452 return;
4453 }
4454
4455 if (__kmp_affinity_gran == affinity_gran_tile
4456 #if KMP_USE_HWLOC
4457 && __kmp_tile_depth == 0
4458 #endif
4459 ) {
4460 // tiles requested but not detected, warn user on this
4461 KMP_WARNING(AffTilesNoTiles, "KMP_AFFINITY");
4462 }
4463
4464 __kmp_apply_thread_places(&address2os, depth);
4465
4466 // Create the table of masks, indexed by thread Id.
4467 unsigned maxIndex;
4468 unsigned numUnique;
4469 kmp_affin_mask_t *osId2Mask =
4470 __kmp_create_masks(&maxIndex, &numUnique, address2os, __kmp_avail_proc);
4471 if (__kmp_affinity_gran_levels == 0) {
4472 KMP_DEBUG_ASSERT((int)numUnique == __kmp_avail_proc);
4473 }
4474
4475 // Set the childNums vector in all Address objects. This must be done before
4476 // we can sort using __kmp_affinity_cmp_Address_child_num(), which takes into
4477 // account the setting of __kmp_affinity_compact.
4478 __kmp_affinity_assign_child_nums(address2os, __kmp_avail_proc);
4479
4480 switch (__kmp_affinity_type) {
4481
4482 case affinity_explicit:
4483 KMP_DEBUG_ASSERT(__kmp_affinity_proclist != NULL);
4484 if (__kmp_nested_proc_bind.bind_types[0] == proc_bind_intel) {
4485 __kmp_affinity_process_proclist(
4486 &__kmp_affinity_masks, &__kmp_affinity_num_masks,
4487 __kmp_affinity_proclist, osId2Mask, maxIndex);
4488 } else {
4489 __kmp_affinity_process_placelist(
4490 &__kmp_affinity_masks, &__kmp_affinity_num_masks,
4491 __kmp_affinity_proclist, osId2Mask, maxIndex);
4492 }
4493 if (__kmp_affinity_num_masks == 0) {
4494 if (__kmp_affinity_verbose ||
4495 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
4496 KMP_WARNING(AffNoValidProcID);
4497 }
4498 __kmp_affinity_type = affinity_none;
4499 __kmp_create_affinity_none_places();
4500 return;
4501 }
4502 break;
4503
4504 // The other affinity types rely on sorting the Addresses according to some
4505 // permutation of the machine topology tree. Set __kmp_affinity_compact and
4506 // __kmp_affinity_offset appropriately, then jump to a common code fragment
4507 // to do the sort and create the array of affinity masks.
4508
4509 case affinity_logical:
4510 __kmp_affinity_compact = 0;
4511 if (__kmp_affinity_offset) {
4512 __kmp_affinity_offset =
4513 __kmp_nThreadsPerCore * __kmp_affinity_offset % __kmp_avail_proc;
4514 }
4515 goto sortAddresses;
4516
4517 case affinity_physical:
4518 if (__kmp_nThreadsPerCore > 1) {
4519 __kmp_affinity_compact = 1;
4520 if (__kmp_affinity_compact >= depth) {
4521 __kmp_affinity_compact = 0;
4522 }
4523 } else {
4524 __kmp_affinity_compact = 0;
4525 }
4526 if (__kmp_affinity_offset) {
4527 __kmp_affinity_offset =
4528 __kmp_nThreadsPerCore * __kmp_affinity_offset % __kmp_avail_proc;
4529 }
4530 goto sortAddresses;
4531
4532 case affinity_scatter:
4533 if (__kmp_affinity_compact >= depth) {
4534 __kmp_affinity_compact = 0;
4535 } else {
4536 __kmp_affinity_compact = depth - 1 - __kmp_affinity_compact;
4537 }
4538 goto sortAddresses;
4539
4540 case affinity_compact:
4541 if (__kmp_affinity_compact >= depth) {
4542 __kmp_affinity_compact = depth - 1;
4543 }
4544 goto sortAddresses;
4545
4546 case affinity_balanced:
4547 if (depth <= 1) {
4548 if (__kmp_affinity_verbose || __kmp_affinity_warnings) {
4549 KMP_WARNING(AffBalancedNotAvail, "KMP_AFFINITY");
4550 }
4551 __kmp_affinity_type = affinity_none;
4552 __kmp_create_affinity_none_places();
4553 return;
4554 } else if (!__kmp_affinity_uniform_topology()) {
4555 // Save the depth for further usage
4556 __kmp_aff_depth = depth;
4557
4558 int core_level = __kmp_affinity_find_core_level(
4559 address2os, __kmp_avail_proc, depth - 1);
4560 int ncores = __kmp_affinity_compute_ncores(address2os, __kmp_avail_proc,
4561 depth - 1, core_level);
4562 int maxprocpercore = __kmp_affinity_max_proc_per_core(
4563 address2os, __kmp_avail_proc, depth - 1, core_level);
4564
4565 int nproc = ncores * maxprocpercore;
4566 if ((nproc < 2) || (nproc < __kmp_avail_proc)) {
4567 if (__kmp_affinity_verbose || __kmp_affinity_warnings) {
4568 KMP_WARNING(AffBalancedNotAvail, "KMP_AFFINITY");
4569 }
4570 __kmp_affinity_type = affinity_none;
4571 return;
4572 }
4573
4574 procarr = (int *)__kmp_allocate(sizeof(int) * nproc);
4575 for (int i = 0; i < nproc; i++) {
4576 procarr[i] = -1;
4577 }
4578
4579 int lastcore = -1;
4580 int inlastcore = 0;
4581 for (int i = 0; i < __kmp_avail_proc; i++) {
4582 int proc = address2os[i].second;
4583 int core =
4584 __kmp_affinity_find_core(address2os, i, depth - 1, core_level);
4585
4586 if (core == lastcore) {
4587 inlastcore++;
4588 } else {
4589 inlastcore = 0;
4590 }
4591 lastcore = core;
4592
4593 procarr[core * maxprocpercore + inlastcore] = proc;
4594 }
4595 }
4596 if (__kmp_affinity_compact >= depth) {
4597 __kmp_affinity_compact = depth - 1;
4598 }
4599
4600 sortAddresses:
4601 // Allocate the gtid->affinity mask table.
4602 if (__kmp_affinity_dups) {
4603 __kmp_affinity_num_masks = __kmp_avail_proc;
4604 } else {
4605 __kmp_affinity_num_masks = numUnique;
4606 }
4607
4608 if ((__kmp_nested_proc_bind.bind_types[0] != proc_bind_intel) &&
4609 (__kmp_affinity_num_places > 0) &&
4610 ((unsigned)__kmp_affinity_num_places < __kmp_affinity_num_masks)) {
4611 __kmp_affinity_num_masks = __kmp_affinity_num_places;
4612 }
4613
4614 KMP_CPU_ALLOC_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4615
4616 // Sort the address2os table according to the current setting of
4617 // __kmp_affinity_compact, then fill out __kmp_affinity_masks.
4618 qsort(address2os, __kmp_avail_proc, sizeof(*address2os),
4619 __kmp_affinity_cmp_Address_child_num);
4620 {
4621 int i;
4622 unsigned j;
4623 for (i = 0, j = 0; i < __kmp_avail_proc; i++) {
4624 if ((!__kmp_affinity_dups) && (!address2os[i].first.leader)) {
4625 continue;
4626 }
4627 unsigned osId = address2os[i].second;
4628 kmp_affin_mask_t *src = KMP_CPU_INDEX(osId2Mask, osId);
4629 kmp_affin_mask_t *dest = KMP_CPU_INDEX(__kmp_affinity_masks, j);
4630 KMP_ASSERT(KMP_CPU_ISSET(osId, src));
4631 KMP_CPU_COPY(dest, src);
4632 if (++j >= __kmp_affinity_num_masks) {
4633 break;
4634 }
4635 }
4636 KMP_DEBUG_ASSERT(j == __kmp_affinity_num_masks);
4637 }
4638 break;
4639
4640 default:
4641 KMP_ASSERT2(0, "Unexpected affinity setting");
4642 }
4643
4644 KMP_CPU_FREE_ARRAY(osId2Mask, maxIndex + 1);
4645 machine_hierarchy.init(address2os, __kmp_avail_proc);
4646 }
4647 #undef KMP_EXIT_AFF_NONE
4648
__kmp_affinity_initialize(void)4649 void __kmp_affinity_initialize(void) {
4650 // Much of the code above was written assumming that if a machine was not
4651 // affinity capable, then __kmp_affinity_type == affinity_none. We now
4652 // explicitly represent this as __kmp_affinity_type == affinity_disabled.
4653 // There are too many checks for __kmp_affinity_type == affinity_none
4654 // in this code. Instead of trying to change them all, check if
4655 // __kmp_affinity_type == affinity_disabled, and if so, slam it with
4656 // affinity_none, call the real initialization routine, then restore
4657 // __kmp_affinity_type to affinity_disabled.
4658 int disabled = (__kmp_affinity_type == affinity_disabled);
4659 if (!KMP_AFFINITY_CAPABLE()) {
4660 KMP_ASSERT(disabled);
4661 }
4662 if (disabled) {
4663 __kmp_affinity_type = affinity_none;
4664 }
4665 __kmp_aux_affinity_initialize();
4666 if (disabled) {
4667 __kmp_affinity_type = affinity_disabled;
4668 }
4669 }
4670
__kmp_affinity_uninitialize(void)4671 void __kmp_affinity_uninitialize(void) {
4672 if (__kmp_affinity_masks != NULL) {
4673 KMP_CPU_FREE_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4674 __kmp_affinity_masks = NULL;
4675 }
4676 if (__kmp_affin_fullMask != NULL) {
4677 KMP_CPU_FREE(__kmp_affin_fullMask);
4678 __kmp_affin_fullMask = NULL;
4679 }
4680 __kmp_affinity_num_masks = 0;
4681 __kmp_affinity_type = affinity_default;
4682 __kmp_affinity_num_places = 0;
4683 if (__kmp_affinity_proclist != NULL) {
4684 __kmp_free(__kmp_affinity_proclist);
4685 __kmp_affinity_proclist = NULL;
4686 }
4687 if (address2os != NULL) {
4688 __kmp_free(address2os);
4689 address2os = NULL;
4690 }
4691 if (procarr != NULL) {
4692 __kmp_free(procarr);
4693 procarr = NULL;
4694 }
4695 #if KMP_USE_HWLOC
4696 if (__kmp_hwloc_topology != NULL) {
4697 hwloc_topology_destroy(__kmp_hwloc_topology);
4698 __kmp_hwloc_topology = NULL;
4699 }
4700 #endif
4701 KMPAffinity::destroy_api();
4702 }
4703
__kmp_affinity_set_init_mask(int gtid,int isa_root)4704 void __kmp_affinity_set_init_mask(int gtid, int isa_root) {
4705 if (!KMP_AFFINITY_CAPABLE()) {
4706 return;
4707 }
4708
4709 kmp_info_t *th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[gtid]);
4710 if (th->th.th_affin_mask == NULL) {
4711 KMP_CPU_ALLOC(th->th.th_affin_mask);
4712 } else {
4713 KMP_CPU_ZERO(th->th.th_affin_mask);
4714 }
4715
4716 // Copy the thread mask to the kmp_info_t strucuture. If
4717 // __kmp_affinity_type == affinity_none, copy the "full" mask, i.e. one that
4718 // has all of the OS proc ids set, or if __kmp_affinity_respect_mask is set,
4719 // then the full mask is the same as the mask of the initialization thread.
4720 kmp_affin_mask_t *mask;
4721 int i;
4722
4723 if (KMP_AFFINITY_NON_PROC_BIND) {
4724 if ((__kmp_affinity_type == affinity_none) ||
4725 (__kmp_affinity_type == affinity_balanced)) {
4726 #if KMP_GROUP_AFFINITY
4727 if (__kmp_num_proc_groups > 1) {
4728 return;
4729 }
4730 #endif
4731 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4732 i = 0;
4733 mask = __kmp_affin_fullMask;
4734 } else {
4735 KMP_DEBUG_ASSERT(__kmp_affinity_num_masks > 0);
4736 i = (gtid + __kmp_affinity_offset) % __kmp_affinity_num_masks;
4737 mask = KMP_CPU_INDEX(__kmp_affinity_masks, i);
4738 }
4739 } else {
4740 if ((!isa_root) ||
4741 (__kmp_nested_proc_bind.bind_types[0] == proc_bind_false)) {
4742 #if KMP_GROUP_AFFINITY
4743 if (__kmp_num_proc_groups > 1) {
4744 return;
4745 }
4746 #endif
4747 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4748 i = KMP_PLACE_ALL;
4749 mask = __kmp_affin_fullMask;
4750 } else {
4751 // int i = some hash function or just a counter that doesn't
4752 // always start at 0. Use gtid for now.
4753 KMP_DEBUG_ASSERT(__kmp_affinity_num_masks > 0);
4754 i = (gtid + __kmp_affinity_offset) % __kmp_affinity_num_masks;
4755 mask = KMP_CPU_INDEX(__kmp_affinity_masks, i);
4756 }
4757 }
4758
4759 th->th.th_current_place = i;
4760 if (isa_root) {
4761 th->th.th_new_place = i;
4762 th->th.th_first_place = 0;
4763 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4764 } else if (KMP_AFFINITY_NON_PROC_BIND) {
4765 // When using a Non-OMP_PROC_BIND affinity method,
4766 // set all threads' place-partition-var to the entire place list
4767 th->th.th_first_place = 0;
4768 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4769 }
4770
4771 if (i == KMP_PLACE_ALL) {
4772 KA_TRACE(100, ("__kmp_affinity_set_init_mask: binding T#%d to all places\n",
4773 gtid));
4774 } else {
4775 KA_TRACE(100, ("__kmp_affinity_set_init_mask: binding T#%d to place %d\n",
4776 gtid, i));
4777 }
4778
4779 KMP_CPU_COPY(th->th.th_affin_mask, mask);
4780
4781 if (__kmp_affinity_verbose
4782 /* to avoid duplicate printing (will be correctly printed on barrier) */
4783 && (__kmp_affinity_type == affinity_none ||
4784 (i != KMP_PLACE_ALL && __kmp_affinity_type != affinity_balanced))) {
4785 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4786 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4787 th->th.th_affin_mask);
4788 KMP_INFORM(BoundToOSProcSet, "KMP_AFFINITY", (kmp_int32)getpid(),
4789 __kmp_gettid(), gtid, buf);
4790 }
4791
4792 #if KMP_OS_WINDOWS
4793 // On Windows* OS, the process affinity mask might have changed. If the user
4794 // didn't request affinity and this call fails, just continue silently.
4795 // See CQ171393.
4796 if (__kmp_affinity_type == affinity_none) {
4797 __kmp_set_system_affinity(th->th.th_affin_mask, FALSE);
4798 } else
4799 #endif
4800 __kmp_set_system_affinity(th->th.th_affin_mask, TRUE);
4801 }
4802
__kmp_affinity_set_place(int gtid)4803 void __kmp_affinity_set_place(int gtid) {
4804 if (!KMP_AFFINITY_CAPABLE()) {
4805 return;
4806 }
4807
4808 kmp_info_t *th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[gtid]);
4809
4810 KA_TRACE(100, ("__kmp_affinity_set_place: binding T#%d to place %d (current "
4811 "place = %d)\n",
4812 gtid, th->th.th_new_place, th->th.th_current_place));
4813
4814 // Check that the new place is within this thread's partition.
4815 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4816 KMP_ASSERT(th->th.th_new_place >= 0);
4817 KMP_ASSERT((unsigned)th->th.th_new_place <= __kmp_affinity_num_masks);
4818 if (th->th.th_first_place <= th->th.th_last_place) {
4819 KMP_ASSERT((th->th.th_new_place >= th->th.th_first_place) &&
4820 (th->th.th_new_place <= th->th.th_last_place));
4821 } else {
4822 KMP_ASSERT((th->th.th_new_place <= th->th.th_first_place) ||
4823 (th->th.th_new_place >= th->th.th_last_place));
4824 }
4825
4826 // Copy the thread mask to the kmp_info_t strucuture,
4827 // and set this thread's affinity.
4828 kmp_affin_mask_t *mask =
4829 KMP_CPU_INDEX(__kmp_affinity_masks, th->th.th_new_place);
4830 KMP_CPU_COPY(th->th.th_affin_mask, mask);
4831 th->th.th_current_place = th->th.th_new_place;
4832
4833 if (__kmp_affinity_verbose) {
4834 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4835 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4836 th->th.th_affin_mask);
4837 KMP_INFORM(BoundToOSProcSet, "OMP_PROC_BIND", (kmp_int32)getpid(),
4838 __kmp_gettid(), gtid, buf);
4839 }
4840 __kmp_set_system_affinity(th->th.th_affin_mask, TRUE);
4841 }
4842
__kmp_aux_set_affinity(void ** mask)4843 int __kmp_aux_set_affinity(void **mask) {
4844 int gtid;
4845 kmp_info_t *th;
4846 int retval;
4847
4848 if (!KMP_AFFINITY_CAPABLE()) {
4849 return -1;
4850 }
4851
4852 gtid = __kmp_entry_gtid();
4853 KA_TRACE(1000, (""); {
4854 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4855 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4856 (kmp_affin_mask_t *)(*mask));
4857 __kmp_debug_printf(
4858 "kmp_set_affinity: setting affinity mask for thread %d = %s\n", gtid,
4859 buf);
4860 });
4861
4862 if (__kmp_env_consistency_check) {
4863 if ((mask == NULL) || (*mask == NULL)) {
4864 KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
4865 } else {
4866 unsigned proc;
4867 int num_procs = 0;
4868
4869 KMP_CPU_SET_ITERATE(proc, ((kmp_affin_mask_t *)(*mask))) {
4870 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
4871 KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
4872 }
4873 if (!KMP_CPU_ISSET(proc, (kmp_affin_mask_t *)(*mask))) {
4874 continue;
4875 }
4876 num_procs++;
4877 }
4878 if (num_procs == 0) {
4879 KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
4880 }
4881
4882 #if KMP_GROUP_AFFINITY
4883 if (__kmp_get_proc_group((kmp_affin_mask_t *)(*mask)) < 0) {
4884 KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
4885 }
4886 #endif /* KMP_GROUP_AFFINITY */
4887 }
4888 }
4889
4890 th = __kmp_threads[gtid];
4891 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4892 retval = __kmp_set_system_affinity((kmp_affin_mask_t *)(*mask), FALSE);
4893 if (retval == 0) {
4894 KMP_CPU_COPY(th->th.th_affin_mask, (kmp_affin_mask_t *)(*mask));
4895 }
4896
4897 th->th.th_current_place = KMP_PLACE_UNDEFINED;
4898 th->th.th_new_place = KMP_PLACE_UNDEFINED;
4899 th->th.th_first_place = 0;
4900 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4901
4902 // Turn off 4.0 affinity for the current tread at this parallel level.
4903 th->th.th_current_task->td_icvs.proc_bind = proc_bind_false;
4904
4905 return retval;
4906 }
4907
__kmp_aux_get_affinity(void ** mask)4908 int __kmp_aux_get_affinity(void **mask) {
4909 int gtid;
4910 int retval;
4911 kmp_info_t *th;
4912
4913 if (!KMP_AFFINITY_CAPABLE()) {
4914 return -1;
4915 }
4916
4917 gtid = __kmp_entry_gtid();
4918 th = __kmp_threads[gtid];
4919 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4920
4921 KA_TRACE(1000, (""); {
4922 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4923 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4924 th->th.th_affin_mask);
4925 __kmp_printf("kmp_get_affinity: stored affinity mask for thread %d = %s\n",
4926 gtid, buf);
4927 });
4928
4929 if (__kmp_env_consistency_check) {
4930 if ((mask == NULL) || (*mask == NULL)) {
4931 KMP_FATAL(AffinityInvalidMask, "kmp_get_affinity");
4932 }
4933 }
4934
4935 #if !KMP_OS_WINDOWS
4936
4937 retval = __kmp_get_system_affinity((kmp_affin_mask_t *)(*mask), FALSE);
4938 KA_TRACE(1000, (""); {
4939 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4940 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4941 (kmp_affin_mask_t *)(*mask));
4942 __kmp_printf("kmp_get_affinity: system affinity mask for thread %d = %s\n",
4943 gtid, buf);
4944 });
4945 return retval;
4946
4947 #else
4948
4949 KMP_CPU_COPY((kmp_affin_mask_t *)(*mask), th->th.th_affin_mask);
4950 return 0;
4951
4952 #endif /* KMP_OS_WINDOWS */
4953 }
4954
__kmp_aux_get_affinity_max_proc()4955 int __kmp_aux_get_affinity_max_proc() {
4956 if (!KMP_AFFINITY_CAPABLE()) {
4957 return 0;
4958 }
4959 #if KMP_GROUP_AFFINITY
4960 if (__kmp_num_proc_groups > 1) {
4961 return (int)(__kmp_num_proc_groups * sizeof(DWORD_PTR) * CHAR_BIT);
4962 }
4963 #endif
4964 return __kmp_xproc;
4965 }
4966
__kmp_aux_set_affinity_mask_proc(int proc,void ** mask)4967 int __kmp_aux_set_affinity_mask_proc(int proc, void **mask) {
4968 if (!KMP_AFFINITY_CAPABLE()) {
4969 return -1;
4970 }
4971
4972 KA_TRACE(1000, (""); {
4973 int gtid = __kmp_entry_gtid();
4974 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4975 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4976 (kmp_affin_mask_t *)(*mask));
4977 __kmp_debug_printf("kmp_set_affinity_mask_proc: setting proc %d in "
4978 "affinity mask for thread %d = %s\n",
4979 proc, gtid, buf);
4980 });
4981
4982 if (__kmp_env_consistency_check) {
4983 if ((mask == NULL) || (*mask == NULL)) {
4984 KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity_mask_proc");
4985 }
4986 }
4987
4988 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
4989 return -1;
4990 }
4991 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
4992 return -2;
4993 }
4994
4995 KMP_CPU_SET(proc, (kmp_affin_mask_t *)(*mask));
4996 return 0;
4997 }
4998
__kmp_aux_unset_affinity_mask_proc(int proc,void ** mask)4999 int __kmp_aux_unset_affinity_mask_proc(int proc, void **mask) {
5000 if (!KMP_AFFINITY_CAPABLE()) {
5001 return -1;
5002 }
5003
5004 KA_TRACE(1000, (""); {
5005 int gtid = __kmp_entry_gtid();
5006 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5007 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
5008 (kmp_affin_mask_t *)(*mask));
5009 __kmp_debug_printf("kmp_unset_affinity_mask_proc: unsetting proc %d in "
5010 "affinity mask for thread %d = %s\n",
5011 proc, gtid, buf);
5012 });
5013
5014 if (__kmp_env_consistency_check) {
5015 if ((mask == NULL) || (*mask == NULL)) {
5016 KMP_FATAL(AffinityInvalidMask, "kmp_unset_affinity_mask_proc");
5017 }
5018 }
5019
5020 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
5021 return -1;
5022 }
5023 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
5024 return -2;
5025 }
5026
5027 KMP_CPU_CLR(proc, (kmp_affin_mask_t *)(*mask));
5028 return 0;
5029 }
5030
__kmp_aux_get_affinity_mask_proc(int proc,void ** mask)5031 int __kmp_aux_get_affinity_mask_proc(int proc, void **mask) {
5032 if (!KMP_AFFINITY_CAPABLE()) {
5033 return -1;
5034 }
5035
5036 KA_TRACE(1000, (""); {
5037 int gtid = __kmp_entry_gtid();
5038 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5039 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
5040 (kmp_affin_mask_t *)(*mask));
5041 __kmp_debug_printf("kmp_get_affinity_mask_proc: getting proc %d in "
5042 "affinity mask for thread %d = %s\n",
5043 proc, gtid, buf);
5044 });
5045
5046 if (__kmp_env_consistency_check) {
5047 if ((mask == NULL) || (*mask == NULL)) {
5048 KMP_FATAL(AffinityInvalidMask, "kmp_get_affinity_mask_proc");
5049 }
5050 }
5051
5052 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
5053 return -1;
5054 }
5055 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
5056 return 0;
5057 }
5058
5059 return KMP_CPU_ISSET(proc, (kmp_affin_mask_t *)(*mask));
5060 }
5061
5062 // Dynamic affinity settings - Affinity balanced
__kmp_balanced_affinity(kmp_info_t * th,int nthreads)5063 void __kmp_balanced_affinity(kmp_info_t *th, int nthreads) {
5064 KMP_DEBUG_ASSERT(th);
5065 bool fine_gran = true;
5066 int tid = th->th.th_info.ds.ds_tid;
5067
5068 switch (__kmp_affinity_gran) {
5069 case affinity_gran_fine:
5070 case affinity_gran_thread:
5071 break;
5072 case affinity_gran_core:
5073 if (__kmp_nThreadsPerCore > 1) {
5074 fine_gran = false;
5075 }
5076 break;
5077 case affinity_gran_package:
5078 if (nCoresPerPkg > 1) {
5079 fine_gran = false;
5080 }
5081 break;
5082 default:
5083 fine_gran = false;
5084 }
5085
5086 if (__kmp_affinity_uniform_topology()) {
5087 int coreID;
5088 int threadID;
5089 // Number of hyper threads per core in HT machine
5090 int __kmp_nth_per_core = __kmp_avail_proc / __kmp_ncores;
5091 // Number of cores
5092 int ncores = __kmp_ncores;
5093 if ((nPackages > 1) && (__kmp_nth_per_core <= 1)) {
5094 __kmp_nth_per_core = __kmp_avail_proc / nPackages;
5095 ncores = nPackages;
5096 }
5097 // How many threads will be bound to each core
5098 int chunk = nthreads / ncores;
5099 // How many cores will have an additional thread bound to it - "big cores"
5100 int big_cores = nthreads % ncores;
5101 // Number of threads on the big cores
5102 int big_nth = (chunk + 1) * big_cores;
5103 if (tid < big_nth) {
5104 coreID = tid / (chunk + 1);
5105 threadID = (tid % (chunk + 1)) % __kmp_nth_per_core;
5106 } else { // tid >= big_nth
5107 coreID = (tid - big_cores) / chunk;
5108 threadID = ((tid - big_cores) % chunk) % __kmp_nth_per_core;
5109 }
5110
5111 KMP_DEBUG_ASSERT2(KMP_AFFINITY_CAPABLE(),
5112 "Illegal set affinity operation when not capable");
5113
5114 kmp_affin_mask_t *mask = th->th.th_affin_mask;
5115 KMP_CPU_ZERO(mask);
5116
5117 if (fine_gran) {
5118 int osID = address2os[coreID * __kmp_nth_per_core + threadID].second;
5119 KMP_CPU_SET(osID, mask);
5120 } else {
5121 for (int i = 0; i < __kmp_nth_per_core; i++) {
5122 int osID;
5123 osID = address2os[coreID * __kmp_nth_per_core + i].second;
5124 KMP_CPU_SET(osID, mask);
5125 }
5126 }
5127 if (__kmp_affinity_verbose) {
5128 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5129 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, mask);
5130 KMP_INFORM(BoundToOSProcSet, "KMP_AFFINITY", (kmp_int32)getpid(),
5131 __kmp_gettid(), tid, buf);
5132 }
5133 __kmp_set_system_affinity(mask, TRUE);
5134 } else { // Non-uniform topology
5135
5136 kmp_affin_mask_t *mask = th->th.th_affin_mask;
5137 KMP_CPU_ZERO(mask);
5138
5139 int core_level = __kmp_affinity_find_core_level(
5140 address2os, __kmp_avail_proc, __kmp_aff_depth - 1);
5141 int ncores = __kmp_affinity_compute_ncores(address2os, __kmp_avail_proc,
5142 __kmp_aff_depth - 1, core_level);
5143 int nth_per_core = __kmp_affinity_max_proc_per_core(
5144 address2os, __kmp_avail_proc, __kmp_aff_depth - 1, core_level);
5145
5146 // For performance gain consider the special case nthreads ==
5147 // __kmp_avail_proc
5148 if (nthreads == __kmp_avail_proc) {
5149 if (fine_gran) {
5150 int osID = address2os[tid].second;
5151 KMP_CPU_SET(osID, mask);
5152 } else {
5153 int core = __kmp_affinity_find_core(address2os, tid,
5154 __kmp_aff_depth - 1, core_level);
5155 for (int i = 0; i < __kmp_avail_proc; i++) {
5156 int osID = address2os[i].second;
5157 if (__kmp_affinity_find_core(address2os, i, __kmp_aff_depth - 1,
5158 core_level) == core) {
5159 KMP_CPU_SET(osID, mask);
5160 }
5161 }
5162 }
5163 } else if (nthreads <= ncores) {
5164
5165 int core = 0;
5166 for (int i = 0; i < ncores; i++) {
5167 // Check if this core from procarr[] is in the mask
5168 int in_mask = 0;
5169 for (int j = 0; j < nth_per_core; j++) {
5170 if (procarr[i * nth_per_core + j] != -1) {
5171 in_mask = 1;
5172 break;
5173 }
5174 }
5175 if (in_mask) {
5176 if (tid == core) {
5177 for (int j = 0; j < nth_per_core; j++) {
5178 int osID = procarr[i * nth_per_core + j];
5179 if (osID != -1) {
5180 KMP_CPU_SET(osID, mask);
5181 // For fine granularity it is enough to set the first available
5182 // osID for this core
5183 if (fine_gran) {
5184 break;
5185 }
5186 }
5187 }
5188 break;
5189 } else {
5190 core++;
5191 }
5192 }
5193 }
5194 } else { // nthreads > ncores
5195 // Array to save the number of processors at each core
5196 int *nproc_at_core = (int *)KMP_ALLOCA(sizeof(int) * ncores);
5197 // Array to save the number of cores with "x" available processors;
5198 int *ncores_with_x_procs =
5199 (int *)KMP_ALLOCA(sizeof(int) * (nth_per_core + 1));
5200 // Array to save the number of cores with # procs from x to nth_per_core
5201 int *ncores_with_x_to_max_procs =
5202 (int *)KMP_ALLOCA(sizeof(int) * (nth_per_core + 1));
5203
5204 for (int i = 0; i <= nth_per_core; i++) {
5205 ncores_with_x_procs[i] = 0;
5206 ncores_with_x_to_max_procs[i] = 0;
5207 }
5208
5209 for (int i = 0; i < ncores; i++) {
5210 int cnt = 0;
5211 for (int j = 0; j < nth_per_core; j++) {
5212 if (procarr[i * nth_per_core + j] != -1) {
5213 cnt++;
5214 }
5215 }
5216 nproc_at_core[i] = cnt;
5217 ncores_with_x_procs[cnt]++;
5218 }
5219
5220 for (int i = 0; i <= nth_per_core; i++) {
5221 for (int j = i; j <= nth_per_core; j++) {
5222 ncores_with_x_to_max_procs[i] += ncores_with_x_procs[j];
5223 }
5224 }
5225
5226 // Max number of processors
5227 int nproc = nth_per_core * ncores;
5228 // An array to keep number of threads per each context
5229 int *newarr = (int *)__kmp_allocate(sizeof(int) * nproc);
5230 for (int i = 0; i < nproc; i++) {
5231 newarr[i] = 0;
5232 }
5233
5234 int nth = nthreads;
5235 int flag = 0;
5236 while (nth > 0) {
5237 for (int j = 1; j <= nth_per_core; j++) {
5238 int cnt = ncores_with_x_to_max_procs[j];
5239 for (int i = 0; i < ncores; i++) {
5240 // Skip the core with 0 processors
5241 if (nproc_at_core[i] == 0) {
5242 continue;
5243 }
5244 for (int k = 0; k < nth_per_core; k++) {
5245 if (procarr[i * nth_per_core + k] != -1) {
5246 if (newarr[i * nth_per_core + k] == 0) {
5247 newarr[i * nth_per_core + k] = 1;
5248 cnt--;
5249 nth--;
5250 break;
5251 } else {
5252 if (flag != 0) {
5253 newarr[i * nth_per_core + k]++;
5254 cnt--;
5255 nth--;
5256 break;
5257 }
5258 }
5259 }
5260 }
5261 if (cnt == 0 || nth == 0) {
5262 break;
5263 }
5264 }
5265 if (nth == 0) {
5266 break;
5267 }
5268 }
5269 flag = 1;
5270 }
5271 int sum = 0;
5272 for (int i = 0; i < nproc; i++) {
5273 sum += newarr[i];
5274 if (sum > tid) {
5275 if (fine_gran) {
5276 int osID = procarr[i];
5277 KMP_CPU_SET(osID, mask);
5278 } else {
5279 int coreID = i / nth_per_core;
5280 for (int ii = 0; ii < nth_per_core; ii++) {
5281 int osID = procarr[coreID * nth_per_core + ii];
5282 if (osID != -1) {
5283 KMP_CPU_SET(osID, mask);
5284 }
5285 }
5286 }
5287 break;
5288 }
5289 }
5290 __kmp_free(newarr);
5291 }
5292
5293 if (__kmp_affinity_verbose) {
5294 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5295 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, mask);
5296 KMP_INFORM(BoundToOSProcSet, "KMP_AFFINITY", (kmp_int32)getpid(),
5297 __kmp_gettid(), tid, buf);
5298 }
5299 __kmp_set_system_affinity(mask, TRUE);
5300 }
5301 }
5302
5303 #if KMP_OS_LINUX || KMP_OS_FREEBSD
5304 // We don't need this entry for Windows because
5305 // there is GetProcessAffinityMask() api
5306 //
5307 // The intended usage is indicated by these steps:
5308 // 1) The user gets the current affinity mask
5309 // 2) Then sets the affinity by calling this function
5310 // 3) Error check the return value
5311 // 4) Use non-OpenMP parallelization
5312 // 5) Reset the affinity to what was stored in step 1)
5313 #ifdef __cplusplus
5314 extern "C"
5315 #endif
5316 int
kmp_set_thread_affinity_mask_initial()5317 kmp_set_thread_affinity_mask_initial()
5318 // the function returns 0 on success,
5319 // -1 if we cannot bind thread
5320 // >0 (errno) if an error happened during binding
5321 {
5322 int gtid = __kmp_get_gtid();
5323 if (gtid < 0) {
5324 // Do not touch non-omp threads
5325 KA_TRACE(30, ("kmp_set_thread_affinity_mask_initial: "
5326 "non-omp thread, returning\n"));
5327 return -1;
5328 }
5329 if (!KMP_AFFINITY_CAPABLE() || !__kmp_init_middle) {
5330 KA_TRACE(30, ("kmp_set_thread_affinity_mask_initial: "
5331 "affinity not initialized, returning\n"));
5332 return -1;
5333 }
5334 KA_TRACE(30, ("kmp_set_thread_affinity_mask_initial: "
5335 "set full mask for thread %d\n",
5336 gtid));
5337 KMP_DEBUG_ASSERT(__kmp_affin_fullMask != NULL);
5338 return __kmp_set_system_affinity(__kmp_affin_fullMask, FALSE);
5339 }
5340 #endif
5341
5342 #endif // KMP_AFFINITY_SUPPORTED
5343