1 /* Copyright (C) 2006-2022 Free Software Foundation, Inc.
2 Contributed by Jakub Jelinek <jakub@redhat.com>.
3
4 This file is part of the GNU Offloading and Multi Processing Library
5 (libgomp).
6
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 more details.
16
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
20
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
25
26 /* This is a Linux specific implementation of a CPU affinity setting. */
27
28 #ifndef _GNU_SOURCE
29 #define _GNU_SOURCE 1
30 #endif
31 #include "libgomp.h"
32 #include "proc.h"
33 #include <errno.h>
34 #include <stdlib.h>
35 #include <stdio.h>
36 #include <string.h>
37 #include <unistd.h>
38 #include <limits.h>
39
40 #ifdef HAVE_PTHREAD_AFFINITY_NP
41
42 #ifndef CPU_ALLOC_SIZE
43 #define CPU_ISSET_S(idx, size, set) CPU_ISSET(idx, set)
44 #define CPU_ZERO_S(size, set) CPU_ZERO(set)
45 #define CPU_SET_S(idx, size, set) CPU_SET(idx, set)
46 #define CPU_CLR_S(idx, size, set) CPU_CLR(idx, set)
47 #endif
48
49 void
gomp_init_affinity(void)50 gomp_init_affinity (void)
51 {
52 if (gomp_places_list == NULL)
53 {
54 if (!gomp_affinity_init_level (1, ULONG_MAX, true))
55 return;
56 }
57
58 struct gomp_thread *thr = gomp_thread ();
59 pthread_setaffinity_np (pthread_self (), gomp_cpuset_size,
60 (cpu_set_t *) gomp_places_list[0]);
61 thr->place = 1;
62 thr->ts.place_partition_off = 0;
63 thr->ts.place_partition_len = gomp_places_list_len;
64 }
65
66 void
gomp_init_thread_affinity(pthread_attr_t * attr,unsigned int place)67 gomp_init_thread_affinity (pthread_attr_t *attr, unsigned int place)
68 {
69 pthread_attr_setaffinity_np (attr, gomp_cpuset_size,
70 (cpu_set_t *) gomp_places_list[place]);
71 }
72
73 void **
gomp_affinity_alloc(unsigned long count,bool quiet)74 gomp_affinity_alloc (unsigned long count, bool quiet)
75 {
76 unsigned long i;
77 void **ret;
78 char *p;
79
80 if (gomp_cpusetp == NULL)
81 {
82 if (!quiet)
83 gomp_error ("Could not get CPU affinity set");
84 return NULL;
85 }
86
87 ret = malloc (count * sizeof (void *) + count * gomp_cpuset_size);
88 if (ret == NULL)
89 {
90 if (!quiet)
91 gomp_error ("Out of memory trying to allocate places list");
92 return NULL;
93 }
94
95 p = (char *) (ret + count);
96 for (i = 0; i < count; i++, p += gomp_cpuset_size)
97 ret[i] = p;
98 return ret;
99 }
100
101 void
gomp_affinity_init_place(void * p)102 gomp_affinity_init_place (void *p)
103 {
104 cpu_set_t *cpusetp = (cpu_set_t *) p;
105 CPU_ZERO_S (gomp_cpuset_size, cpusetp);
106 }
107
108 bool
gomp_affinity_add_cpus(void * p,unsigned long num,unsigned long len,long stride,bool quiet)109 gomp_affinity_add_cpus (void *p, unsigned long num,
110 unsigned long len, long stride, bool quiet)
111 {
112 cpu_set_t *cpusetp = (cpu_set_t *) p;
113 unsigned long max = 8 * gomp_cpuset_size;
114 for (;;)
115 {
116 if (num >= max)
117 {
118 if (!quiet)
119 gomp_error ("Logical CPU number %lu out of range", num);
120 return false;
121 }
122 CPU_SET_S (num, gomp_cpuset_size, cpusetp);
123 if (--len == 0)
124 return true;
125 if ((stride < 0 && num + stride > num)
126 || (stride > 0 && num + stride < num))
127 {
128 if (!quiet)
129 gomp_error ("Logical CPU number %lu+%ld out of range",
130 num, stride);
131 return false;
132 }
133 num += stride;
134 }
135 }
136
137 bool
gomp_affinity_remove_cpu(void * p,unsigned long num)138 gomp_affinity_remove_cpu (void *p, unsigned long num)
139 {
140 cpu_set_t *cpusetp = (cpu_set_t *) p;
141 if (num >= 8 * gomp_cpuset_size)
142 {
143 gomp_error ("Logical CPU number %lu out of range", num);
144 return false;
145 }
146 if (!CPU_ISSET_S (num, gomp_cpuset_size, cpusetp))
147 {
148 gomp_error ("Logical CPU %lu to be removed is not in the set", num);
149 return false;
150 }
151 CPU_CLR_S (num, gomp_cpuset_size, cpusetp);
152 return true;
153 }
154
155 bool
gomp_affinity_copy_place(void * p,void * q,long stride)156 gomp_affinity_copy_place (void *p, void *q, long stride)
157 {
158 unsigned long i, max = 8 * gomp_cpuset_size;
159 cpu_set_t *destp = (cpu_set_t *) p;
160 cpu_set_t *srcp = (cpu_set_t *) q;
161
162 CPU_ZERO_S (gomp_cpuset_size, destp);
163 for (i = 0; i < max; i++)
164 if (CPU_ISSET_S (i, gomp_cpuset_size, srcp))
165 {
166 if ((stride < 0 && i + stride > i)
167 || (stride > 0 && (i + stride < i || i + stride >= max)))
168 {
169 gomp_error ("Logical CPU number %lu+%ld out of range", i, stride);
170 return false;
171 }
172 CPU_SET_S (i + stride, gomp_cpuset_size, destp);
173 }
174 return true;
175 }
176
177 bool
gomp_affinity_same_place(void * p,void * q)178 gomp_affinity_same_place (void *p, void *q)
179 {
180 #ifdef CPU_EQUAL_S
181 return CPU_EQUAL_S (gomp_cpuset_size, (cpu_set_t *) p, (cpu_set_t *) q);
182 #else
183 return memcmp (p, q, gomp_cpuset_size) == 0;
184 #endif
185 }
186
187 bool
gomp_affinity_finalize_place_list(bool quiet)188 gomp_affinity_finalize_place_list (bool quiet)
189 {
190 unsigned long i, j;
191
192 for (i = 0, j = 0; i < gomp_places_list_len; i++)
193 {
194 cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[i];
195 bool nonempty = false;
196 #ifdef CPU_AND_S
197 CPU_AND_S (gomp_cpuset_size, cpusetp, cpusetp, gomp_cpusetp);
198 nonempty = gomp_cpuset_popcount (gomp_cpuset_size, cpusetp) != 0;
199 #else
200 unsigned long k, max = gomp_cpuset_size / sizeof (cpusetp->__bits[0]);
201 for (k = 0; k < max; k++)
202 if ((cpusetp->__bits[k] &= gomp_cpusetp->__bits[k]) != 0)
203 nonempty = true;
204 #endif
205 if (nonempty)
206 gomp_places_list[j++] = gomp_places_list[i];
207 }
208
209 if (j == 0)
210 {
211 if (!quiet)
212 gomp_error ("None of the places contain usable logical CPUs");
213 return false;
214 }
215 else if (j < gomp_places_list_len)
216 {
217 if (!quiet)
218 gomp_error ("Number of places reduced from %ld to %ld because some "
219 "places didn't contain any usable logical CPUs",
220 gomp_places_list_len, j);
221 gomp_places_list_len = j;
222 }
223 return true;
224 }
225
226 /* Find the index of the last level cache. We assume the index
227 of the last level cache is the same for all logical CPUs.
228 Also, if there are multiple caches with the same highest level,
229 assume they have the same shared_cpu_list and pick the last one
230 from them (highest index number). */
231
232 static int
gomp_affinity_find_last_cache_level(char * name,size_t prefix_len,unsigned long cpu)233 gomp_affinity_find_last_cache_level (char *name, size_t prefix_len,
234 unsigned long cpu)
235 {
236 int ret = -1;
237 unsigned long maxval = 0;
238 char *line = NULL;
239 size_t linelen = 0;
240 FILE *f;
241
242 for (int l = 0; l < 128; l++)
243 {
244 sprintf (name + prefix_len, "%lu/cache/index%u/level", cpu, l);
245 f = fopen (name, "r");
246 if (f == NULL)
247 break;
248 if (getline (&line, &linelen, f) > 0)
249 {
250 unsigned long val;
251 char *p;
252 errno = 0;
253 val = strtoul (line, &p, 10);
254 if (!errno && p > line && val >= maxval)
255 {
256 ret = l;
257 maxval = val;
258 }
259 }
260 fclose (f);
261 }
262 free (line);
263 return ret;
264 }
265
266 static void
gomp_affinity_init_level_1(int level,int this_level,unsigned long count,cpu_set_t * copy,char * name,bool quiet)267 gomp_affinity_init_level_1 (int level, int this_level, unsigned long count,
268 cpu_set_t *copy, char *name, bool quiet)
269 {
270 size_t prefix_len = sizeof ("/sys/devices/system/cpu/cpu") - 1;
271 FILE *f;
272 char *line = NULL;
273 size_t linelen = 0;
274 unsigned long i, max = 8 * gomp_cpuset_size;
275 int init = -1;
276
277 for (i = 0; i < max && gomp_places_list_len < count; i++)
278 if (CPU_ISSET_S (i, gomp_cpuset_size, copy))
279 {
280 if (level == 4)
281 {
282 if (init == -1)
283 {
284 init = gomp_affinity_find_last_cache_level (name, prefix_len,
285 i);
286 if (init == -1)
287 {
288 CPU_CLR_S (i, gomp_cpuset_size, copy);
289 continue;
290 }
291 sprintf (name + prefix_len,
292 "%lu/cache/index%u/shared_cpu_list", i, init);
293 }
294 }
295 else
296 sprintf (name + prefix_len, "%lu/topology/%s_siblings_list",
297 i, this_level == 3 ? "core" : "thread");
298 f = fopen (name, "r");
299 if (f == NULL)
300 {
301 CPU_CLR_S (i, gomp_cpuset_size, copy);
302 continue;
303 }
304 if (getline (&line, &linelen, f) > 0)
305 {
306 char *p = line, *end;
307 void *pl = gomp_places_list[gomp_places_list_len];
308 if (level == this_level)
309 gomp_affinity_init_place (pl);
310 while (*p && *p != '\n')
311 {
312 unsigned long first, last;
313 errno = 0;
314 first = strtoul (p, &end, 10);
315 if (errno || end == p)
316 break;
317 p = end;
318 last = first;
319 if (*p == '-')
320 {
321 errno = 0;
322 last = strtoul (p + 1, &end, 10);
323 if (errno || end == p + 1 || last < first)
324 break;
325 p = end;
326 }
327 for (; first <= last; first++)
328 if (!CPU_ISSET_S (first, gomp_cpuset_size, copy))
329 continue;
330 else if (this_level == 3 && level < this_level)
331 gomp_affinity_init_level_1 (level, 2, count, copy,
332 name, quiet);
333 else
334 {
335 if (level == 1)
336 {
337 pl = gomp_places_list[gomp_places_list_len];
338 gomp_affinity_init_place (pl);
339 }
340 if (gomp_affinity_add_cpus (pl, first, 1, 0, true))
341 {
342 CPU_CLR_S (first, gomp_cpuset_size, copy);
343 if (level == 1
344 && ++gomp_places_list_len >= count)
345 {
346 fclose (f);
347 free (line);
348 return;
349 }
350 }
351 }
352 if (*p == ',')
353 ++p;
354 }
355 if (level == this_level
356 && !CPU_ISSET_S (i, gomp_cpuset_size, copy))
357 gomp_places_list_len++;
358 CPU_CLR_S (i, gomp_cpuset_size, copy);
359 }
360 fclose (f);
361 }
362 free (line);
363 }
364
365 static void
gomp_affinity_init_numa_domains(unsigned long count,cpu_set_t * copy,char * name)366 gomp_affinity_init_numa_domains (unsigned long count, cpu_set_t *copy,
367 char *name)
368 {
369 FILE *f;
370 char *nline = NULL, *line = NULL;
371 size_t nlinelen = 0, linelen = 0;
372 char *q;
373 size_t prefix_len = sizeof ("/sys/devices/system/node/") - 1;
374
375 strcpy (name, "/sys/devices/system/node/online");
376 f = fopen (name, "r");
377 if (f == NULL || getline (&nline, &nlinelen, f) <= 0)
378 {
379 if (f)
380 fclose (f);
381 return;
382 }
383 fclose (f);
384 q = nline;
385 while (*q && *q != '\n' && gomp_places_list_len < count)
386 {
387 unsigned long nfirst, nlast;
388 char *end;
389
390 errno = 0;
391 nfirst = strtoul (q, &end, 10);
392 if (errno || end == q)
393 break;
394 q = end;
395 nlast = nfirst;
396 if (*q == '-')
397 {
398 errno = 0;
399 nlast = strtoul (q + 1, &end, 10);
400 if (errno || end == q + 1 || nlast < nfirst)
401 break;
402 q = end;
403 }
404 for (; nfirst <= nlast && gomp_places_list_len < count; nfirst++)
405 {
406 sprintf (name + prefix_len, "node%lu/cpulist", nfirst);
407 f = fopen (name, "r");
408 if (f == NULL)
409 continue;
410 if (getline (&line, &linelen, f) > 0)
411 {
412 char *p = line;
413 void *pl = NULL;
414 bool seen = false;
415
416 while (*p && *p != '\n')
417 {
418 unsigned long first, last;
419
420 errno = 0;
421 first = strtoul (p, &end, 10);
422 if (errno || end == p)
423 break;
424 p = end;
425 last = first;
426 if (*p == '-')
427 {
428 errno = 0;
429 last = strtoul (p + 1, &end, 10);
430 if (errno || end == p + 1 || last < first)
431 break;
432 p = end;
433 }
434 for (; first <= last; first++)
435 {
436 if (!CPU_ISSET_S (first, gomp_cpuset_size, copy))
437 continue;
438 if (pl == NULL)
439 {
440 pl = gomp_places_list[gomp_places_list_len];
441 gomp_affinity_init_place (pl);
442 }
443 if (gomp_affinity_add_cpus (pl, first, 1, 0, true))
444 {
445 CPU_CLR_S (first, gomp_cpuset_size, copy);
446 if (!seen)
447 {
448 gomp_places_list_len++;
449 seen = true;
450 }
451 }
452 }
453 if (*p == ',')
454 ++p;
455 }
456 }
457 fclose (f);
458 }
459 if (*q == ',')
460 ++q;
461 }
462 free (line);
463 free (nline);
464 }
465
466 bool
gomp_affinity_init_level(int level,unsigned long count,bool quiet)467 gomp_affinity_init_level (int level, unsigned long count, bool quiet)
468 {
469 char name[sizeof ("/sys/devices/system/cpu/cpu/topology/"
470 "thread_siblings_list") + 6 * sizeof (unsigned long)];
471 cpu_set_t *copy;
472
473 if (gomp_cpusetp)
474 {
475 unsigned long maxcount
476 = gomp_cpuset_popcount (gomp_cpuset_size, gomp_cpusetp);
477 if (count > maxcount)
478 count = maxcount;
479 }
480 gomp_places_list = gomp_affinity_alloc (count, quiet);
481 gomp_places_list_len = 0;
482 if (gomp_places_list == NULL)
483 return false;
484
485 copy = gomp_alloca (gomp_cpuset_size);
486 strcpy (name, "/sys/devices/system/cpu/cpu");
487 memcpy (copy, gomp_cpusetp, gomp_cpuset_size);
488 if (level == 5)
489 gomp_affinity_init_numa_domains (count, copy, name);
490 else
491 gomp_affinity_init_level_1 (level, level > 3 ? level : 3, count, copy,
492 name, quiet);
493 if (gomp_places_list_len == 0)
494 {
495 if (!quiet)
496 gomp_error ("Error reading core/socket topology");
497 free (gomp_places_list);
498 gomp_places_list = NULL;
499 return false;
500 }
501 return true;
502 }
503
504 void
gomp_affinity_print_place(void * p)505 gomp_affinity_print_place (void *p)
506 {
507 unsigned long i, max = 8 * gomp_cpuset_size, len;
508 cpu_set_t *cpusetp = (cpu_set_t *) p;
509 bool notfirst = false;
510
511 for (i = 0, len = 0; i < max; i++)
512 if (CPU_ISSET_S (i, gomp_cpuset_size, cpusetp))
513 {
514 if (len == 0)
515 {
516 if (notfirst)
517 fputc (',', stderr);
518 notfirst = true;
519 fprintf (stderr, "%lu", i);
520 }
521 ++len;
522 }
523 else
524 {
525 if (len > 1)
526 fprintf (stderr, ":%lu", len);
527 len = 0;
528 }
529 if (len > 1)
530 fprintf (stderr, ":%lu", len);
531 }
532
533 int
omp_get_place_num_procs(int place_num)534 omp_get_place_num_procs (int place_num)
535 {
536 if (place_num < 0 || place_num >= gomp_places_list_len)
537 return 0;
538
539 cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[place_num];
540 return gomp_cpuset_popcount (gomp_cpuset_size, cpusetp);
541 }
542
543 void
omp_get_place_proc_ids(int place_num,int * ids)544 omp_get_place_proc_ids (int place_num, int *ids)
545 {
546 if (place_num < 0 || place_num >= gomp_places_list_len)
547 return;
548
549 cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[place_num];
550 unsigned long i, max = 8 * gomp_cpuset_size;
551 for (i = 0; i < max; i++)
552 if (CPU_ISSET_S (i, gomp_cpuset_size, cpusetp))
553 *ids++ = i;
554 }
555
556 void
gomp_get_place_proc_ids_8(int place_num,int64_t * ids)557 gomp_get_place_proc_ids_8 (int place_num, int64_t *ids)
558 {
559 if (place_num < 0 || place_num >= gomp_places_list_len)
560 return;
561
562 cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[place_num];
563 unsigned long i, max = 8 * gomp_cpuset_size;
564 for (i = 0; i < max; i++)
565 if (CPU_ISSET_S (i, gomp_cpuset_size, cpusetp))
566 *ids++ = i;
567 }
568
569 void
gomp_display_affinity_place(char * buffer,size_t size,size_t * ret,int place)570 gomp_display_affinity_place (char *buffer, size_t size, size_t *ret,
571 int place)
572 {
573 cpu_set_t *cpusetp;
574 char buf[sizeof (long) * 3 + 4];
575 if (place >= 0 && place < gomp_places_list_len)
576 cpusetp = (cpu_set_t *) gomp_places_list[place];
577 else if (gomp_cpusetp)
578 cpusetp = gomp_cpusetp;
579 else
580 {
581 if (gomp_available_cpus > 1)
582 sprintf (buf, "0-%lu", gomp_available_cpus - 1);
583 else
584 strcpy (buf, "0");
585 gomp_display_string (buffer, size, ret, buf, strlen (buf));
586 return;
587 }
588
589 unsigned long i, max = 8 * gomp_cpuset_size, start;
590 bool prev_set = false;
591 start = max;
592 for (i = 0; i <= max; i++)
593 {
594 bool this_set;
595 if (i == max)
596 this_set = false;
597 else
598 this_set = CPU_ISSET_S (i, gomp_cpuset_size, cpusetp);
599 if (this_set != prev_set)
600 {
601 prev_set = this_set;
602 if (this_set)
603 {
604 char *p = buf;
605 if (start != max)
606 *p++ = ',';
607 sprintf (p, "%lu", i);
608 start = i;
609 }
610 else if (i == start + 1)
611 continue;
612 else
613 sprintf (buf, "-%lu", i - 1);
614 gomp_display_string (buffer, size, ret, buf, strlen (buf));
615 }
616 }
617 }
618
619 ialias(omp_get_place_num_procs)
620 ialias(omp_get_place_proc_ids)
621
622 #else
623
624 #include "../../affinity.c"
625
626 #endif
627