1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * numa.c 4 * 5 * numa: Simulate NUMA-sensitive workload and measure their NUMA performance 6 */ 7 8 #include <inttypes.h> 9 /* For the CLR_() macros */ 10 #include <pthread.h> 11 12 #include <subcmd/parse-options.h> 13 #include "../util/cloexec.h" 14 15 #include "bench.h" 16 17 #include <errno.h> 18 #include <sched.h> 19 #include <stdio.h> 20 #include <assert.h> 21 #include <malloc.h> 22 #include <signal.h> 23 #include <stdlib.h> 24 #include <string.h> 25 #include <unistd.h> 26 #include <sys/mman.h> 27 #include <sys/time.h> 28 #include <sys/resource.h> 29 #include <sys/wait.h> 30 #include <sys/prctl.h> 31 #include <sys/types.h> 32 #include <linux/kernel.h> 33 #include <linux/time64.h> 34 #include <linux/numa.h> 35 #include <linux/zalloc.h> 36 37 #include <numa.h> 38 #include <numaif.h> 39 40 #ifndef RUSAGE_THREAD 41 # define RUSAGE_THREAD 1 42 #endif 43 44 /* 45 * Regular printout to the terminal, supressed if -q is specified: 46 */ 47 #define tprintf(x...) do { if (g && g->p.show_details >= 0) printf(x); } while (0) 48 49 /* 50 * Debug printf: 51 */ 52 #undef dprintf 53 #define dprintf(x...) do { if (g && g->p.show_details >= 1) printf(x); } while (0) 54 55 struct thread_data { 56 int curr_cpu; 57 cpu_set_t bind_cpumask; 58 int bind_node; 59 u8 *process_data; 60 int process_nr; 61 int thread_nr; 62 int task_nr; 63 unsigned int loops_done; 64 u64 val; 65 u64 runtime_ns; 66 u64 system_time_ns; 67 u64 user_time_ns; 68 double speed_gbs; 69 pthread_mutex_t *process_lock; 70 }; 71 72 /* Parameters set by options: */ 73 74 struct params { 75 /* Startup synchronization: */ 76 bool serialize_startup; 77 78 /* Task hierarchy: */ 79 int nr_proc; 80 int nr_threads; 81 82 /* Working set sizes: */ 83 const char *mb_global_str; 84 const char *mb_proc_str; 85 const char *mb_proc_locked_str; 86 const char *mb_thread_str; 87 88 double mb_global; 89 double mb_proc; 90 double mb_proc_locked; 91 double mb_thread; 92 93 /* Access patterns to the working set: */ 94 bool data_reads; 95 bool data_writes; 96 bool data_backwards; 97 bool data_zero_memset; 98 bool data_rand_walk; 99 u32 nr_loops; 100 u32 nr_secs; 101 u32 sleep_usecs; 102 103 /* Working set initialization: */ 104 bool init_zero; 105 bool init_random; 106 bool init_cpu0; 107 108 /* Misc options: */ 109 int show_details; 110 int run_all; 111 int thp; 112 113 long bytes_global; 114 long bytes_process; 115 long bytes_process_locked; 116 long bytes_thread; 117 118 int nr_tasks; 119 bool show_quiet; 120 121 bool show_convergence; 122 bool measure_convergence; 123 124 int perturb_secs; 125 int nr_cpus; 126 int nr_nodes; 127 128 /* Affinity options -C and -N: */ 129 char *cpu_list_str; 130 char *node_list_str; 131 }; 132 133 134 /* Global, read-writable area, accessible to all processes and threads: */ 135 136 struct global_info { 137 u8 *data; 138 139 pthread_mutex_t startup_mutex; 140 int nr_tasks_started; 141 142 pthread_mutex_t startup_done_mutex; 143 144 pthread_mutex_t start_work_mutex; 145 int nr_tasks_working; 146 147 pthread_mutex_t stop_work_mutex; 148 u64 bytes_done; 149 150 struct thread_data *threads; 151 152 /* Convergence latency measurement: */ 153 bool all_converged; 154 bool stop_work; 155 156 int print_once; 157 158 struct params p; 159 }; 160 161 static struct global_info *g = NULL; 162 163 static int parse_cpus_opt(const struct option *opt, const char *arg, int unset); 164 static int parse_nodes_opt(const struct option *opt, const char *arg, int unset); 165 166 struct params p0; 167 168 static const struct option options[] = { 169 OPT_INTEGER('p', "nr_proc" , &p0.nr_proc, "number of processes"), 170 OPT_INTEGER('t', "nr_threads" , &p0.nr_threads, "number of threads per process"), 171 172 OPT_STRING('G', "mb_global" , &p0.mb_global_str, "MB", "global memory (MBs)"), 173 OPT_STRING('P', "mb_proc" , &p0.mb_proc_str, "MB", "process memory (MBs)"), 174 OPT_STRING('L', "mb_proc_locked", &p0.mb_proc_locked_str,"MB", "process serialized/locked memory access (MBs), <= process_memory"), 175 OPT_STRING('T', "mb_thread" , &p0.mb_thread_str, "MB", "thread memory (MBs)"), 176 177 OPT_UINTEGER('l', "nr_loops" , &p0.nr_loops, "max number of loops to run (default: unlimited)"), 178 OPT_UINTEGER('s', "nr_secs" , &p0.nr_secs, "max number of seconds to run (default: 5 secs)"), 179 OPT_UINTEGER('u', "usleep" , &p0.sleep_usecs, "usecs to sleep per loop iteration"), 180 181 OPT_BOOLEAN('R', "data_reads" , &p0.data_reads, "access the data via reads (can be mixed with -W)"), 182 OPT_BOOLEAN('W', "data_writes" , &p0.data_writes, "access the data via writes (can be mixed with -R)"), 183 OPT_BOOLEAN('B', "data_backwards", &p0.data_backwards, "access the data backwards as well"), 184 OPT_BOOLEAN('Z', "data_zero_memset", &p0.data_zero_memset,"access the data via glibc bzero only"), 185 OPT_BOOLEAN('r', "data_rand_walk", &p0.data_rand_walk, "access the data with random (32bit LFSR) walk"), 186 187 188 OPT_BOOLEAN('z', "init_zero" , &p0.init_zero, "bzero the initial allocations"), 189 OPT_BOOLEAN('I', "init_random" , &p0.init_random, "randomize the contents of the initial allocations"), 190 OPT_BOOLEAN('0', "init_cpu0" , &p0.init_cpu0, "do the initial allocations on CPU#0"), 191 OPT_INTEGER('x', "perturb_secs", &p0.perturb_secs, "perturb thread 0/0 every X secs, to test convergence stability"), 192 193 OPT_INCR ('d', "show_details" , &p0.show_details, "Show details"), 194 OPT_INCR ('a', "all" , &p0.run_all, "Run all tests in the suite"), 195 OPT_INTEGER('H', "thp" , &p0.thp, "MADV_NOHUGEPAGE < 0 < MADV_HUGEPAGE"), 196 OPT_BOOLEAN('c', "show_convergence", &p0.show_convergence, "show convergence details, " 197 "convergence is reached when each process (all its threads) is running on a single NUMA node."), 198 OPT_BOOLEAN('m', "measure_convergence", &p0.measure_convergence, "measure convergence latency"), 199 OPT_BOOLEAN('q', "quiet" , &p0.show_quiet, "quiet mode"), 200 OPT_BOOLEAN('S', "serialize-startup", &p0.serialize_startup,"serialize thread startup"), 201 202 /* Special option string parsing callbacks: */ 203 OPT_CALLBACK('C', "cpus", NULL, "cpu[,cpu2,...cpuN]", 204 "bind the first N tasks to these specific cpus (the rest is unbound)", 205 parse_cpus_opt), 206 OPT_CALLBACK('M', "memnodes", NULL, "node[,node2,...nodeN]", 207 "bind the first N tasks to these specific memory nodes (the rest is unbound)", 208 parse_nodes_opt), 209 OPT_END() 210 }; 211 212 static const char * const bench_numa_usage[] = { 213 "perf bench numa <options>", 214 NULL 215 }; 216 217 static const char * const numa_usage[] = { 218 "perf bench numa mem [<options>]", 219 NULL 220 }; 221 222 /* 223 * To get number of numa nodes present. 224 */ 225 static int nr_numa_nodes(void) 226 { 227 int i, nr_nodes = 0; 228 229 for (i = 0; i < g->p.nr_nodes; i++) { 230 if (numa_bitmask_isbitset(numa_nodes_ptr, i)) 231 nr_nodes++; 232 } 233 234 return nr_nodes; 235 } 236 237 /* 238 * To check if given numa node is present. 239 */ 240 static int is_node_present(int node) 241 { 242 return numa_bitmask_isbitset(numa_nodes_ptr, node); 243 } 244 245 /* 246 * To check given numa node has cpus. 247 */ 248 static bool node_has_cpus(int node) 249 { 250 struct bitmask *cpu = numa_allocate_cpumask(); 251 unsigned int i; 252 253 if (cpu && !numa_node_to_cpus(node, cpu)) { 254 for (i = 0; i < cpu->size; i++) { 255 if (numa_bitmask_isbitset(cpu, i)) 256 return true; 257 } 258 } 259 260 return false; /* lets fall back to nocpus safely */ 261 } 262 263 static cpu_set_t bind_to_cpu(int target_cpu) 264 { 265 cpu_set_t orig_mask, mask; 266 int ret; 267 268 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask); 269 BUG_ON(ret); 270 271 CPU_ZERO(&mask); 272 273 if (target_cpu == -1) { 274 int cpu; 275 276 for (cpu = 0; cpu < g->p.nr_cpus; cpu++) 277 CPU_SET(cpu, &mask); 278 } else { 279 BUG_ON(target_cpu < 0 || target_cpu >= g->p.nr_cpus); 280 CPU_SET(target_cpu, &mask); 281 } 282 283 ret = sched_setaffinity(0, sizeof(mask), &mask); 284 BUG_ON(ret); 285 286 return orig_mask; 287 } 288 289 static cpu_set_t bind_to_node(int target_node) 290 { 291 int cpus_per_node = g->p.nr_cpus / nr_numa_nodes(); 292 cpu_set_t orig_mask, mask; 293 int cpu; 294 int ret; 295 296 BUG_ON(cpus_per_node * nr_numa_nodes() != g->p.nr_cpus); 297 BUG_ON(!cpus_per_node); 298 299 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask); 300 BUG_ON(ret); 301 302 CPU_ZERO(&mask); 303 304 if (target_node == NUMA_NO_NODE) { 305 for (cpu = 0; cpu < g->p.nr_cpus; cpu++) 306 CPU_SET(cpu, &mask); 307 } else { 308 int cpu_start = (target_node + 0) * cpus_per_node; 309 int cpu_stop = (target_node + 1) * cpus_per_node; 310 311 BUG_ON(cpu_stop > g->p.nr_cpus); 312 313 for (cpu = cpu_start; cpu < cpu_stop; cpu++) 314 CPU_SET(cpu, &mask); 315 } 316 317 ret = sched_setaffinity(0, sizeof(mask), &mask); 318 BUG_ON(ret); 319 320 return orig_mask; 321 } 322 323 static void bind_to_cpumask(cpu_set_t mask) 324 { 325 int ret; 326 327 ret = sched_setaffinity(0, sizeof(mask), &mask); 328 BUG_ON(ret); 329 } 330 331 static void mempol_restore(void) 332 { 333 int ret; 334 335 ret = set_mempolicy(MPOL_DEFAULT, NULL, g->p.nr_nodes-1); 336 337 BUG_ON(ret); 338 } 339 340 static void bind_to_memnode(int node) 341 { 342 unsigned long nodemask; 343 int ret; 344 345 if (node == NUMA_NO_NODE) 346 return; 347 348 BUG_ON(g->p.nr_nodes > (int)sizeof(nodemask)*8); 349 nodemask = 1L << node; 350 351 ret = set_mempolicy(MPOL_BIND, &nodemask, sizeof(nodemask)*8); 352 dprintf("binding to node %d, mask: %016lx => %d\n", node, nodemask, ret); 353 354 BUG_ON(ret); 355 } 356 357 #define HPSIZE (2*1024*1024) 358 359 #define set_taskname(fmt...) \ 360 do { \ 361 char name[20]; \ 362 \ 363 snprintf(name, 20, fmt); \ 364 prctl(PR_SET_NAME, name); \ 365 } while (0) 366 367 static u8 *alloc_data(ssize_t bytes0, int map_flags, 368 int init_zero, int init_cpu0, int thp, int init_random) 369 { 370 cpu_set_t orig_mask; 371 ssize_t bytes; 372 u8 *buf; 373 int ret; 374 375 if (!bytes0) 376 return NULL; 377 378 /* Allocate and initialize all memory on CPU#0: */ 379 if (init_cpu0) { 380 int node = numa_node_of_cpu(0); 381 382 orig_mask = bind_to_node(node); 383 bind_to_memnode(node); 384 } 385 386 bytes = bytes0 + HPSIZE; 387 388 buf = (void *)mmap(0, bytes, PROT_READ|PROT_WRITE, MAP_ANON|map_flags, -1, 0); 389 BUG_ON(buf == (void *)-1); 390 391 if (map_flags == MAP_PRIVATE) { 392 if (thp > 0) { 393 ret = madvise(buf, bytes, MADV_HUGEPAGE); 394 if (ret && !g->print_once) { 395 g->print_once = 1; 396 printf("WARNING: Could not enable THP - do: 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'\n"); 397 } 398 } 399 if (thp < 0) { 400 ret = madvise(buf, bytes, MADV_NOHUGEPAGE); 401 if (ret && !g->print_once) { 402 g->print_once = 1; 403 printf("WARNING: Could not disable THP: run a CONFIG_TRANSPARENT_HUGEPAGE kernel?\n"); 404 } 405 } 406 } 407 408 if (init_zero) { 409 bzero(buf, bytes); 410 } else { 411 /* Initialize random contents, different in each word: */ 412 if (init_random) { 413 u64 *wbuf = (void *)buf; 414 long off = rand(); 415 long i; 416 417 for (i = 0; i < bytes/8; i++) 418 wbuf[i] = i + off; 419 } 420 } 421 422 /* Align to 2MB boundary: */ 423 buf = (void *)(((unsigned long)buf + HPSIZE-1) & ~(HPSIZE-1)); 424 425 /* Restore affinity: */ 426 if (init_cpu0) { 427 bind_to_cpumask(orig_mask); 428 mempol_restore(); 429 } 430 431 return buf; 432 } 433 434 static void free_data(void *data, ssize_t bytes) 435 { 436 int ret; 437 438 if (!data) 439 return; 440 441 ret = munmap(data, bytes); 442 BUG_ON(ret); 443 } 444 445 /* 446 * Create a shared memory buffer that can be shared between processes, zeroed: 447 */ 448 static void * zalloc_shared_data(ssize_t bytes) 449 { 450 return alloc_data(bytes, MAP_SHARED, 1, g->p.init_cpu0, g->p.thp, g->p.init_random); 451 } 452 453 /* 454 * Create a shared memory buffer that can be shared between processes: 455 */ 456 static void * setup_shared_data(ssize_t bytes) 457 { 458 return alloc_data(bytes, MAP_SHARED, 0, g->p.init_cpu0, g->p.thp, g->p.init_random); 459 } 460 461 /* 462 * Allocate process-local memory - this will either be shared between 463 * threads of this process, or only be accessed by this thread: 464 */ 465 static void * setup_private_data(ssize_t bytes) 466 { 467 return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0, g->p.thp, g->p.init_random); 468 } 469 470 /* 471 * Return a process-shared (global) mutex: 472 */ 473 static void init_global_mutex(pthread_mutex_t *mutex) 474 { 475 pthread_mutexattr_t attr; 476 477 pthread_mutexattr_init(&attr); 478 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); 479 pthread_mutex_init(mutex, &attr); 480 } 481 482 static int parse_cpu_list(const char *arg) 483 { 484 p0.cpu_list_str = strdup(arg); 485 486 dprintf("got CPU list: {%s}\n", p0.cpu_list_str); 487 488 return 0; 489 } 490 491 static int parse_setup_cpu_list(void) 492 { 493 struct thread_data *td; 494 char *str0, *str; 495 int t; 496 497 if (!g->p.cpu_list_str) 498 return 0; 499 500 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks); 501 502 str0 = str = strdup(g->p.cpu_list_str); 503 t = 0; 504 505 BUG_ON(!str); 506 507 tprintf("# binding tasks to CPUs:\n"); 508 tprintf("# "); 509 510 while (true) { 511 int bind_cpu, bind_cpu_0, bind_cpu_1; 512 char *tok, *tok_end, *tok_step, *tok_len, *tok_mul; 513 int bind_len; 514 int step; 515 int mul; 516 517 tok = strsep(&str, ","); 518 if (!tok) 519 break; 520 521 tok_end = strstr(tok, "-"); 522 523 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end); 524 if (!tok_end) { 525 /* Single CPU specified: */ 526 bind_cpu_0 = bind_cpu_1 = atol(tok); 527 } else { 528 /* CPU range specified (for example: "5-11"): */ 529 bind_cpu_0 = atol(tok); 530 bind_cpu_1 = atol(tok_end + 1); 531 } 532 533 step = 1; 534 tok_step = strstr(tok, "#"); 535 if (tok_step) { 536 step = atol(tok_step + 1); 537 BUG_ON(step <= 0 || step >= g->p.nr_cpus); 538 } 539 540 /* 541 * Mask length. 542 * Eg: "--cpus 8_4-16#4" means: '--cpus 8_4,12_4,16_4', 543 * where the _4 means the next 4 CPUs are allowed. 544 */ 545 bind_len = 1; 546 tok_len = strstr(tok, "_"); 547 if (tok_len) { 548 bind_len = atol(tok_len + 1); 549 BUG_ON(bind_len <= 0 || bind_len > g->p.nr_cpus); 550 } 551 552 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */ 553 mul = 1; 554 tok_mul = strstr(tok, "x"); 555 if (tok_mul) { 556 mul = atol(tok_mul + 1); 557 BUG_ON(mul <= 0); 558 } 559 560 dprintf("CPUs: %d_%d-%d#%dx%d\n", bind_cpu_0, bind_len, bind_cpu_1, step, mul); 561 562 if (bind_cpu_0 >= g->p.nr_cpus || bind_cpu_1 >= g->p.nr_cpus) { 563 printf("\nTest not applicable, system has only %d CPUs.\n", g->p.nr_cpus); 564 return -1; 565 } 566 567 BUG_ON(bind_cpu_0 < 0 || bind_cpu_1 < 0); 568 BUG_ON(bind_cpu_0 > bind_cpu_1); 569 570 for (bind_cpu = bind_cpu_0; bind_cpu <= bind_cpu_1; bind_cpu += step) { 571 int i; 572 573 for (i = 0; i < mul; i++) { 574 int cpu; 575 576 if (t >= g->p.nr_tasks) { 577 printf("\n# NOTE: ignoring bind CPUs starting at CPU#%d\n #", bind_cpu); 578 goto out; 579 } 580 td = g->threads + t; 581 582 if (t) 583 tprintf(","); 584 if (bind_len > 1) { 585 tprintf("%2d/%d", bind_cpu, bind_len); 586 } else { 587 tprintf("%2d", bind_cpu); 588 } 589 590 CPU_ZERO(&td->bind_cpumask); 591 for (cpu = bind_cpu; cpu < bind_cpu+bind_len; cpu++) { 592 BUG_ON(cpu < 0 || cpu >= g->p.nr_cpus); 593 CPU_SET(cpu, &td->bind_cpumask); 594 } 595 t++; 596 } 597 } 598 } 599 out: 600 601 tprintf("\n"); 602 603 if (t < g->p.nr_tasks) 604 printf("# NOTE: %d tasks bound, %d tasks unbound\n", t, g->p.nr_tasks - t); 605 606 free(str0); 607 return 0; 608 } 609 610 static int parse_cpus_opt(const struct option *opt __maybe_unused, 611 const char *arg, int unset __maybe_unused) 612 { 613 if (!arg) 614 return -1; 615 616 return parse_cpu_list(arg); 617 } 618 619 static int parse_node_list(const char *arg) 620 { 621 p0.node_list_str = strdup(arg); 622 623 dprintf("got NODE list: {%s}\n", p0.node_list_str); 624 625 return 0; 626 } 627 628 static int parse_setup_node_list(void) 629 { 630 struct thread_data *td; 631 char *str0, *str; 632 int t; 633 634 if (!g->p.node_list_str) 635 return 0; 636 637 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks); 638 639 str0 = str = strdup(g->p.node_list_str); 640 t = 0; 641 642 BUG_ON(!str); 643 644 tprintf("# binding tasks to NODEs:\n"); 645 tprintf("# "); 646 647 while (true) { 648 int bind_node, bind_node_0, bind_node_1; 649 char *tok, *tok_end, *tok_step, *tok_mul; 650 int step; 651 int mul; 652 653 tok = strsep(&str, ","); 654 if (!tok) 655 break; 656 657 tok_end = strstr(tok, "-"); 658 659 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end); 660 if (!tok_end) { 661 /* Single NODE specified: */ 662 bind_node_0 = bind_node_1 = atol(tok); 663 } else { 664 /* NODE range specified (for example: "5-11"): */ 665 bind_node_0 = atol(tok); 666 bind_node_1 = atol(tok_end + 1); 667 } 668 669 step = 1; 670 tok_step = strstr(tok, "#"); 671 if (tok_step) { 672 step = atol(tok_step + 1); 673 BUG_ON(step <= 0 || step >= g->p.nr_nodes); 674 } 675 676 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */ 677 mul = 1; 678 tok_mul = strstr(tok, "x"); 679 if (tok_mul) { 680 mul = atol(tok_mul + 1); 681 BUG_ON(mul <= 0); 682 } 683 684 dprintf("NODEs: %d-%d #%d\n", bind_node_0, bind_node_1, step); 685 686 if (bind_node_0 >= g->p.nr_nodes || bind_node_1 >= g->p.nr_nodes) { 687 printf("\nTest not applicable, system has only %d nodes.\n", g->p.nr_nodes); 688 return -1; 689 } 690 691 BUG_ON(bind_node_0 < 0 || bind_node_1 < 0); 692 BUG_ON(bind_node_0 > bind_node_1); 693 694 for (bind_node = bind_node_0; bind_node <= bind_node_1; bind_node += step) { 695 int i; 696 697 for (i = 0; i < mul; i++) { 698 if (t >= g->p.nr_tasks || !node_has_cpus(bind_node)) { 699 printf("\n# NOTE: ignoring bind NODEs starting at NODE#%d\n", bind_node); 700 goto out; 701 } 702 td = g->threads + t; 703 704 if (!t) 705 tprintf(" %2d", bind_node); 706 else 707 tprintf(",%2d", bind_node); 708 709 td->bind_node = bind_node; 710 t++; 711 } 712 } 713 } 714 out: 715 716 tprintf("\n"); 717 718 if (t < g->p.nr_tasks) 719 printf("# NOTE: %d tasks mem-bound, %d tasks unbound\n", t, g->p.nr_tasks - t); 720 721 free(str0); 722 return 0; 723 } 724 725 static int parse_nodes_opt(const struct option *opt __maybe_unused, 726 const char *arg, int unset __maybe_unused) 727 { 728 if (!arg) 729 return -1; 730 731 return parse_node_list(arg); 732 733 return 0; 734 } 735 736 #define BIT(x) (1ul << x) 737 738 static inline uint32_t lfsr_32(uint32_t lfsr) 739 { 740 const uint32_t taps = BIT(1) | BIT(5) | BIT(6) | BIT(31); 741 return (lfsr>>1) ^ ((0x0u - (lfsr & 0x1u)) & taps); 742 } 743 744 /* 745 * Make sure there's real data dependency to RAM (when read 746 * accesses are enabled), so the compiler, the CPU and the 747 * kernel (KSM, zero page, etc.) cannot optimize away RAM 748 * accesses: 749 */ 750 static inline u64 access_data(u64 *data, u64 val) 751 { 752 if (g->p.data_reads) 753 val += *data; 754 if (g->p.data_writes) 755 *data = val + 1; 756 return val; 757 } 758 759 /* 760 * The worker process does two types of work, a forwards going 761 * loop and a backwards going loop. 762 * 763 * We do this so that on multiprocessor systems we do not create 764 * a 'train' of processing, with highly synchronized processes, 765 * skewing the whole benchmark. 766 */ 767 static u64 do_work(u8 *__data, long bytes, int nr, int nr_max, int loop, u64 val) 768 { 769 long words = bytes/sizeof(u64); 770 u64 *data = (void *)__data; 771 long chunk_0, chunk_1; 772 u64 *d0, *d, *d1; 773 long off; 774 long i; 775 776 BUG_ON(!data && words); 777 BUG_ON(data && !words); 778 779 if (!data) 780 return val; 781 782 /* Very simple memset() work variant: */ 783 if (g->p.data_zero_memset && !g->p.data_rand_walk) { 784 bzero(data, bytes); 785 return val; 786 } 787 788 /* Spread out by PID/TID nr and by loop nr: */ 789 chunk_0 = words/nr_max; 790 chunk_1 = words/g->p.nr_loops; 791 off = nr*chunk_0 + loop*chunk_1; 792 793 while (off >= words) 794 off -= words; 795 796 if (g->p.data_rand_walk) { 797 u32 lfsr = nr + loop + val; 798 int j; 799 800 for (i = 0; i < words/1024; i++) { 801 long start, end; 802 803 lfsr = lfsr_32(lfsr); 804 805 start = lfsr % words; 806 end = min(start + 1024, words-1); 807 808 if (g->p.data_zero_memset) { 809 bzero(data + start, (end-start) * sizeof(u64)); 810 } else { 811 for (j = start; j < end; j++) 812 val = access_data(data + j, val); 813 } 814 } 815 } else if (!g->p.data_backwards || (nr + loop) & 1) { 816 817 d0 = data + off; 818 d = data + off + 1; 819 d1 = data + words; 820 821 /* Process data forwards: */ 822 for (;;) { 823 if (unlikely(d >= d1)) 824 d = data; 825 if (unlikely(d == d0)) 826 break; 827 828 val = access_data(d, val); 829 830 d++; 831 } 832 } else { 833 /* Process data backwards: */ 834 835 d0 = data + off; 836 d = data + off - 1; 837 d1 = data + words; 838 839 /* Process data forwards: */ 840 for (;;) { 841 if (unlikely(d < data)) 842 d = data + words-1; 843 if (unlikely(d == d0)) 844 break; 845 846 val = access_data(d, val); 847 848 d--; 849 } 850 } 851 852 return val; 853 } 854 855 static void update_curr_cpu(int task_nr, unsigned long bytes_worked) 856 { 857 unsigned int cpu; 858 859 cpu = sched_getcpu(); 860 861 g->threads[task_nr].curr_cpu = cpu; 862 prctl(0, bytes_worked); 863 } 864 865 #define MAX_NR_NODES 64 866 867 /* 868 * Count the number of nodes a process's threads 869 * are spread out on. 870 * 871 * A count of 1 means that the process is compressed 872 * to a single node. A count of g->p.nr_nodes means it's 873 * spread out on the whole system. 874 */ 875 static int count_process_nodes(int process_nr) 876 { 877 char node_present[MAX_NR_NODES] = { 0, }; 878 int nodes; 879 int n, t; 880 881 for (t = 0; t < g->p.nr_threads; t++) { 882 struct thread_data *td; 883 int task_nr; 884 int node; 885 886 task_nr = process_nr*g->p.nr_threads + t; 887 td = g->threads + task_nr; 888 889 node = numa_node_of_cpu(td->curr_cpu); 890 if (node < 0) /* curr_cpu was likely still -1 */ 891 return 0; 892 893 node_present[node] = 1; 894 } 895 896 nodes = 0; 897 898 for (n = 0; n < MAX_NR_NODES; n++) 899 nodes += node_present[n]; 900 901 return nodes; 902 } 903 904 /* 905 * Count the number of distinct process-threads a node contains. 906 * 907 * A count of 1 means that the node contains only a single 908 * process. If all nodes on the system contain at most one 909 * process then we are well-converged. 910 */ 911 static int count_node_processes(int node) 912 { 913 int processes = 0; 914 int t, p; 915 916 for (p = 0; p < g->p.nr_proc; p++) { 917 for (t = 0; t < g->p.nr_threads; t++) { 918 struct thread_data *td; 919 int task_nr; 920 int n; 921 922 task_nr = p*g->p.nr_threads + t; 923 td = g->threads + task_nr; 924 925 n = numa_node_of_cpu(td->curr_cpu); 926 if (n == node) { 927 processes++; 928 break; 929 } 930 } 931 } 932 933 return processes; 934 } 935 936 static void calc_convergence_compression(int *strong) 937 { 938 unsigned int nodes_min, nodes_max; 939 int p; 940 941 nodes_min = -1; 942 nodes_max = 0; 943 944 for (p = 0; p < g->p.nr_proc; p++) { 945 unsigned int nodes = count_process_nodes(p); 946 947 if (!nodes) { 948 *strong = 0; 949 return; 950 } 951 952 nodes_min = min(nodes, nodes_min); 953 nodes_max = max(nodes, nodes_max); 954 } 955 956 /* Strong convergence: all threads compress on a single node: */ 957 if (nodes_min == 1 && nodes_max == 1) { 958 *strong = 1; 959 } else { 960 *strong = 0; 961 tprintf(" {%d-%d}", nodes_min, nodes_max); 962 } 963 } 964 965 static void calc_convergence(double runtime_ns_max, double *convergence) 966 { 967 unsigned int loops_done_min, loops_done_max; 968 int process_groups; 969 int nodes[MAX_NR_NODES]; 970 int distance; 971 int nr_min; 972 int nr_max; 973 int strong; 974 int sum; 975 int nr; 976 int node; 977 int cpu; 978 int t; 979 980 if (!g->p.show_convergence && !g->p.measure_convergence) 981 return; 982 983 for (node = 0; node < g->p.nr_nodes; node++) 984 nodes[node] = 0; 985 986 loops_done_min = -1; 987 loops_done_max = 0; 988 989 for (t = 0; t < g->p.nr_tasks; t++) { 990 struct thread_data *td = g->threads + t; 991 unsigned int loops_done; 992 993 cpu = td->curr_cpu; 994 995 /* Not all threads have written it yet: */ 996 if (cpu < 0) 997 continue; 998 999 node = numa_node_of_cpu(cpu); 1000 1001 nodes[node]++; 1002 1003 loops_done = td->loops_done; 1004 loops_done_min = min(loops_done, loops_done_min); 1005 loops_done_max = max(loops_done, loops_done_max); 1006 } 1007 1008 nr_max = 0; 1009 nr_min = g->p.nr_tasks; 1010 sum = 0; 1011 1012 for (node = 0; node < g->p.nr_nodes; node++) { 1013 if (!is_node_present(node)) 1014 continue; 1015 nr = nodes[node]; 1016 nr_min = min(nr, nr_min); 1017 nr_max = max(nr, nr_max); 1018 sum += nr; 1019 } 1020 BUG_ON(nr_min > nr_max); 1021 1022 BUG_ON(sum > g->p.nr_tasks); 1023 1024 if (0 && (sum < g->p.nr_tasks)) 1025 return; 1026 1027 /* 1028 * Count the number of distinct process groups present 1029 * on nodes - when we are converged this will decrease 1030 * to g->p.nr_proc: 1031 */ 1032 process_groups = 0; 1033 1034 for (node = 0; node < g->p.nr_nodes; node++) { 1035 int processes; 1036 1037 if (!is_node_present(node)) 1038 continue; 1039 processes = count_node_processes(node); 1040 nr = nodes[node]; 1041 tprintf(" %2d/%-2d", nr, processes); 1042 1043 process_groups += processes; 1044 } 1045 1046 distance = nr_max - nr_min; 1047 1048 tprintf(" [%2d/%-2d]", distance, process_groups); 1049 1050 tprintf(" l:%3d-%-3d (%3d)", 1051 loops_done_min, loops_done_max, loops_done_max-loops_done_min); 1052 1053 if (loops_done_min && loops_done_max) { 1054 double skew = 1.0 - (double)loops_done_min/loops_done_max; 1055 1056 tprintf(" [%4.1f%%]", skew * 100.0); 1057 } 1058 1059 calc_convergence_compression(&strong); 1060 1061 if (strong && process_groups == g->p.nr_proc) { 1062 if (!*convergence) { 1063 *convergence = runtime_ns_max; 1064 tprintf(" (%6.1fs converged)\n", *convergence / NSEC_PER_SEC); 1065 if (g->p.measure_convergence) { 1066 g->all_converged = true; 1067 g->stop_work = true; 1068 } 1069 } 1070 } else { 1071 if (*convergence) { 1072 tprintf(" (%6.1fs de-converged)", runtime_ns_max / NSEC_PER_SEC); 1073 *convergence = 0; 1074 } 1075 tprintf("\n"); 1076 } 1077 } 1078 1079 static void show_summary(double runtime_ns_max, int l, double *convergence) 1080 { 1081 tprintf("\r # %5.1f%% [%.1f mins]", 1082 (double)(l+1)/g->p.nr_loops*100.0, runtime_ns_max / NSEC_PER_SEC / 60.0); 1083 1084 calc_convergence(runtime_ns_max, convergence); 1085 1086 if (g->p.show_details >= 0) 1087 fflush(stdout); 1088 } 1089 1090 static void *worker_thread(void *__tdata) 1091 { 1092 struct thread_data *td = __tdata; 1093 struct timeval start0, start, stop, diff; 1094 int process_nr = td->process_nr; 1095 int thread_nr = td->thread_nr; 1096 unsigned long last_perturbance; 1097 int task_nr = td->task_nr; 1098 int details = g->p.show_details; 1099 int first_task, last_task; 1100 double convergence = 0; 1101 u64 val = td->val; 1102 double runtime_ns_max; 1103 u8 *global_data; 1104 u8 *process_data; 1105 u8 *thread_data; 1106 u64 bytes_done, secs; 1107 long work_done; 1108 u32 l; 1109 struct rusage rusage; 1110 1111 bind_to_cpumask(td->bind_cpumask); 1112 bind_to_memnode(td->bind_node); 1113 1114 set_taskname("thread %d/%d", process_nr, thread_nr); 1115 1116 global_data = g->data; 1117 process_data = td->process_data; 1118 thread_data = setup_private_data(g->p.bytes_thread); 1119 1120 bytes_done = 0; 1121 1122 last_task = 0; 1123 if (process_nr == g->p.nr_proc-1 && thread_nr == g->p.nr_threads-1) 1124 last_task = 1; 1125 1126 first_task = 0; 1127 if (process_nr == 0 && thread_nr == 0) 1128 first_task = 1; 1129 1130 if (details >= 2) { 1131 printf("# thread %2d / %2d global mem: %p, process mem: %p, thread mem: %p\n", 1132 process_nr, thread_nr, global_data, process_data, thread_data); 1133 } 1134 1135 if (g->p.serialize_startup) { 1136 pthread_mutex_lock(&g->startup_mutex); 1137 g->nr_tasks_started++; 1138 pthread_mutex_unlock(&g->startup_mutex); 1139 1140 /* Here we will wait for the main process to start us all at once: */ 1141 pthread_mutex_lock(&g->start_work_mutex); 1142 g->nr_tasks_working++; 1143 1144 /* Last one wake the main process: */ 1145 if (g->nr_tasks_working == g->p.nr_tasks) 1146 pthread_mutex_unlock(&g->startup_done_mutex); 1147 1148 pthread_mutex_unlock(&g->start_work_mutex); 1149 } 1150 1151 gettimeofday(&start0, NULL); 1152 1153 start = stop = start0; 1154 last_perturbance = start.tv_sec; 1155 1156 for (l = 0; l < g->p.nr_loops; l++) { 1157 start = stop; 1158 1159 if (g->stop_work) 1160 break; 1161 1162 val += do_work(global_data, g->p.bytes_global, process_nr, g->p.nr_proc, l, val); 1163 val += do_work(process_data, g->p.bytes_process, thread_nr, g->p.nr_threads, l, val); 1164 val += do_work(thread_data, g->p.bytes_thread, 0, 1, l, val); 1165 1166 if (g->p.sleep_usecs) { 1167 pthread_mutex_lock(td->process_lock); 1168 usleep(g->p.sleep_usecs); 1169 pthread_mutex_unlock(td->process_lock); 1170 } 1171 /* 1172 * Amount of work to be done under a process-global lock: 1173 */ 1174 if (g->p.bytes_process_locked) { 1175 pthread_mutex_lock(td->process_lock); 1176 val += do_work(process_data, g->p.bytes_process_locked, thread_nr, g->p.nr_threads, l, val); 1177 pthread_mutex_unlock(td->process_lock); 1178 } 1179 1180 work_done = g->p.bytes_global + g->p.bytes_process + 1181 g->p.bytes_process_locked + g->p.bytes_thread; 1182 1183 update_curr_cpu(task_nr, work_done); 1184 bytes_done += work_done; 1185 1186 if (details < 0 && !g->p.perturb_secs && !g->p.measure_convergence && !g->p.nr_secs) 1187 continue; 1188 1189 td->loops_done = l; 1190 1191 gettimeofday(&stop, NULL); 1192 1193 /* Check whether our max runtime timed out: */ 1194 if (g->p.nr_secs) { 1195 timersub(&stop, &start0, &diff); 1196 if ((u32)diff.tv_sec >= g->p.nr_secs) { 1197 g->stop_work = true; 1198 break; 1199 } 1200 } 1201 1202 /* Update the summary at most once per second: */ 1203 if (start.tv_sec == stop.tv_sec) 1204 continue; 1205 1206 /* 1207 * Perturb the first task's equilibrium every g->p.perturb_secs seconds, 1208 * by migrating to CPU#0: 1209 */ 1210 if (first_task && g->p.perturb_secs && (int)(stop.tv_sec - last_perturbance) >= g->p.perturb_secs) { 1211 cpu_set_t orig_mask; 1212 int target_cpu; 1213 int this_cpu; 1214 1215 last_perturbance = stop.tv_sec; 1216 1217 /* 1218 * Depending on where we are running, move into 1219 * the other half of the system, to create some 1220 * real disturbance: 1221 */ 1222 this_cpu = g->threads[task_nr].curr_cpu; 1223 if (this_cpu < g->p.nr_cpus/2) 1224 target_cpu = g->p.nr_cpus-1; 1225 else 1226 target_cpu = 0; 1227 1228 orig_mask = bind_to_cpu(target_cpu); 1229 1230 /* Here we are running on the target CPU already */ 1231 if (details >= 1) 1232 printf(" (injecting perturbalance, moved to CPU#%d)\n", target_cpu); 1233 1234 bind_to_cpumask(orig_mask); 1235 } 1236 1237 if (details >= 3) { 1238 timersub(&stop, &start, &diff); 1239 runtime_ns_max = diff.tv_sec * NSEC_PER_SEC; 1240 runtime_ns_max += diff.tv_usec * NSEC_PER_USEC; 1241 1242 if (details >= 0) { 1243 printf(" #%2d / %2d: %14.2lf nsecs/op [val: %016"PRIx64"]\n", 1244 process_nr, thread_nr, runtime_ns_max / bytes_done, val); 1245 } 1246 fflush(stdout); 1247 } 1248 if (!last_task) 1249 continue; 1250 1251 timersub(&stop, &start0, &diff); 1252 runtime_ns_max = diff.tv_sec * NSEC_PER_SEC; 1253 runtime_ns_max += diff.tv_usec * NSEC_PER_USEC; 1254 1255 show_summary(runtime_ns_max, l, &convergence); 1256 } 1257 1258 gettimeofday(&stop, NULL); 1259 timersub(&stop, &start0, &diff); 1260 td->runtime_ns = diff.tv_sec * NSEC_PER_SEC; 1261 td->runtime_ns += diff.tv_usec * NSEC_PER_USEC; 1262 secs = td->runtime_ns / NSEC_PER_SEC; 1263 td->speed_gbs = secs ? bytes_done / secs / 1e9 : 0; 1264 1265 getrusage(RUSAGE_THREAD, &rusage); 1266 td->system_time_ns = rusage.ru_stime.tv_sec * NSEC_PER_SEC; 1267 td->system_time_ns += rusage.ru_stime.tv_usec * NSEC_PER_USEC; 1268 td->user_time_ns = rusage.ru_utime.tv_sec * NSEC_PER_SEC; 1269 td->user_time_ns += rusage.ru_utime.tv_usec * NSEC_PER_USEC; 1270 1271 free_data(thread_data, g->p.bytes_thread); 1272 1273 pthread_mutex_lock(&g->stop_work_mutex); 1274 g->bytes_done += bytes_done; 1275 pthread_mutex_unlock(&g->stop_work_mutex); 1276 1277 return NULL; 1278 } 1279 1280 /* 1281 * A worker process starts a couple of threads: 1282 */ 1283 static void worker_process(int process_nr) 1284 { 1285 pthread_mutex_t process_lock; 1286 struct thread_data *td; 1287 pthread_t *pthreads; 1288 u8 *process_data; 1289 int task_nr; 1290 int ret; 1291 int t; 1292 1293 pthread_mutex_init(&process_lock, NULL); 1294 set_taskname("process %d", process_nr); 1295 1296 /* 1297 * Pick up the memory policy and the CPU binding of our first thread, 1298 * so that we initialize memory accordingly: 1299 */ 1300 task_nr = process_nr*g->p.nr_threads; 1301 td = g->threads + task_nr; 1302 1303 bind_to_memnode(td->bind_node); 1304 bind_to_cpumask(td->bind_cpumask); 1305 1306 pthreads = zalloc(g->p.nr_threads * sizeof(pthread_t)); 1307 process_data = setup_private_data(g->p.bytes_process); 1308 1309 if (g->p.show_details >= 3) { 1310 printf(" # process %2d global mem: %p, process mem: %p\n", 1311 process_nr, g->data, process_data); 1312 } 1313 1314 for (t = 0; t < g->p.nr_threads; t++) { 1315 task_nr = process_nr*g->p.nr_threads + t; 1316 td = g->threads + task_nr; 1317 1318 td->process_data = process_data; 1319 td->process_nr = process_nr; 1320 td->thread_nr = t; 1321 td->task_nr = task_nr; 1322 td->val = rand(); 1323 td->curr_cpu = -1; 1324 td->process_lock = &process_lock; 1325 1326 ret = pthread_create(pthreads + t, NULL, worker_thread, td); 1327 BUG_ON(ret); 1328 } 1329 1330 for (t = 0; t < g->p.nr_threads; t++) { 1331 ret = pthread_join(pthreads[t], NULL); 1332 BUG_ON(ret); 1333 } 1334 1335 free_data(process_data, g->p.bytes_process); 1336 free(pthreads); 1337 } 1338 1339 static void print_summary(void) 1340 { 1341 if (g->p.show_details < 0) 1342 return; 1343 1344 printf("\n ###\n"); 1345 printf(" # %d %s will execute (on %d nodes, %d CPUs):\n", 1346 g->p.nr_tasks, g->p.nr_tasks == 1 ? "task" : "tasks", nr_numa_nodes(), g->p.nr_cpus); 1347 printf(" # %5dx %5ldMB global shared mem operations\n", 1348 g->p.nr_loops, g->p.bytes_global/1024/1024); 1349 printf(" # %5dx %5ldMB process shared mem operations\n", 1350 g->p.nr_loops, g->p.bytes_process/1024/1024); 1351 printf(" # %5dx %5ldMB thread local mem operations\n", 1352 g->p.nr_loops, g->p.bytes_thread/1024/1024); 1353 1354 printf(" ###\n"); 1355 1356 printf("\n ###\n"); fflush(stdout); 1357 } 1358 1359 static void init_thread_data(void) 1360 { 1361 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks; 1362 int t; 1363 1364 g->threads = zalloc_shared_data(size); 1365 1366 for (t = 0; t < g->p.nr_tasks; t++) { 1367 struct thread_data *td = g->threads + t; 1368 int cpu; 1369 1370 /* Allow all nodes by default: */ 1371 td->bind_node = NUMA_NO_NODE; 1372 1373 /* Allow all CPUs by default: */ 1374 CPU_ZERO(&td->bind_cpumask); 1375 for (cpu = 0; cpu < g->p.nr_cpus; cpu++) 1376 CPU_SET(cpu, &td->bind_cpumask); 1377 } 1378 } 1379 1380 static void deinit_thread_data(void) 1381 { 1382 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks; 1383 1384 free_data(g->threads, size); 1385 } 1386 1387 static int init(void) 1388 { 1389 g = (void *)alloc_data(sizeof(*g), MAP_SHARED, 1, 0, 0 /* THP */, 0); 1390 1391 /* Copy over options: */ 1392 g->p = p0; 1393 1394 g->p.nr_cpus = numa_num_configured_cpus(); 1395 1396 g->p.nr_nodes = numa_max_node() + 1; 1397 1398 /* char array in count_process_nodes(): */ 1399 BUG_ON(g->p.nr_nodes > MAX_NR_NODES || g->p.nr_nodes < 0); 1400 1401 if (g->p.show_quiet && !g->p.show_details) 1402 g->p.show_details = -1; 1403 1404 /* Some memory should be specified: */ 1405 if (!g->p.mb_global_str && !g->p.mb_proc_str && !g->p.mb_thread_str) 1406 return -1; 1407 1408 if (g->p.mb_global_str) { 1409 g->p.mb_global = atof(g->p.mb_global_str); 1410 BUG_ON(g->p.mb_global < 0); 1411 } 1412 1413 if (g->p.mb_proc_str) { 1414 g->p.mb_proc = atof(g->p.mb_proc_str); 1415 BUG_ON(g->p.mb_proc < 0); 1416 } 1417 1418 if (g->p.mb_proc_locked_str) { 1419 g->p.mb_proc_locked = atof(g->p.mb_proc_locked_str); 1420 BUG_ON(g->p.mb_proc_locked < 0); 1421 BUG_ON(g->p.mb_proc_locked > g->p.mb_proc); 1422 } 1423 1424 if (g->p.mb_thread_str) { 1425 g->p.mb_thread = atof(g->p.mb_thread_str); 1426 BUG_ON(g->p.mb_thread < 0); 1427 } 1428 1429 BUG_ON(g->p.nr_threads <= 0); 1430 BUG_ON(g->p.nr_proc <= 0); 1431 1432 g->p.nr_tasks = g->p.nr_proc*g->p.nr_threads; 1433 1434 g->p.bytes_global = g->p.mb_global *1024L*1024L; 1435 g->p.bytes_process = g->p.mb_proc *1024L*1024L; 1436 g->p.bytes_process_locked = g->p.mb_proc_locked *1024L*1024L; 1437 g->p.bytes_thread = g->p.mb_thread *1024L*1024L; 1438 1439 g->data = setup_shared_data(g->p.bytes_global); 1440 1441 /* Startup serialization: */ 1442 init_global_mutex(&g->start_work_mutex); 1443 init_global_mutex(&g->startup_mutex); 1444 init_global_mutex(&g->startup_done_mutex); 1445 init_global_mutex(&g->stop_work_mutex); 1446 1447 init_thread_data(); 1448 1449 tprintf("#\n"); 1450 if (parse_setup_cpu_list() || parse_setup_node_list()) 1451 return -1; 1452 tprintf("#\n"); 1453 1454 print_summary(); 1455 1456 return 0; 1457 } 1458 1459 static void deinit(void) 1460 { 1461 free_data(g->data, g->p.bytes_global); 1462 g->data = NULL; 1463 1464 deinit_thread_data(); 1465 1466 free_data(g, sizeof(*g)); 1467 g = NULL; 1468 } 1469 1470 /* 1471 * Print a short or long result, depending on the verbosity setting: 1472 */ 1473 static void print_res(const char *name, double val, 1474 const char *txt_unit, const char *txt_short, const char *txt_long) 1475 { 1476 if (!name) 1477 name = "main,"; 1478 1479 if (!g->p.show_quiet) 1480 printf(" %-30s %15.3f, %-15s %s\n", name, val, txt_unit, txt_short); 1481 else 1482 printf(" %14.3f %s\n", val, txt_long); 1483 } 1484 1485 static int __bench_numa(const char *name) 1486 { 1487 struct timeval start, stop, diff; 1488 u64 runtime_ns_min, runtime_ns_sum; 1489 pid_t *pids, pid, wpid; 1490 double delta_runtime; 1491 double runtime_avg; 1492 double runtime_sec_max; 1493 double runtime_sec_min; 1494 int wait_stat; 1495 double bytes; 1496 int i, t, p; 1497 1498 if (init()) 1499 return -1; 1500 1501 pids = zalloc(g->p.nr_proc * sizeof(*pids)); 1502 pid = -1; 1503 1504 /* All threads try to acquire it, this way we can wait for them to start up: */ 1505 pthread_mutex_lock(&g->start_work_mutex); 1506 1507 if (g->p.serialize_startup) { 1508 tprintf(" #\n"); 1509 tprintf(" # Startup synchronization: ..."); fflush(stdout); 1510 } 1511 1512 gettimeofday(&start, NULL); 1513 1514 for (i = 0; i < g->p.nr_proc; i++) { 1515 pid = fork(); 1516 dprintf(" # process %2d: PID %d\n", i, pid); 1517 1518 BUG_ON(pid < 0); 1519 if (!pid) { 1520 /* Child process: */ 1521 worker_process(i); 1522 1523 exit(0); 1524 } 1525 pids[i] = pid; 1526 1527 } 1528 /* Wait for all the threads to start up: */ 1529 while (g->nr_tasks_started != g->p.nr_tasks) 1530 usleep(USEC_PER_MSEC); 1531 1532 BUG_ON(g->nr_tasks_started != g->p.nr_tasks); 1533 1534 if (g->p.serialize_startup) { 1535 double startup_sec; 1536 1537 pthread_mutex_lock(&g->startup_done_mutex); 1538 1539 /* This will start all threads: */ 1540 pthread_mutex_unlock(&g->start_work_mutex); 1541 1542 /* This mutex is locked - the last started thread will wake us: */ 1543 pthread_mutex_lock(&g->startup_done_mutex); 1544 1545 gettimeofday(&stop, NULL); 1546 1547 timersub(&stop, &start, &diff); 1548 1549 startup_sec = diff.tv_sec * NSEC_PER_SEC; 1550 startup_sec += diff.tv_usec * NSEC_PER_USEC; 1551 startup_sec /= NSEC_PER_SEC; 1552 1553 tprintf(" threads initialized in %.6f seconds.\n", startup_sec); 1554 tprintf(" #\n"); 1555 1556 start = stop; 1557 pthread_mutex_unlock(&g->startup_done_mutex); 1558 } else { 1559 gettimeofday(&start, NULL); 1560 } 1561 1562 /* Parent process: */ 1563 1564 1565 for (i = 0; i < g->p.nr_proc; i++) { 1566 wpid = waitpid(pids[i], &wait_stat, 0); 1567 BUG_ON(wpid < 0); 1568 BUG_ON(!WIFEXITED(wait_stat)); 1569 1570 } 1571 1572 runtime_ns_sum = 0; 1573 runtime_ns_min = -1LL; 1574 1575 for (t = 0; t < g->p.nr_tasks; t++) { 1576 u64 thread_runtime_ns = g->threads[t].runtime_ns; 1577 1578 runtime_ns_sum += thread_runtime_ns; 1579 runtime_ns_min = min(thread_runtime_ns, runtime_ns_min); 1580 } 1581 1582 gettimeofday(&stop, NULL); 1583 timersub(&stop, &start, &diff); 1584 1585 BUG_ON(bench_format != BENCH_FORMAT_DEFAULT); 1586 1587 tprintf("\n ###\n"); 1588 tprintf("\n"); 1589 1590 runtime_sec_max = diff.tv_sec * NSEC_PER_SEC; 1591 runtime_sec_max += diff.tv_usec * NSEC_PER_USEC; 1592 runtime_sec_max /= NSEC_PER_SEC; 1593 1594 runtime_sec_min = runtime_ns_min / NSEC_PER_SEC; 1595 1596 bytes = g->bytes_done; 1597 runtime_avg = (double)runtime_ns_sum / g->p.nr_tasks / NSEC_PER_SEC; 1598 1599 if (g->p.measure_convergence) { 1600 print_res(name, runtime_sec_max, 1601 "secs,", "NUMA-convergence-latency", "secs latency to NUMA-converge"); 1602 } 1603 1604 print_res(name, runtime_sec_max, 1605 "secs,", "runtime-max/thread", "secs slowest (max) thread-runtime"); 1606 1607 print_res(name, runtime_sec_min, 1608 "secs,", "runtime-min/thread", "secs fastest (min) thread-runtime"); 1609 1610 print_res(name, runtime_avg, 1611 "secs,", "runtime-avg/thread", "secs average thread-runtime"); 1612 1613 delta_runtime = (runtime_sec_max - runtime_sec_min)/2.0; 1614 print_res(name, delta_runtime / runtime_sec_max * 100.0, 1615 "%,", "spread-runtime/thread", "% difference between max/avg runtime"); 1616 1617 print_res(name, bytes / g->p.nr_tasks / 1e9, 1618 "GB,", "data/thread", "GB data processed, per thread"); 1619 1620 print_res(name, bytes / 1e9, 1621 "GB,", "data-total", "GB data processed, total"); 1622 1623 print_res(name, runtime_sec_max * NSEC_PER_SEC / (bytes / g->p.nr_tasks), 1624 "nsecs,", "runtime/byte/thread","nsecs/byte/thread runtime"); 1625 1626 print_res(name, bytes / g->p.nr_tasks / 1e9 / runtime_sec_max, 1627 "GB/sec,", "thread-speed", "GB/sec/thread speed"); 1628 1629 print_res(name, bytes / runtime_sec_max / 1e9, 1630 "GB/sec,", "total-speed", "GB/sec total speed"); 1631 1632 if (g->p.show_details >= 2) { 1633 char tname[14 + 2 * 10 + 1]; 1634 struct thread_data *td; 1635 for (p = 0; p < g->p.nr_proc; p++) { 1636 for (t = 0; t < g->p.nr_threads; t++) { 1637 memset(tname, 0, sizeof(tname)); 1638 td = g->threads + p*g->p.nr_threads + t; 1639 snprintf(tname, sizeof(tname), "process%d:thread%d", p, t); 1640 print_res(tname, td->speed_gbs, 1641 "GB/sec", "thread-speed", "GB/sec/thread speed"); 1642 print_res(tname, td->system_time_ns / NSEC_PER_SEC, 1643 "secs", "thread-system-time", "system CPU time/thread"); 1644 print_res(tname, td->user_time_ns / NSEC_PER_SEC, 1645 "secs", "thread-user-time", "user CPU time/thread"); 1646 } 1647 } 1648 } 1649 1650 free(pids); 1651 1652 deinit(); 1653 1654 return 0; 1655 } 1656 1657 #define MAX_ARGS 50 1658 1659 static int command_size(const char **argv) 1660 { 1661 int size = 0; 1662 1663 while (*argv) { 1664 size++; 1665 argv++; 1666 } 1667 1668 BUG_ON(size >= MAX_ARGS); 1669 1670 return size; 1671 } 1672 1673 static void init_params(struct params *p, const char *name, int argc, const char **argv) 1674 { 1675 int i; 1676 1677 printf("\n # Running %s \"perf bench numa", name); 1678 1679 for (i = 0; i < argc; i++) 1680 printf(" %s", argv[i]); 1681 1682 printf("\"\n"); 1683 1684 memset(p, 0, sizeof(*p)); 1685 1686 /* Initialize nonzero defaults: */ 1687 1688 p->serialize_startup = 1; 1689 p->data_reads = true; 1690 p->data_writes = true; 1691 p->data_backwards = true; 1692 p->data_rand_walk = true; 1693 p->nr_loops = -1; 1694 p->init_random = true; 1695 p->mb_global_str = "1"; 1696 p->nr_proc = 1; 1697 p->nr_threads = 1; 1698 p->nr_secs = 5; 1699 p->run_all = argc == 1; 1700 } 1701 1702 static int run_bench_numa(const char *name, const char **argv) 1703 { 1704 int argc = command_size(argv); 1705 1706 init_params(&p0, name, argc, argv); 1707 argc = parse_options(argc, argv, options, bench_numa_usage, 0); 1708 if (argc) 1709 goto err; 1710 1711 if (__bench_numa(name)) 1712 goto err; 1713 1714 return 0; 1715 1716 err: 1717 return -1; 1718 } 1719 1720 #define OPT_BW_RAM "-s", "20", "-zZq", "--thp", " 1", "--no-data_rand_walk" 1721 #define OPT_BW_RAM_NOTHP OPT_BW_RAM, "--thp", "-1" 1722 1723 #define OPT_CONV "-s", "100", "-zZ0qcm", "--thp", " 1" 1724 #define OPT_CONV_NOTHP OPT_CONV, "--thp", "-1" 1725 1726 #define OPT_BW "-s", "20", "-zZ0q", "--thp", " 1" 1727 #define OPT_BW_NOTHP OPT_BW, "--thp", "-1" 1728 1729 /* 1730 * The built-in test-suite executed by "perf bench numa -a". 1731 * 1732 * (A minimum of 4 nodes and 16 GB of RAM is recommended.) 1733 */ 1734 static const char *tests[][MAX_ARGS] = { 1735 /* Basic single-stream NUMA bandwidth measurements: */ 1736 { "RAM-bw-local,", "mem", "-p", "1", "-t", "1", "-P", "1024", 1737 "-C" , "0", "-M", "0", OPT_BW_RAM }, 1738 { "RAM-bw-local-NOTHP,", 1739 "mem", "-p", "1", "-t", "1", "-P", "1024", 1740 "-C" , "0", "-M", "0", OPT_BW_RAM_NOTHP }, 1741 { "RAM-bw-remote,", "mem", "-p", "1", "-t", "1", "-P", "1024", 1742 "-C" , "0", "-M", "1", OPT_BW_RAM }, 1743 1744 /* 2-stream NUMA bandwidth measurements: */ 1745 { "RAM-bw-local-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024", 1746 "-C", "0,2", "-M", "0x2", OPT_BW_RAM }, 1747 { "RAM-bw-remote-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024", 1748 "-C", "0,2", "-M", "1x2", OPT_BW_RAM }, 1749 1750 /* Cross-stream NUMA bandwidth measurement: */ 1751 { "RAM-bw-cross,", "mem", "-p", "2", "-t", "1", "-P", "1024", 1752 "-C", "0,8", "-M", "1,0", OPT_BW_RAM }, 1753 1754 /* Convergence latency measurements: */ 1755 { " 1x3-convergence,", "mem", "-p", "1", "-t", "3", "-P", "512", OPT_CONV }, 1756 { " 1x4-convergence,", "mem", "-p", "1", "-t", "4", "-P", "512", OPT_CONV }, 1757 { " 1x6-convergence,", "mem", "-p", "1", "-t", "6", "-P", "1020", OPT_CONV }, 1758 { " 2x3-convergence,", "mem", "-p", "3", "-t", "3", "-P", "1020", OPT_CONV }, 1759 { " 3x3-convergence,", "mem", "-p", "3", "-t", "3", "-P", "1020", OPT_CONV }, 1760 { " 4x4-convergence,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV }, 1761 { " 4x4-convergence-NOTHP,", 1762 "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV_NOTHP }, 1763 { " 4x6-convergence,", "mem", "-p", "4", "-t", "6", "-P", "1020", OPT_CONV }, 1764 { " 4x8-convergence,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_CONV }, 1765 { " 8x4-convergence,", "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV }, 1766 { " 8x4-convergence-NOTHP,", 1767 "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV_NOTHP }, 1768 { " 3x1-convergence,", "mem", "-p", "3", "-t", "1", "-P", "512", OPT_CONV }, 1769 { " 4x1-convergence,", "mem", "-p", "4", "-t", "1", "-P", "512", OPT_CONV }, 1770 { " 8x1-convergence,", "mem", "-p", "8", "-t", "1", "-P", "512", OPT_CONV }, 1771 { "16x1-convergence,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_CONV }, 1772 { "32x1-convergence,", "mem", "-p", "32", "-t", "1", "-P", "128", OPT_CONV }, 1773 1774 /* Various NUMA process/thread layout bandwidth measurements: */ 1775 { " 2x1-bw-process,", "mem", "-p", "2", "-t", "1", "-P", "1024", OPT_BW }, 1776 { " 3x1-bw-process,", "mem", "-p", "3", "-t", "1", "-P", "1024", OPT_BW }, 1777 { " 4x1-bw-process,", "mem", "-p", "4", "-t", "1", "-P", "1024", OPT_BW }, 1778 { " 8x1-bw-process,", "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW }, 1779 { " 8x1-bw-process-NOTHP,", 1780 "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW_NOTHP }, 1781 { "16x1-bw-process,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_BW }, 1782 1783 { " 4x1-bw-thread,", "mem", "-p", "1", "-t", "4", "-T", "256", OPT_BW }, 1784 { " 8x1-bw-thread,", "mem", "-p", "1", "-t", "8", "-T", "256", OPT_BW }, 1785 { "16x1-bw-thread,", "mem", "-p", "1", "-t", "16", "-T", "128", OPT_BW }, 1786 { "32x1-bw-thread,", "mem", "-p", "1", "-t", "32", "-T", "64", OPT_BW }, 1787 1788 { " 2x3-bw-thread,", "mem", "-p", "2", "-t", "3", "-P", "512", OPT_BW }, 1789 { " 4x4-bw-thread,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_BW }, 1790 { " 4x6-bw-thread,", "mem", "-p", "4", "-t", "6", "-P", "512", OPT_BW }, 1791 { " 4x8-bw-thread,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW }, 1792 { " 4x8-bw-thread-NOTHP,", 1793 "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW_NOTHP }, 1794 { " 3x3-bw-thread,", "mem", "-p", "3", "-t", "3", "-P", "512", OPT_BW }, 1795 { " 5x5-bw-thread,", "mem", "-p", "5", "-t", "5", "-P", "512", OPT_BW }, 1796 1797 { "2x16-bw-thread,", "mem", "-p", "2", "-t", "16", "-P", "512", OPT_BW }, 1798 { "1x32-bw-thread,", "mem", "-p", "1", "-t", "32", "-P", "2048", OPT_BW }, 1799 1800 { "numa02-bw,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW }, 1801 { "numa02-bw-NOTHP,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW_NOTHP }, 1802 { "numa01-bw-thread,", "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW }, 1803 { "numa01-bw-thread-NOTHP,", 1804 "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW_NOTHP }, 1805 }; 1806 1807 static int bench_all(void) 1808 { 1809 int nr = ARRAY_SIZE(tests); 1810 int ret; 1811 int i; 1812 1813 ret = system("echo ' #'; echo ' # Running test on: '$(uname -a); echo ' #'"); 1814 BUG_ON(ret < 0); 1815 1816 for (i = 0; i < nr; i++) { 1817 run_bench_numa(tests[i][0], tests[i] + 1); 1818 } 1819 1820 printf("\n"); 1821 1822 return 0; 1823 } 1824 1825 int bench_numa(int argc, const char **argv) 1826 { 1827 init_params(&p0, "main,", argc, argv); 1828 argc = parse_options(argc, argv, options, bench_numa_usage, 0); 1829 if (argc) 1830 goto err; 1831 1832 if (p0.run_all) 1833 return bench_all(); 1834 1835 if (__bench_numa(NULL)) 1836 goto err; 1837 1838 return 0; 1839 1840 err: 1841 usage_with_options(numa_usage, options); 1842 return -1; 1843 } 1844