1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2016 Facebook 3 */ 4 #define _GNU_SOURCE 5 #include <sched.h> 6 #include <stdio.h> 7 #include <sys/types.h> 8 #include <asm/unistd.h> 9 #include <unistd.h> 10 #include <assert.h> 11 #include <sys/wait.h> 12 #include <stdlib.h> 13 #include <signal.h> 14 #include <string.h> 15 #include <time.h> 16 #include <sys/resource.h> 17 #include <arpa/inet.h> 18 #include <errno.h> 19 20 #include <bpf/bpf.h> 21 #include <bpf/libbpf.h> 22 23 #define TEST_BIT(t) (1U << (t)) 24 #define MAX_NR_CPUS 1024 25 26 static __u64 time_get_ns(void) 27 { 28 struct timespec ts; 29 30 clock_gettime(CLOCK_MONOTONIC, &ts); 31 return ts.tv_sec * 1000000000ull + ts.tv_nsec; 32 } 33 34 enum test_type { 35 HASH_PREALLOC, 36 PERCPU_HASH_PREALLOC, 37 HASH_KMALLOC, 38 PERCPU_HASH_KMALLOC, 39 LRU_HASH_PREALLOC, 40 NOCOMMON_LRU_HASH_PREALLOC, 41 LPM_KMALLOC, 42 HASH_LOOKUP, 43 ARRAY_LOOKUP, 44 INNER_LRU_HASH_PREALLOC, 45 LRU_HASH_LOOKUP, 46 NR_TESTS, 47 }; 48 49 const char *test_map_names[NR_TESTS] = { 50 [HASH_PREALLOC] = "hash_map", 51 [PERCPU_HASH_PREALLOC] = "percpu_hash_map", 52 [HASH_KMALLOC] = "hash_map_alloc", 53 [PERCPU_HASH_KMALLOC] = "percpu_hash_map_alloc", 54 [LRU_HASH_PREALLOC] = "lru_hash_map", 55 [NOCOMMON_LRU_HASH_PREALLOC] = "nocommon_lru_hash_map", 56 [LPM_KMALLOC] = "lpm_trie_map_alloc", 57 [HASH_LOOKUP] = "hash_map", 58 [ARRAY_LOOKUP] = "array_map", 59 [INNER_LRU_HASH_PREALLOC] = "inner_lru_hash_map", 60 [LRU_HASH_LOOKUP] = "lru_hash_lookup_map", 61 }; 62 63 enum map_idx { 64 array_of_lru_hashs_idx, 65 hash_map_alloc_idx, 66 lru_hash_lookup_idx, 67 NR_IDXES, 68 }; 69 70 static int map_fd[NR_IDXES]; 71 72 static int test_flags = ~0; 73 static uint32_t num_map_entries; 74 static uint32_t inner_lru_hash_size; 75 static int lru_hash_lookup_test_entries = 32; 76 static uint32_t max_cnt = 1000000; 77 78 static int check_test_flags(enum test_type t) 79 { 80 return test_flags & TEST_BIT(t); 81 } 82 83 static void test_hash_prealloc(int cpu) 84 { 85 __u64 start_time; 86 int i; 87 88 start_time = time_get_ns(); 89 for (i = 0; i < max_cnt; i++) 90 syscall(__NR_getuid); 91 printf("%d:hash_map_perf pre-alloc %lld events per sec\n", 92 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 93 } 94 95 static int pre_test_lru_hash_lookup(int tasks) 96 { 97 int fd = map_fd[lru_hash_lookup_idx]; 98 uint32_t key; 99 long val = 1; 100 int ret; 101 102 if (num_map_entries > lru_hash_lookup_test_entries) 103 lru_hash_lookup_test_entries = num_map_entries; 104 105 /* Populate the lru_hash_map for LRU_HASH_LOOKUP perf test. 106 * 107 * It is fine that the user requests for a map with 108 * num_map_entries < 32 and some of the later lru hash lookup 109 * may return not found. For LRU map, we are not interested 110 * in such small map performance. 111 */ 112 for (key = 0; key < lru_hash_lookup_test_entries; key++) { 113 ret = bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST); 114 if (ret) 115 return ret; 116 } 117 118 return 0; 119 } 120 121 static void do_test_lru(enum test_type test, int cpu) 122 { 123 static int inner_lru_map_fds[MAX_NR_CPUS]; 124 125 struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 }; 126 const char *test_name; 127 __u64 start_time; 128 int i, ret; 129 130 if (test == INNER_LRU_HASH_PREALLOC && cpu) { 131 /* If CPU is not 0, create inner_lru hash map and insert the fd 132 * value into the array_of_lru_hash map. In case of CPU 0, 133 * 'inner_lru_hash_map' was statically inserted on the map init 134 */ 135 int outer_fd = map_fd[array_of_lru_hashs_idx]; 136 unsigned int mycpu, mynode; 137 LIBBPF_OPTS(bpf_map_create_opts, opts, 138 .map_flags = BPF_F_NUMA_NODE, 139 ); 140 141 assert(cpu < MAX_NR_CPUS); 142 143 ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL); 144 assert(!ret); 145 146 opts.numa_node = mynode; 147 inner_lru_map_fds[cpu] = 148 bpf_map_create(BPF_MAP_TYPE_LRU_HASH, 149 test_map_names[INNER_LRU_HASH_PREALLOC], 150 sizeof(uint32_t), 151 sizeof(long), 152 inner_lru_hash_size, &opts); 153 if (inner_lru_map_fds[cpu] == -1) { 154 printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n", 155 strerror(errno), errno); 156 exit(1); 157 } 158 159 ret = bpf_map_update_elem(outer_fd, &cpu, 160 &inner_lru_map_fds[cpu], 161 BPF_ANY); 162 if (ret) { 163 printf("cannot update ARRAY_OF_LRU_HASHS with key:%u. %s(%d)\n", 164 cpu, strerror(errno), errno); 165 exit(1); 166 } 167 } 168 169 in6.sin6_addr.s6_addr16[0] = 0xdead; 170 in6.sin6_addr.s6_addr16[1] = 0xbeef; 171 172 if (test == LRU_HASH_PREALLOC) { 173 test_name = "lru_hash_map_perf"; 174 in6.sin6_addr.s6_addr16[2] = 0; 175 } else if (test == NOCOMMON_LRU_HASH_PREALLOC) { 176 test_name = "nocommon_lru_hash_map_perf"; 177 in6.sin6_addr.s6_addr16[2] = 1; 178 } else if (test == INNER_LRU_HASH_PREALLOC) { 179 test_name = "inner_lru_hash_map_perf"; 180 in6.sin6_addr.s6_addr16[2] = 2; 181 } else if (test == LRU_HASH_LOOKUP) { 182 test_name = "lru_hash_lookup_perf"; 183 in6.sin6_addr.s6_addr16[2] = 3; 184 in6.sin6_addr.s6_addr32[3] = 0; 185 } else { 186 assert(0); 187 } 188 189 start_time = time_get_ns(); 190 for (i = 0; i < max_cnt; i++) { 191 ret = connect(-1, (const struct sockaddr *)&in6, sizeof(in6)); 192 assert(ret == -1 && errno == EBADF); 193 if (in6.sin6_addr.s6_addr32[3] < 194 lru_hash_lookup_test_entries - 32) 195 in6.sin6_addr.s6_addr32[3] += 32; 196 else 197 in6.sin6_addr.s6_addr32[3] = 0; 198 } 199 printf("%d:%s pre-alloc %lld events per sec\n", 200 cpu, test_name, 201 max_cnt * 1000000000ll / (time_get_ns() - start_time)); 202 } 203 204 static void test_lru_hash_prealloc(int cpu) 205 { 206 do_test_lru(LRU_HASH_PREALLOC, cpu); 207 } 208 209 static void test_nocommon_lru_hash_prealloc(int cpu) 210 { 211 do_test_lru(NOCOMMON_LRU_HASH_PREALLOC, cpu); 212 } 213 214 static void test_inner_lru_hash_prealloc(int cpu) 215 { 216 do_test_lru(INNER_LRU_HASH_PREALLOC, cpu); 217 } 218 219 static void test_lru_hash_lookup(int cpu) 220 { 221 do_test_lru(LRU_HASH_LOOKUP, cpu); 222 } 223 224 static void test_percpu_hash_prealloc(int cpu) 225 { 226 __u64 start_time; 227 int i; 228 229 start_time = time_get_ns(); 230 for (i = 0; i < max_cnt; i++) 231 syscall(__NR_geteuid); 232 printf("%d:percpu_hash_map_perf pre-alloc %lld events per sec\n", 233 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 234 } 235 236 static void test_hash_kmalloc(int cpu) 237 { 238 __u64 start_time; 239 int i; 240 241 start_time = time_get_ns(); 242 for (i = 0; i < max_cnt; i++) 243 syscall(__NR_getgid); 244 printf("%d:hash_map_perf kmalloc %lld events per sec\n", 245 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 246 } 247 248 static void test_percpu_hash_kmalloc(int cpu) 249 { 250 __u64 start_time; 251 int i; 252 253 start_time = time_get_ns(); 254 for (i = 0; i < max_cnt; i++) 255 syscall(__NR_getegid); 256 printf("%d:percpu_hash_map_perf kmalloc %lld events per sec\n", 257 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 258 } 259 260 static void test_lpm_kmalloc(int cpu) 261 { 262 __u64 start_time; 263 int i; 264 265 start_time = time_get_ns(); 266 for (i = 0; i < max_cnt; i++) 267 syscall(__NR_gettid); 268 printf("%d:lpm_perf kmalloc %lld events per sec\n", 269 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 270 } 271 272 static void test_hash_lookup(int cpu) 273 { 274 __u64 start_time; 275 int i; 276 277 start_time = time_get_ns(); 278 for (i = 0; i < max_cnt; i++) 279 syscall(__NR_getpgid, 0); 280 printf("%d:hash_lookup %lld lookups per sec\n", 281 cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time)); 282 } 283 284 static void test_array_lookup(int cpu) 285 { 286 __u64 start_time; 287 int i; 288 289 start_time = time_get_ns(); 290 for (i = 0; i < max_cnt; i++) 291 syscall(__NR_getppid, 0); 292 printf("%d:array_lookup %lld lookups per sec\n", 293 cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time)); 294 } 295 296 typedef int (*pre_test_func)(int tasks); 297 const pre_test_func pre_test_funcs[] = { 298 [LRU_HASH_LOOKUP] = pre_test_lru_hash_lookup, 299 }; 300 301 typedef void (*test_func)(int cpu); 302 const test_func test_funcs[] = { 303 [HASH_PREALLOC] = test_hash_prealloc, 304 [PERCPU_HASH_PREALLOC] = test_percpu_hash_prealloc, 305 [HASH_KMALLOC] = test_hash_kmalloc, 306 [PERCPU_HASH_KMALLOC] = test_percpu_hash_kmalloc, 307 [LRU_HASH_PREALLOC] = test_lru_hash_prealloc, 308 [NOCOMMON_LRU_HASH_PREALLOC] = test_nocommon_lru_hash_prealloc, 309 [LPM_KMALLOC] = test_lpm_kmalloc, 310 [HASH_LOOKUP] = test_hash_lookup, 311 [ARRAY_LOOKUP] = test_array_lookup, 312 [INNER_LRU_HASH_PREALLOC] = test_inner_lru_hash_prealloc, 313 [LRU_HASH_LOOKUP] = test_lru_hash_lookup, 314 }; 315 316 static int pre_test(int tasks) 317 { 318 int i; 319 320 for (i = 0; i < NR_TESTS; i++) { 321 if (pre_test_funcs[i] && check_test_flags(i)) { 322 int ret = pre_test_funcs[i](tasks); 323 324 if (ret) 325 return ret; 326 } 327 } 328 329 return 0; 330 } 331 332 static void loop(int cpu) 333 { 334 cpu_set_t cpuset; 335 int i; 336 337 CPU_ZERO(&cpuset); 338 CPU_SET(cpu, &cpuset); 339 sched_setaffinity(0, sizeof(cpuset), &cpuset); 340 341 for (i = 0; i < NR_TESTS; i++) { 342 if (check_test_flags(i)) 343 test_funcs[i](cpu); 344 } 345 } 346 347 static void run_perf_test(int tasks) 348 { 349 pid_t pid[tasks]; 350 int i; 351 352 assert(!pre_test(tasks)); 353 354 for (i = 0; i < tasks; i++) { 355 pid[i] = fork(); 356 if (pid[i] == 0) { 357 loop(i); 358 exit(0); 359 } else if (pid[i] == -1) { 360 printf("couldn't spawn #%d process\n", i); 361 exit(1); 362 } 363 } 364 for (i = 0; i < tasks; i++) { 365 int status; 366 367 assert(waitpid(pid[i], &status, 0) == pid[i]); 368 assert(status == 0); 369 } 370 } 371 372 static void fill_lpm_trie(void) 373 { 374 struct bpf_lpm_trie_key *key; 375 unsigned long value = 0; 376 unsigned int i; 377 int r; 378 379 key = alloca(sizeof(*key) + 4); 380 key->prefixlen = 32; 381 382 for (i = 0; i < 512; ++i) { 383 key->prefixlen = rand() % 33; 384 key->data[0] = rand() & 0xff; 385 key->data[1] = rand() & 0xff; 386 key->data[2] = rand() & 0xff; 387 key->data[3] = rand() & 0xff; 388 r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], 389 key, &value, 0); 390 assert(!r); 391 } 392 393 key->prefixlen = 32; 394 key->data[0] = 192; 395 key->data[1] = 168; 396 key->data[2] = 0; 397 key->data[3] = 1; 398 value = 128; 399 400 r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0); 401 assert(!r); 402 } 403 404 static void fixup_map(struct bpf_object *obj) 405 { 406 struct bpf_map *map; 407 int i; 408 409 bpf_object__for_each_map(map, obj) { 410 const char *name = bpf_map__name(map); 411 412 /* Only change the max_entries for the enabled test(s) */ 413 for (i = 0; i < NR_TESTS; i++) { 414 if (!strcmp(test_map_names[i], name) && 415 (check_test_flags(i))) { 416 bpf_map__set_max_entries(map, num_map_entries); 417 continue; 418 } 419 } 420 } 421 422 inner_lru_hash_size = num_map_entries; 423 } 424 425 int main(int argc, char **argv) 426 { 427 int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); 428 struct bpf_link *links[8]; 429 struct bpf_program *prog; 430 struct bpf_object *obj; 431 struct bpf_map *map; 432 char filename[256]; 433 int i = 0; 434 435 if (argc > 1) 436 test_flags = atoi(argv[1]) ? : test_flags; 437 438 if (argc > 2) 439 nr_cpus = atoi(argv[2]) ? : nr_cpus; 440 441 if (argc > 3) 442 num_map_entries = atoi(argv[3]); 443 444 if (argc > 4) 445 max_cnt = atoi(argv[4]); 446 447 snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); 448 obj = bpf_object__open_file(filename, NULL); 449 if (libbpf_get_error(obj)) { 450 fprintf(stderr, "ERROR: opening BPF object file failed\n"); 451 return 0; 452 } 453 454 map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map"); 455 if (libbpf_get_error(map)) { 456 fprintf(stderr, "ERROR: finding a map in obj file failed\n"); 457 goto cleanup; 458 } 459 460 inner_lru_hash_size = bpf_map__max_entries(map); 461 if (!inner_lru_hash_size) { 462 fprintf(stderr, "ERROR: failed to get map attribute\n"); 463 goto cleanup; 464 } 465 466 /* resize BPF map prior to loading */ 467 if (num_map_entries > 0) 468 fixup_map(obj); 469 470 /* load BPF program */ 471 if (bpf_object__load(obj)) { 472 fprintf(stderr, "ERROR: loading BPF object file failed\n"); 473 goto cleanup; 474 } 475 476 map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs"); 477 map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc"); 478 map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map"); 479 if (map_fd[0] < 0 || map_fd[1] < 0 || map_fd[2] < 0) { 480 fprintf(stderr, "ERROR: finding a map in obj file failed\n"); 481 goto cleanup; 482 } 483 484 bpf_object__for_each_program(prog, obj) { 485 links[i] = bpf_program__attach(prog); 486 if (libbpf_get_error(links[i])) { 487 fprintf(stderr, "ERROR: bpf_program__attach failed\n"); 488 links[i] = NULL; 489 goto cleanup; 490 } 491 i++; 492 } 493 494 fill_lpm_trie(); 495 496 run_perf_test(nr_cpus); 497 498 cleanup: 499 for (i--; i >= 0; i--) 500 bpf_link__destroy(links[i]); 501 502 bpf_object__close(obj); 503 return 0; 504 } 505