1 // SPDX-License-Identifier: GPL-2.0 2 #include <test_progs.h> 3 #include "kprobe_multi.skel.h" 4 #include "trace_helpers.h" 5 #include "kprobe_multi_empty.skel.h" 6 #include "bpf/libbpf_internal.h" 7 #include "bpf/hashmap.h" 8 9 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return) 10 { 11 LIBBPF_OPTS(bpf_test_run_opts, topts); 12 int err, prog_fd; 13 14 prog_fd = bpf_program__fd(skel->progs.trigger); 15 err = bpf_prog_test_run_opts(prog_fd, &topts); 16 ASSERT_OK(err, "test_run"); 17 ASSERT_EQ(topts.retval, 0, "test_run"); 18 19 ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result"); 20 ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result"); 21 ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result"); 22 ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result"); 23 ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result"); 24 ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result"); 25 ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result"); 26 ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result"); 27 28 if (test_return) { 29 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result"); 30 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result"); 31 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result"); 32 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result"); 33 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result"); 34 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result"); 35 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result"); 36 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result"); 37 } 38 } 39 40 static void test_skel_api(void) 41 { 42 struct kprobe_multi *skel = NULL; 43 int err; 44 45 skel = kprobe_multi__open_and_load(); 46 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load")) 47 goto cleanup; 48 49 skel->bss->pid = getpid(); 50 err = kprobe_multi__attach(skel); 51 if (!ASSERT_OK(err, "kprobe_multi__attach")) 52 goto cleanup; 53 54 kprobe_multi_test_run(skel, true); 55 56 cleanup: 57 kprobe_multi__destroy(skel); 58 } 59 60 static void test_link_api(struct bpf_link_create_opts *opts) 61 { 62 int prog_fd, link1_fd = -1, link2_fd = -1; 63 struct kprobe_multi *skel = NULL; 64 65 skel = kprobe_multi__open_and_load(); 66 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load")) 67 goto cleanup; 68 69 skel->bss->pid = getpid(); 70 prog_fd = bpf_program__fd(skel->progs.test_kprobe); 71 link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts); 72 if (!ASSERT_GE(link1_fd, 0, "link_fd")) 73 goto cleanup; 74 75 opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN; 76 prog_fd = bpf_program__fd(skel->progs.test_kretprobe); 77 link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts); 78 if (!ASSERT_GE(link2_fd, 0, "link_fd")) 79 goto cleanup; 80 81 kprobe_multi_test_run(skel, true); 82 83 cleanup: 84 if (link1_fd != -1) 85 close(link1_fd); 86 if (link2_fd != -1) 87 close(link2_fd); 88 kprobe_multi__destroy(skel); 89 } 90 91 #define GET_ADDR(__sym, __addr) ({ \ 92 __addr = ksym_get_addr(__sym); \ 93 if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \ 94 return; \ 95 }) 96 97 static void test_link_api_addrs(void) 98 { 99 LIBBPF_OPTS(bpf_link_create_opts, opts); 100 unsigned long long addrs[8]; 101 102 GET_ADDR("bpf_fentry_test1", addrs[0]); 103 GET_ADDR("bpf_fentry_test2", addrs[1]); 104 GET_ADDR("bpf_fentry_test3", addrs[2]); 105 GET_ADDR("bpf_fentry_test4", addrs[3]); 106 GET_ADDR("bpf_fentry_test5", addrs[4]); 107 GET_ADDR("bpf_fentry_test6", addrs[5]); 108 GET_ADDR("bpf_fentry_test7", addrs[6]); 109 GET_ADDR("bpf_fentry_test8", addrs[7]); 110 111 opts.kprobe_multi.addrs = (const unsigned long*) addrs; 112 opts.kprobe_multi.cnt = ARRAY_SIZE(addrs); 113 test_link_api(&opts); 114 } 115 116 static void test_link_api_syms(void) 117 { 118 LIBBPF_OPTS(bpf_link_create_opts, opts); 119 const char *syms[8] = { 120 "bpf_fentry_test1", 121 "bpf_fentry_test2", 122 "bpf_fentry_test3", 123 "bpf_fentry_test4", 124 "bpf_fentry_test5", 125 "bpf_fentry_test6", 126 "bpf_fentry_test7", 127 "bpf_fentry_test8", 128 }; 129 130 opts.kprobe_multi.syms = syms; 131 opts.kprobe_multi.cnt = ARRAY_SIZE(syms); 132 test_link_api(&opts); 133 } 134 135 static void 136 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts) 137 { 138 struct bpf_link *link1 = NULL, *link2 = NULL; 139 struct kprobe_multi *skel = NULL; 140 141 skel = kprobe_multi__open_and_load(); 142 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load")) 143 goto cleanup; 144 145 skel->bss->pid = getpid(); 146 link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 147 pattern, opts); 148 if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts")) 149 goto cleanup; 150 151 if (opts) { 152 opts->retprobe = true; 153 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual, 154 pattern, opts); 155 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts")) 156 goto cleanup; 157 } 158 159 kprobe_multi_test_run(skel, !!opts); 160 161 cleanup: 162 bpf_link__destroy(link2); 163 bpf_link__destroy(link1); 164 kprobe_multi__destroy(skel); 165 } 166 167 static void test_attach_api_pattern(void) 168 { 169 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 170 171 test_attach_api("bpf_fentry_test*", &opts); 172 test_attach_api("bpf_fentry_test?", NULL); 173 } 174 175 static void test_attach_api_addrs(void) 176 { 177 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 178 unsigned long long addrs[8]; 179 180 GET_ADDR("bpf_fentry_test1", addrs[0]); 181 GET_ADDR("bpf_fentry_test2", addrs[1]); 182 GET_ADDR("bpf_fentry_test3", addrs[2]); 183 GET_ADDR("bpf_fentry_test4", addrs[3]); 184 GET_ADDR("bpf_fentry_test5", addrs[4]); 185 GET_ADDR("bpf_fentry_test6", addrs[5]); 186 GET_ADDR("bpf_fentry_test7", addrs[6]); 187 GET_ADDR("bpf_fentry_test8", addrs[7]); 188 189 opts.addrs = (const unsigned long *) addrs; 190 opts.cnt = ARRAY_SIZE(addrs); 191 test_attach_api(NULL, &opts); 192 } 193 194 static void test_attach_api_syms(void) 195 { 196 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 197 const char *syms[8] = { 198 "bpf_fentry_test1", 199 "bpf_fentry_test2", 200 "bpf_fentry_test3", 201 "bpf_fentry_test4", 202 "bpf_fentry_test5", 203 "bpf_fentry_test6", 204 "bpf_fentry_test7", 205 "bpf_fentry_test8", 206 }; 207 208 opts.syms = syms; 209 opts.cnt = ARRAY_SIZE(syms); 210 test_attach_api(NULL, &opts); 211 } 212 213 static void test_attach_api_fails(void) 214 { 215 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 216 struct kprobe_multi *skel = NULL; 217 struct bpf_link *link = NULL; 218 unsigned long long addrs[2]; 219 const char *syms[2] = { 220 "bpf_fentry_test1", 221 "bpf_fentry_test2", 222 }; 223 __u64 cookies[2]; 224 225 addrs[0] = ksym_get_addr("bpf_fentry_test1"); 226 addrs[1] = ksym_get_addr("bpf_fentry_test2"); 227 228 if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr")) 229 goto cleanup; 230 231 skel = kprobe_multi__open_and_load(); 232 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load")) 233 goto cleanup; 234 235 skel->bss->pid = getpid(); 236 237 /* fail_1 - pattern and opts NULL */ 238 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 239 NULL, NULL); 240 if (!ASSERT_ERR_PTR(link, "fail_1")) 241 goto cleanup; 242 243 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error")) 244 goto cleanup; 245 246 /* fail_2 - both addrs and syms set */ 247 opts.addrs = (const unsigned long *) addrs; 248 opts.syms = syms; 249 opts.cnt = ARRAY_SIZE(syms); 250 opts.cookies = NULL; 251 252 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 253 NULL, &opts); 254 if (!ASSERT_ERR_PTR(link, "fail_2")) 255 goto cleanup; 256 257 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error")) 258 goto cleanup; 259 260 /* fail_3 - pattern and addrs set */ 261 opts.addrs = (const unsigned long *) addrs; 262 opts.syms = NULL; 263 opts.cnt = ARRAY_SIZE(syms); 264 opts.cookies = NULL; 265 266 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 267 "ksys_*", &opts); 268 if (!ASSERT_ERR_PTR(link, "fail_3")) 269 goto cleanup; 270 271 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error")) 272 goto cleanup; 273 274 /* fail_4 - pattern and cnt set */ 275 opts.addrs = NULL; 276 opts.syms = NULL; 277 opts.cnt = ARRAY_SIZE(syms); 278 opts.cookies = NULL; 279 280 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 281 "ksys_*", &opts); 282 if (!ASSERT_ERR_PTR(link, "fail_4")) 283 goto cleanup; 284 285 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error")) 286 goto cleanup; 287 288 /* fail_5 - pattern and cookies */ 289 opts.addrs = NULL; 290 opts.syms = NULL; 291 opts.cnt = 0; 292 opts.cookies = cookies; 293 294 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, 295 "ksys_*", &opts); 296 if (!ASSERT_ERR_PTR(link, "fail_5")) 297 goto cleanup; 298 299 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error")) 300 goto cleanup; 301 302 cleanup: 303 bpf_link__destroy(link); 304 kprobe_multi__destroy(skel); 305 } 306 307 static inline __u64 get_time_ns(void) 308 { 309 struct timespec t; 310 311 clock_gettime(CLOCK_MONOTONIC, &t); 312 return (__u64) t.tv_sec * 1000000000 + t.tv_nsec; 313 } 314 315 static size_t symbol_hash(long key, void *ctx __maybe_unused) 316 { 317 return str_hash((const char *) key); 318 } 319 320 static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused) 321 { 322 return strcmp((const char *) key1, (const char *) key2) == 0; 323 } 324 325 static int get_syms(char ***symsp, size_t *cntp) 326 { 327 size_t cap = 0, cnt = 0, i; 328 char *name = NULL, **syms = NULL; 329 struct hashmap *map; 330 char buf[256]; 331 FILE *f; 332 int err = 0; 333 334 /* 335 * The available_filter_functions contains many duplicates, 336 * but other than that all symbols are usable in kprobe multi 337 * interface. 338 * Filtering out duplicates by using hashmap__add, which won't 339 * add existing entry. 340 */ 341 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r"); 342 if (!f) 343 return -EINVAL; 344 345 map = hashmap__new(symbol_hash, symbol_equal, NULL); 346 if (IS_ERR(map)) { 347 err = libbpf_get_error(map); 348 goto error; 349 } 350 351 while (fgets(buf, sizeof(buf), f)) { 352 /* skip modules */ 353 if (strchr(buf, '[')) 354 continue; 355 356 free(name); 357 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1) 358 continue; 359 /* 360 * We attach to almost all kernel functions and some of them 361 * will cause 'suspicious RCU usage' when fprobe is attached 362 * to them. Filter out the current culprits - arch_cpu_idle 363 * default_idle and rcu_* functions. 364 */ 365 if (!strcmp(name, "arch_cpu_idle")) 366 continue; 367 if (!strcmp(name, "default_idle")) 368 continue; 369 if (!strncmp(name, "rcu_", 4)) 370 continue; 371 if (!strcmp(name, "bpf_dispatcher_xdp_func")) 372 continue; 373 if (!strncmp(name, "__ftrace_invalid_address__", 374 sizeof("__ftrace_invalid_address__") - 1)) 375 continue; 376 377 err = hashmap__add(map, name, 0); 378 if (err == -EEXIST) 379 continue; 380 if (err) 381 goto error; 382 383 err = libbpf_ensure_mem((void **) &syms, &cap, 384 sizeof(*syms), cnt + 1); 385 if (err) 386 goto error; 387 388 syms[cnt++] = name; 389 name = NULL; 390 } 391 392 *symsp = syms; 393 *cntp = cnt; 394 395 error: 396 free(name); 397 fclose(f); 398 hashmap__free(map); 399 if (err) { 400 for (i = 0; i < cnt; i++) 401 free(syms[i]); 402 free(syms); 403 } 404 return err; 405 } 406 407 void serial_test_kprobe_multi_bench_attach(void) 408 { 409 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 410 struct kprobe_multi_empty *skel = NULL; 411 long attach_start_ns, attach_end_ns; 412 long detach_start_ns, detach_end_ns; 413 double attach_delta, detach_delta; 414 struct bpf_link *link = NULL; 415 char **syms = NULL; 416 size_t cnt = 0, i; 417 418 if (!ASSERT_OK(get_syms(&syms, &cnt), "get_syms")) 419 return; 420 421 skel = kprobe_multi_empty__open_and_load(); 422 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load")) 423 goto cleanup; 424 425 opts.syms = (const char **) syms; 426 opts.cnt = cnt; 427 428 attach_start_ns = get_time_ns(); 429 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty, 430 NULL, &opts); 431 attach_end_ns = get_time_ns(); 432 433 if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts")) 434 goto cleanup; 435 436 detach_start_ns = get_time_ns(); 437 bpf_link__destroy(link); 438 detach_end_ns = get_time_ns(); 439 440 attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0; 441 detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0; 442 443 printf("%s: found %lu functions\n", __func__, cnt); 444 printf("%s: attached in %7.3lfs\n", __func__, attach_delta); 445 printf("%s: detached in %7.3lfs\n", __func__, detach_delta); 446 447 cleanup: 448 kprobe_multi_empty__destroy(skel); 449 if (syms) { 450 for (i = 0; i < cnt; i++) 451 free(syms[i]); 452 free(syms); 453 } 454 } 455 456 void test_kprobe_multi_test(void) 457 { 458 if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) 459 return; 460 461 if (test__start_subtest("skel_api")) 462 test_skel_api(); 463 if (test__start_subtest("link_api_addrs")) 464 test_link_api_syms(); 465 if (test__start_subtest("link_api_syms")) 466 test_link_api_addrs(); 467 if (test__start_subtest("attach_api_pattern")) 468 test_attach_api_pattern(); 469 if (test__start_subtest("attach_api_addrs")) 470 test_attach_api_addrs(); 471 if (test__start_subtest("attach_api_syms")) 472 test_attach_api_syms(); 473 if (test__start_subtest("attach_api_fails")) 474 test_attach_api_fails(); 475 } 476