1 // SPDX-License-Identifier: GPL-2.0 2 3 /** 4 * Test XDP bonding support 5 * 6 * Sets up two bonded veth pairs between two fresh namespaces 7 * and verifies that XDP_TX program loaded on a bond device 8 * are correctly loaded onto the slave devices and XDP_TX'd 9 * packets are balanced using bonding. 10 */ 11 12 #define _GNU_SOURCE 13 #include <sched.h> 14 #include <net/if.h> 15 #include <linux/if_link.h> 16 #include "test_progs.h" 17 #include "network_helpers.h" 18 #include <linux/if_bonding.h> 19 #include <linux/limits.h> 20 #include <linux/udp.h> 21 22 #include "xdp_dummy.skel.h" 23 #include "xdp_redirect_multi_kern.skel.h" 24 #include "xdp_tx.skel.h" 25 26 #define BOND1_MAC {0x00, 0x11, 0x22, 0x33, 0x44, 0x55} 27 #define BOND1_MAC_STR "00:11:22:33:44:55" 28 #define BOND2_MAC {0x00, 0x22, 0x33, 0x44, 0x55, 0x66} 29 #define BOND2_MAC_STR "00:22:33:44:55:66" 30 #define NPACKETS 100 31 32 static int root_netns_fd = -1; 33 34 static void restore_root_netns(void) 35 { 36 ASSERT_OK(setns(root_netns_fd, CLONE_NEWNET), "restore_root_netns"); 37 } 38 39 static int setns_by_name(char *name) 40 { 41 int nsfd, err; 42 char nspath[PATH_MAX]; 43 44 snprintf(nspath, sizeof(nspath), "%s/%s", "/var/run/netns", name); 45 nsfd = open(nspath, O_RDONLY | O_CLOEXEC); 46 if (nsfd < 0) 47 return -1; 48 49 err = setns(nsfd, CLONE_NEWNET); 50 close(nsfd); 51 return err; 52 } 53 54 static int get_rx_packets(const char *iface) 55 { 56 FILE *f; 57 char line[512]; 58 int iface_len = strlen(iface); 59 60 f = fopen("/proc/net/dev", "r"); 61 if (!f) 62 return -1; 63 64 while (fgets(line, sizeof(line), f)) { 65 char *p = line; 66 67 while (*p == ' ') 68 p++; /* skip whitespace */ 69 if (!strncmp(p, iface, iface_len)) { 70 p += iface_len; 71 if (*p++ != ':') 72 continue; 73 while (*p == ' ') 74 p++; /* skip whitespace */ 75 while (*p && *p != ' ') 76 p++; /* skip rx bytes */ 77 while (*p == ' ') 78 p++; /* skip whitespace */ 79 fclose(f); 80 return atoi(p); 81 } 82 } 83 fclose(f); 84 return -1; 85 } 86 87 #define MAX_BPF_LINKS 8 88 89 struct skeletons { 90 struct xdp_dummy *xdp_dummy; 91 struct xdp_tx *xdp_tx; 92 struct xdp_redirect_multi_kern *xdp_redirect_multi_kern; 93 94 int nlinks; 95 struct bpf_link *links[MAX_BPF_LINKS]; 96 }; 97 98 static int xdp_attach(struct skeletons *skeletons, struct bpf_program *prog, char *iface) 99 { 100 struct bpf_link *link; 101 int ifindex; 102 103 ifindex = if_nametoindex(iface); 104 if (!ASSERT_GT(ifindex, 0, "get ifindex")) 105 return -1; 106 107 if (!ASSERT_LE(skeletons->nlinks+1, MAX_BPF_LINKS, "too many XDP programs attached")) 108 return -1; 109 110 link = bpf_program__attach_xdp(prog, ifindex); 111 if (!ASSERT_OK_PTR(link, "attach xdp program")) 112 return -1; 113 114 skeletons->links[skeletons->nlinks++] = link; 115 return 0; 116 } 117 118 enum { 119 BOND_ONE_NO_ATTACH = 0, 120 BOND_BOTH_AND_ATTACH, 121 }; 122 123 static const char * const mode_names[] = { 124 [BOND_MODE_ROUNDROBIN] = "balance-rr", 125 [BOND_MODE_ACTIVEBACKUP] = "active-backup", 126 [BOND_MODE_XOR] = "balance-xor", 127 [BOND_MODE_BROADCAST] = "broadcast", 128 [BOND_MODE_8023AD] = "802.3ad", 129 [BOND_MODE_TLB] = "balance-tlb", 130 [BOND_MODE_ALB] = "balance-alb", 131 }; 132 133 static const char * const xmit_policy_names[] = { 134 [BOND_XMIT_POLICY_LAYER2] = "layer2", 135 [BOND_XMIT_POLICY_LAYER34] = "layer3+4", 136 [BOND_XMIT_POLICY_LAYER23] = "layer2+3", 137 [BOND_XMIT_POLICY_ENCAP23] = "encap2+3", 138 [BOND_XMIT_POLICY_ENCAP34] = "encap3+4", 139 }; 140 141 static int bonding_setup(struct skeletons *skeletons, int mode, int xmit_policy, 142 int bond_both_attach) 143 { 144 #define SYS(fmt, ...) \ 145 ({ \ 146 char cmd[1024]; \ 147 snprintf(cmd, sizeof(cmd), fmt, ##__VA_ARGS__); \ 148 if (!ASSERT_OK(system(cmd), cmd)) \ 149 return -1; \ 150 }) 151 152 SYS("ip netns add ns_dst"); 153 SYS("ip link add veth1_1 type veth peer name veth2_1 netns ns_dst"); 154 SYS("ip link add veth1_2 type veth peer name veth2_2 netns ns_dst"); 155 156 SYS("ip link add bond1 type bond mode %s xmit_hash_policy %s", 157 mode_names[mode], xmit_policy_names[xmit_policy]); 158 SYS("ip link set bond1 up address " BOND1_MAC_STR " addrgenmode none"); 159 SYS("ip -netns ns_dst link add bond2 type bond mode %s xmit_hash_policy %s", 160 mode_names[mode], xmit_policy_names[xmit_policy]); 161 SYS("ip -netns ns_dst link set bond2 up address " BOND2_MAC_STR " addrgenmode none"); 162 163 SYS("ip link set veth1_1 master bond1"); 164 if (bond_both_attach == BOND_BOTH_AND_ATTACH) { 165 SYS("ip link set veth1_2 master bond1"); 166 } else { 167 SYS("ip link set veth1_2 up addrgenmode none"); 168 169 if (xdp_attach(skeletons, skeletons->xdp_dummy->progs.xdp_dummy_prog, "veth1_2")) 170 return -1; 171 } 172 173 SYS("ip -netns ns_dst link set veth2_1 master bond2"); 174 175 if (bond_both_attach == BOND_BOTH_AND_ATTACH) 176 SYS("ip -netns ns_dst link set veth2_2 master bond2"); 177 else 178 SYS("ip -netns ns_dst link set veth2_2 up addrgenmode none"); 179 180 /* Load a dummy program on sending side as with veth peer needs to have a 181 * XDP program loaded as well. 182 */ 183 if (xdp_attach(skeletons, skeletons->xdp_dummy->progs.xdp_dummy_prog, "bond1")) 184 return -1; 185 186 if (bond_both_attach == BOND_BOTH_AND_ATTACH) { 187 if (!ASSERT_OK(setns_by_name("ns_dst"), "set netns to ns_dst")) 188 return -1; 189 190 if (xdp_attach(skeletons, skeletons->xdp_tx->progs.xdp_tx, "bond2")) 191 return -1; 192 193 restore_root_netns(); 194 } 195 196 return 0; 197 198 #undef SYS 199 } 200 201 static void bonding_cleanup(struct skeletons *skeletons) 202 { 203 restore_root_netns(); 204 while (skeletons->nlinks) { 205 skeletons->nlinks--; 206 bpf_link__destroy(skeletons->links[skeletons->nlinks]); 207 } 208 ASSERT_OK(system("ip link delete bond1"), "delete bond1"); 209 ASSERT_OK(system("ip link delete veth1_1"), "delete veth1_1"); 210 ASSERT_OK(system("ip link delete veth1_2"), "delete veth1_2"); 211 ASSERT_OK(system("ip netns delete ns_dst"), "delete ns_dst"); 212 } 213 214 static int send_udp_packets(int vary_dst_ip) 215 { 216 struct ethhdr eh = { 217 .h_source = BOND1_MAC, 218 .h_dest = BOND2_MAC, 219 .h_proto = htons(ETH_P_IP), 220 }; 221 struct iphdr iph = {}; 222 struct udphdr uh = {}; 223 uint8_t buf[128]; 224 int i, s = -1; 225 int ifindex; 226 227 s = socket(AF_PACKET, SOCK_RAW, IPPROTO_RAW); 228 if (!ASSERT_GE(s, 0, "socket")) 229 goto err; 230 231 ifindex = if_nametoindex("bond1"); 232 if (!ASSERT_GT(ifindex, 0, "get bond1 ifindex")) 233 goto err; 234 235 iph.ihl = 5; 236 iph.version = 4; 237 iph.tos = 16; 238 iph.id = 1; 239 iph.ttl = 64; 240 iph.protocol = IPPROTO_UDP; 241 iph.saddr = 1; 242 iph.daddr = 2; 243 iph.tot_len = htons(sizeof(buf) - ETH_HLEN); 244 iph.check = 0; 245 246 for (i = 1; i <= NPACKETS; i++) { 247 int n; 248 struct sockaddr_ll saddr_ll = { 249 .sll_ifindex = ifindex, 250 .sll_halen = ETH_ALEN, 251 .sll_addr = BOND2_MAC, 252 }; 253 254 /* vary the UDP destination port for even distribution with roundrobin/xor modes */ 255 uh.dest++; 256 257 if (vary_dst_ip) 258 iph.daddr++; 259 260 /* construct a packet */ 261 memcpy(buf, &eh, sizeof(eh)); 262 memcpy(buf + sizeof(eh), &iph, sizeof(iph)); 263 memcpy(buf + sizeof(eh) + sizeof(iph), &uh, sizeof(uh)); 264 265 n = sendto(s, buf, sizeof(buf), 0, (struct sockaddr *)&saddr_ll, sizeof(saddr_ll)); 266 if (!ASSERT_EQ(n, sizeof(buf), "sendto")) 267 goto err; 268 } 269 270 return 0; 271 272 err: 273 if (s >= 0) 274 close(s); 275 return -1; 276 } 277 278 static void test_xdp_bonding_with_mode(struct skeletons *skeletons, int mode, int xmit_policy) 279 { 280 int bond1_rx; 281 282 if (bonding_setup(skeletons, mode, xmit_policy, BOND_BOTH_AND_ATTACH)) 283 goto out; 284 285 if (send_udp_packets(xmit_policy != BOND_XMIT_POLICY_LAYER34)) 286 goto out; 287 288 bond1_rx = get_rx_packets("bond1"); 289 ASSERT_EQ(bond1_rx, NPACKETS, "expected more received packets"); 290 291 switch (mode) { 292 case BOND_MODE_ROUNDROBIN: 293 case BOND_MODE_XOR: { 294 int veth1_rx = get_rx_packets("veth1_1"); 295 int veth2_rx = get_rx_packets("veth1_2"); 296 int diff = abs(veth1_rx - veth2_rx); 297 298 ASSERT_GE(veth1_rx + veth2_rx, NPACKETS, "expected more packets"); 299 300 switch (xmit_policy) { 301 case BOND_XMIT_POLICY_LAYER2: 302 ASSERT_GE(diff, NPACKETS, 303 "expected packets on only one of the interfaces"); 304 break; 305 case BOND_XMIT_POLICY_LAYER23: 306 case BOND_XMIT_POLICY_LAYER34: 307 ASSERT_LT(diff, NPACKETS/2, 308 "expected even distribution of packets"); 309 break; 310 default: 311 PRINT_FAIL("Unimplemented xmit_policy=%d\n", xmit_policy); 312 break; 313 } 314 break; 315 } 316 case BOND_MODE_ACTIVEBACKUP: { 317 int veth1_rx = get_rx_packets("veth1_1"); 318 int veth2_rx = get_rx_packets("veth1_2"); 319 int diff = abs(veth1_rx - veth2_rx); 320 321 ASSERT_GE(diff, NPACKETS, 322 "expected packets on only one of the interfaces"); 323 break; 324 } 325 default: 326 PRINT_FAIL("Unimplemented xmit_policy=%d\n", xmit_policy); 327 break; 328 } 329 330 out: 331 bonding_cleanup(skeletons); 332 } 333 334 /* Test the broadcast redirection using xdp_redirect_map_multi_prog and adding 335 * all the interfaces to it and checking that broadcasting won't send the packet 336 * to neither the ingress bond device (bond2) or its slave (veth2_1). 337 */ 338 static void test_xdp_bonding_redirect_multi(struct skeletons *skeletons) 339 { 340 static const char * const ifaces[] = {"bond2", "veth2_1", "veth2_2"}; 341 int veth1_1_rx, veth1_2_rx; 342 int err; 343 344 if (bonding_setup(skeletons, BOND_MODE_ROUNDROBIN, BOND_XMIT_POLICY_LAYER23, 345 BOND_ONE_NO_ATTACH)) 346 goto out; 347 348 349 if (!ASSERT_OK(setns_by_name("ns_dst"), "could not set netns to ns_dst")) 350 goto out; 351 352 /* populate the devmap with the relevant interfaces */ 353 for (int i = 0; i < ARRAY_SIZE(ifaces); i++) { 354 int ifindex = if_nametoindex(ifaces[i]); 355 int map_fd = bpf_map__fd(skeletons->xdp_redirect_multi_kern->maps.map_all); 356 357 if (!ASSERT_GT(ifindex, 0, "could not get interface index")) 358 goto out; 359 360 err = bpf_map_update_elem(map_fd, &ifindex, &ifindex, 0); 361 if (!ASSERT_OK(err, "add interface to map_all")) 362 goto out; 363 } 364 365 if (xdp_attach(skeletons, 366 skeletons->xdp_redirect_multi_kern->progs.xdp_redirect_map_multi_prog, 367 "bond2")) 368 goto out; 369 370 restore_root_netns(); 371 372 if (send_udp_packets(BOND_MODE_ROUNDROBIN)) 373 goto out; 374 375 veth1_1_rx = get_rx_packets("veth1_1"); 376 veth1_2_rx = get_rx_packets("veth1_2"); 377 378 ASSERT_EQ(veth1_1_rx, 0, "expected no packets on veth1_1"); 379 ASSERT_GE(veth1_2_rx, NPACKETS, "expected packets on veth1_2"); 380 381 out: 382 restore_root_netns(); 383 bonding_cleanup(skeletons); 384 } 385 386 /* Test that XDP programs cannot be attached to both the bond master and slaves simultaneously */ 387 static void test_xdp_bonding_attach(struct skeletons *skeletons) 388 { 389 struct bpf_link *link = NULL; 390 struct bpf_link *link2 = NULL; 391 int veth, bond, err; 392 393 if (!ASSERT_OK(system("ip link add veth type veth"), "add veth")) 394 goto out; 395 if (!ASSERT_OK(system("ip link add bond type bond"), "add bond")) 396 goto out; 397 398 veth = if_nametoindex("veth"); 399 if (!ASSERT_GE(veth, 0, "if_nametoindex veth")) 400 goto out; 401 bond = if_nametoindex("bond"); 402 if (!ASSERT_GE(bond, 0, "if_nametoindex bond")) 403 goto out; 404 405 /* enslaving with a XDP program loaded is allowed */ 406 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 407 if (!ASSERT_OK_PTR(link, "attach program to veth")) 408 goto out; 409 410 err = system("ip link set veth master bond"); 411 if (!ASSERT_OK(err, "set veth master")) 412 goto out; 413 414 bpf_link__destroy(link); 415 link = NULL; 416 417 /* attaching to slave when master has no program is allowed */ 418 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 419 if (!ASSERT_OK_PTR(link, "attach program to slave when enslaved")) 420 goto out; 421 422 /* attaching to master not allowed when slave has program loaded */ 423 link2 = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 424 if (!ASSERT_ERR_PTR(link2, "attach program to master when slave has program")) 425 goto out; 426 427 bpf_link__destroy(link); 428 link = NULL; 429 430 /* attaching XDP program to master allowed when slave has no program */ 431 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 432 if (!ASSERT_OK_PTR(link, "attach program to master")) 433 goto out; 434 435 /* attaching to slave not allowed when master has program loaded */ 436 link2 = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 437 if (!ASSERT_ERR_PTR(link2, "attach program to slave when master has program")) 438 goto out; 439 440 bpf_link__destroy(link); 441 link = NULL; 442 443 /* test program unwinding with a non-XDP slave */ 444 if (!ASSERT_OK(system("ip link add vxlan type vxlan id 1 remote 1.2.3.4 dstport 0 dev lo"), 445 "add vxlan")) 446 goto out; 447 448 err = system("ip link set vxlan master bond"); 449 if (!ASSERT_OK(err, "set vxlan master")) 450 goto out; 451 452 /* attaching not allowed when one slave does not support XDP */ 453 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 454 if (!ASSERT_ERR_PTR(link, "attach program to master when slave does not support XDP")) 455 goto out; 456 457 out: 458 bpf_link__destroy(link); 459 bpf_link__destroy(link2); 460 461 system("ip link del veth"); 462 system("ip link del bond"); 463 system("ip link del vxlan"); 464 } 465 466 /* Test with nested bonding devices to catch issue with negative jump label count */ 467 static void test_xdp_bonding_nested(struct skeletons *skeletons) 468 { 469 struct bpf_link *link = NULL; 470 int bond, err; 471 472 if (!ASSERT_OK(system("ip link add bond type bond"), "add bond")) 473 goto out; 474 475 bond = if_nametoindex("bond"); 476 if (!ASSERT_GE(bond, 0, "if_nametoindex bond")) 477 goto out; 478 479 if (!ASSERT_OK(system("ip link add bond_nest1 type bond"), "add bond_nest1")) 480 goto out; 481 482 err = system("ip link set bond_nest1 master bond"); 483 if (!ASSERT_OK(err, "set bond_nest1 master")) 484 goto out; 485 486 if (!ASSERT_OK(system("ip link add bond_nest2 type bond"), "add bond_nest1")) 487 goto out; 488 489 err = system("ip link set bond_nest2 master bond_nest1"); 490 if (!ASSERT_OK(err, "set bond_nest2 master")) 491 goto out; 492 493 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 494 ASSERT_OK_PTR(link, "attach program to master"); 495 496 out: 497 bpf_link__destroy(link); 498 system("ip link del bond"); 499 system("ip link del bond_nest1"); 500 system("ip link del bond_nest2"); 501 } 502 503 static int libbpf_debug_print(enum libbpf_print_level level, 504 const char *format, va_list args) 505 { 506 if (level != LIBBPF_WARN) 507 vprintf(format, args); 508 return 0; 509 } 510 511 struct bond_test_case { 512 char *name; 513 int mode; 514 int xmit_policy; 515 }; 516 517 static struct bond_test_case bond_test_cases[] = { 518 { "xdp_bonding_roundrobin", BOND_MODE_ROUNDROBIN, BOND_XMIT_POLICY_LAYER23, }, 519 { "xdp_bonding_activebackup", BOND_MODE_ACTIVEBACKUP, BOND_XMIT_POLICY_LAYER23 }, 520 521 { "xdp_bonding_xor_layer2", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER2, }, 522 { "xdp_bonding_xor_layer23", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER23, }, 523 { "xdp_bonding_xor_layer34", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER34, }, 524 }; 525 526 void serial_test_xdp_bonding(void) 527 { 528 libbpf_print_fn_t old_print_fn; 529 struct skeletons skeletons = {}; 530 int i; 531 532 old_print_fn = libbpf_set_print(libbpf_debug_print); 533 534 root_netns_fd = open("/proc/self/ns/net", O_RDONLY); 535 if (!ASSERT_GE(root_netns_fd, 0, "open /proc/self/ns/net")) 536 goto out; 537 538 skeletons.xdp_dummy = xdp_dummy__open_and_load(); 539 if (!ASSERT_OK_PTR(skeletons.xdp_dummy, "xdp_dummy__open_and_load")) 540 goto out; 541 542 skeletons.xdp_tx = xdp_tx__open_and_load(); 543 if (!ASSERT_OK_PTR(skeletons.xdp_tx, "xdp_tx__open_and_load")) 544 goto out; 545 546 skeletons.xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load(); 547 if (!ASSERT_OK_PTR(skeletons.xdp_redirect_multi_kern, 548 "xdp_redirect_multi_kern__open_and_load")) 549 goto out; 550 551 if (test__start_subtest("xdp_bonding_attach")) 552 test_xdp_bonding_attach(&skeletons); 553 554 if (test__start_subtest("xdp_bonding_nested")) 555 test_xdp_bonding_nested(&skeletons); 556 557 for (i = 0; i < ARRAY_SIZE(bond_test_cases); i++) { 558 struct bond_test_case *test_case = &bond_test_cases[i]; 559 560 if (test__start_subtest(test_case->name)) 561 test_xdp_bonding_with_mode( 562 &skeletons, 563 test_case->mode, 564 test_case->xmit_policy); 565 } 566 567 if (test__start_subtest("xdp_bonding_redirect_multi")) 568 test_xdp_bonding_redirect_multi(&skeletons); 569 570 out: 571 xdp_dummy__destroy(skeletons.xdp_dummy); 572 xdp_tx__destroy(skeletons.xdp_tx); 573 xdp_redirect_multi_kern__destroy(skeletons.xdp_redirect_multi_kern); 574 575 libbpf_set_print(old_print_fn); 576 if (root_netns_fd >= 0) 577 close(root_netns_fd); 578 } 579