1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 /* 5 * Some functions in this program are taken from 6 * Linux kernel samples/bpf/xdpsock* and modified 7 * for use. 8 * 9 * See test_xsk.sh for detailed information on test topology 10 * and prerequisite network setup. 11 * 12 * This test program contains two threads, each thread is single socket with 13 * a unique UMEM. It validates in-order packet delivery and packet content 14 * by sending packets to each other. 15 * 16 * Tests Information: 17 * ------------------ 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 * Virtual Ethernet interfaces. 20 * 21 * For each mode, the following tests are run: 22 * a. nopoll - soft-irq processing in run-to-completion mode 23 * b. poll - using poll() syscall 24 * c. Socket Teardown 25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy 26 * both sockets, then repeat multiple times. Only nopoll mode is used 27 * d. Bi-directional sockets 28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 * mode is used 31 * e. Statistics 32 * Trigger some error conditions and ensure that the appropriate statistics 33 * are incremented. Within this test, the following statistics are tested: 34 * i. rx dropped 35 * Increase the UMEM frame headroom to a value which results in 36 * insufficient space in the rx buffer for both the packet and the headroom. 37 * ii. tx invalid 38 * Set the 'len' field of tx descriptors to an invalid value (umem frame 39 * size + 1). 40 * iii. rx ring full 41 * Reduce the size of the RX ring to a fraction of the fill ring size. 42 * iv. fill queue empty 43 * Do not populate the fill queue and then try to receive pkts. 44 * f. bpf_link resource persistence 45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, 46 * then remove xsk sockets from queue 0 on both veth interfaces and 47 * finally run a traffic on queues ids 1 48 * g. unaligned mode 49 * h. tests for invalid and corner case Tx descriptors so that the correct ones 50 * are discarded and let through, respectively. 51 * i. 2K frame size tests 52 * 53 * Total tests: 12 54 * 55 * Flow: 56 * ----- 57 * - Single process spawns two threads: Tx and Rx 58 * - Each of these two threads attach to a veth interface 59 * - Each thread creates one AF_XDP socket connected to a unique umem for each 60 * veth interface 61 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy> 62 * - Rx thread verifies if all packets were received and delivered in-order, 63 * and have the right content 64 * 65 * Enable/disable packet dump mode: 66 * -------------------------- 67 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 68 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") 69 */ 70 71 #define _GNU_SOURCE 72 #include <assert.h> 73 #include <fcntl.h> 74 #include <errno.h> 75 #include <getopt.h> 76 #include <asm/barrier.h> 77 #include <linux/if_link.h> 78 #include <linux/if_ether.h> 79 #include <linux/ip.h> 80 #include <linux/mman.h> 81 #include <linux/udp.h> 82 #include <arpa/inet.h> 83 #include <net/if.h> 84 #include <locale.h> 85 #include <poll.h> 86 #include <pthread.h> 87 #include <signal.h> 88 #include <stdbool.h> 89 #include <stdio.h> 90 #include <stdlib.h> 91 #include <string.h> 92 #include <stddef.h> 93 #include <sys/mman.h> 94 #include <sys/socket.h> 95 #include <sys/time.h> 96 #include <sys/types.h> 97 #include <sys/queue.h> 98 #include <time.h> 99 #include <unistd.h> 100 #include <stdatomic.h> 101 102 #include "xsk_xdp_progs.skel.h" 103 #include "xsk.h" 104 #include "xskxceiver.h" 105 #include <bpf/bpf.h> 106 #include <linux/filter.h> 107 #include "../kselftest.h" 108 #include "xsk_xdp_metadata.h" 109 110 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62"; 111 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61"; 112 static const char *IP1 = "192.168.100.162"; 113 static const char *IP2 = "192.168.100.161"; 114 static const u16 UDP_PORT1 = 2020; 115 static const u16 UDP_PORT2 = 2121; 116 117 static void __exit_with_error(int error, const char *file, const char *func, int line) 118 { 119 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, 120 strerror(error)); 121 ksft_exit_xfail(); 122 } 123 124 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 125 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 126 static char *mode_string(struct test_spec *test) 127 { 128 switch (test->mode) { 129 case TEST_MODE_SKB: 130 return "SKB"; 131 case TEST_MODE_DRV: 132 return "DRV"; 133 case TEST_MODE_ZC: 134 return "ZC"; 135 default: 136 return "BOGUS"; 137 } 138 } 139 140 static void report_failure(struct test_spec *test) 141 { 142 if (test->fail) 143 return; 144 145 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), 146 test->name); 147 test->fail = true; 148 } 149 150 static void memset32_htonl(void *dest, u32 val, u32 size) 151 { 152 u32 *ptr = (u32 *)dest; 153 int i; 154 155 val = htonl(val); 156 157 for (i = 0; i < (size & (~0x3)); i += 4) 158 ptr[i >> 2] = val; 159 } 160 161 /* 162 * Fold a partial checksum 163 * This function code has been taken from 164 * Linux kernel include/asm-generic/checksum.h 165 */ 166 static __u16 csum_fold(__u32 csum) 167 { 168 u32 sum = (__force u32)csum; 169 170 sum = (sum & 0xffff) + (sum >> 16); 171 sum = (sum & 0xffff) + (sum >> 16); 172 return (__force __u16)~sum; 173 } 174 175 /* 176 * This function code has been taken from 177 * Linux kernel lib/checksum.c 178 */ 179 static u32 from64to32(u64 x) 180 { 181 /* add up 32-bit and 32-bit for 32+c bit */ 182 x = (x & 0xffffffff) + (x >> 32); 183 /* add up carry.. */ 184 x = (x & 0xffffffff) + (x >> 32); 185 return (u32)x; 186 } 187 188 /* 189 * This function code has been taken from 190 * Linux kernel lib/checksum.c 191 */ 192 static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum) 193 { 194 unsigned long long s = (__force u32)sum; 195 196 s += (__force u32)saddr; 197 s += (__force u32)daddr; 198 #ifdef __BIG_ENDIAN__ 199 s += proto + len; 200 #else 201 s += (proto + len) << 8; 202 #endif 203 return (__force __u32)from64to32(s); 204 } 205 206 /* 207 * This function has been taken from 208 * Linux kernel include/asm-generic/checksum.h 209 */ 210 static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum) 211 { 212 return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); 213 } 214 215 static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt) 216 { 217 u32 csum = 0; 218 u32 cnt = 0; 219 220 /* udp hdr and data */ 221 for (; cnt < len; cnt += 2) 222 csum += udp_pkt[cnt >> 1]; 223 224 return csum_tcpudp_magic(saddr, daddr, len, proto, csum); 225 } 226 227 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr) 228 { 229 memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN); 230 memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN); 231 eth_hdr->h_proto = htons(ETH_P_IP); 232 } 233 234 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr) 235 { 236 ip_hdr->version = IP_PKT_VER; 237 ip_hdr->ihl = 0x5; 238 ip_hdr->tos = IP_PKT_TOS; 239 ip_hdr->tot_len = htons(IP_PKT_SIZE); 240 ip_hdr->id = 0; 241 ip_hdr->frag_off = 0; 242 ip_hdr->ttl = IPDEFTTL; 243 ip_hdr->protocol = IPPROTO_UDP; 244 ip_hdr->saddr = ifobject->src_ip; 245 ip_hdr->daddr = ifobject->dst_ip; 246 ip_hdr->check = 0; 247 } 248 249 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject, 250 struct udphdr *udp_hdr) 251 { 252 udp_hdr->source = htons(ifobject->src_port); 253 udp_hdr->dest = htons(ifobject->dst_port); 254 udp_hdr->len = htons(UDP_PKT_SIZE); 255 memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE); 256 } 257 258 static bool is_umem_valid(struct ifobject *ifobj) 259 { 260 return !!ifobj->umem->umem; 261 } 262 263 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr) 264 { 265 udp_hdr->check = 0; 266 udp_hdr->check = 267 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr); 268 } 269 270 static u32 mode_to_xdp_flags(enum test_mode mode) 271 { 272 return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; 273 } 274 275 static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size) 276 { 277 struct xsk_umem_config cfg = { 278 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, 279 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, 280 .frame_size = umem->frame_size, 281 .frame_headroom = umem->frame_headroom, 282 .flags = XSK_UMEM__DEFAULT_FLAGS 283 }; 284 int ret; 285 286 if (umem->unaligned_mode) 287 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; 288 289 ret = xsk_umem__create(&umem->umem, buffer, size, 290 &umem->fq, &umem->cq, &cfg); 291 if (ret) 292 return ret; 293 294 umem->buffer = buffer; 295 return 0; 296 } 297 298 static void enable_busy_poll(struct xsk_socket_info *xsk) 299 { 300 int sock_opt; 301 302 sock_opt = 1; 303 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, 304 (void *)&sock_opt, sizeof(sock_opt)) < 0) 305 exit_with_error(errno); 306 307 sock_opt = 20; 308 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, 309 (void *)&sock_opt, sizeof(sock_opt)) < 0) 310 exit_with_error(errno); 311 312 sock_opt = BATCH_SIZE; 313 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, 314 (void *)&sock_opt, sizeof(sock_opt)) < 0) 315 exit_with_error(errno); 316 } 317 318 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 319 struct ifobject *ifobject, bool shared) 320 { 321 struct xsk_socket_config cfg = {}; 322 struct xsk_ring_cons *rxr; 323 struct xsk_ring_prod *txr; 324 325 xsk->umem = umem; 326 cfg.rx_size = xsk->rxqsize; 327 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 328 cfg.bind_flags = ifobject->bind_flags; 329 if (shared) 330 cfg.bind_flags |= XDP_SHARED_UMEM; 331 332 txr = ifobject->tx_on ? &xsk->tx : NULL; 333 rxr = ifobject->rx_on ? &xsk->rx : NULL; 334 return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); 335 } 336 337 static bool ifobj_zc_avail(struct ifobject *ifobject) 338 { 339 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 340 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 341 struct xsk_socket_info *xsk; 342 struct xsk_umem_info *umem; 343 bool zc_avail = false; 344 void *bufs; 345 int ret; 346 347 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 348 if (bufs == MAP_FAILED) 349 exit_with_error(errno); 350 351 umem = calloc(1, sizeof(struct xsk_umem_info)); 352 if (!umem) { 353 munmap(bufs, umem_sz); 354 exit_with_error(ENOMEM); 355 } 356 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 357 ret = xsk_configure_umem(umem, bufs, umem_sz); 358 if (ret) 359 exit_with_error(-ret); 360 361 xsk = calloc(1, sizeof(struct xsk_socket_info)); 362 if (!xsk) 363 goto out; 364 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; 365 ifobject->rx_on = true; 366 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 367 ret = __xsk_configure_socket(xsk, umem, ifobject, false); 368 if (!ret) 369 zc_avail = true; 370 371 xsk_socket__delete(xsk->xsk); 372 free(xsk); 373 out: 374 munmap(umem->buffer, umem_sz); 375 xsk_umem__delete(umem->umem); 376 free(umem); 377 return zc_avail; 378 } 379 380 static struct option long_options[] = { 381 {"interface", required_argument, 0, 'i'}, 382 {"busy-poll", no_argument, 0, 'b'}, 383 {"dump-pkts", no_argument, 0, 'D'}, 384 {"verbose", no_argument, 0, 'v'}, 385 {0, 0, 0, 0} 386 }; 387 388 static void usage(const char *prog) 389 { 390 const char *str = 391 " Usage: %s [OPTIONS]\n" 392 " Options:\n" 393 " -i, --interface Use interface\n" 394 " -D, --dump-pkts Dump packets L2 - L5\n" 395 " -v, --verbose Verbose output\n" 396 " -b, --busy-poll Enable busy poll\n"; 397 398 ksft_print_msg(str, prog); 399 } 400 401 static bool validate_interface(struct ifobject *ifobj) 402 { 403 if (!strcmp(ifobj->ifname, "")) 404 return false; 405 return true; 406 } 407 408 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, 409 char **argv) 410 { 411 struct ifobject *ifobj; 412 u32 interface_nb = 0; 413 int option_index, c; 414 415 opterr = 0; 416 417 for (;;) { 418 c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index); 419 if (c == -1) 420 break; 421 422 switch (c) { 423 case 'i': 424 if (interface_nb == 0) 425 ifobj = ifobj_tx; 426 else if (interface_nb == 1) 427 ifobj = ifobj_rx; 428 else 429 break; 430 431 memcpy(ifobj->ifname, optarg, 432 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg))); 433 434 ifobj->ifindex = if_nametoindex(ifobj->ifname); 435 if (!ifobj->ifindex) 436 exit_with_error(errno); 437 438 interface_nb++; 439 break; 440 case 'D': 441 opt_pkt_dump = true; 442 break; 443 case 'v': 444 opt_verbose = true; 445 break; 446 case 'b': 447 ifobj_tx->busy_poll = true; 448 ifobj_rx->busy_poll = true; 449 break; 450 default: 451 usage(basename(argv[0])); 452 ksft_exit_xfail(); 453 } 454 } 455 } 456 457 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 458 struct ifobject *ifobj_rx) 459 { 460 u32 i, j; 461 462 for (i = 0; i < MAX_INTERFACES; i++) { 463 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 464 465 ifobj->xsk = &ifobj->xsk_arr[0]; 466 ifobj->use_poll = false; 467 ifobj->use_fill_ring = true; 468 ifobj->release_rx = true; 469 ifobj->validation_func = NULL; 470 ifobj->use_metadata = false; 471 472 if (i == 0) { 473 ifobj->rx_on = false; 474 ifobj->tx_on = true; 475 ifobj->pkt_stream = test->tx_pkt_stream_default; 476 } else { 477 ifobj->rx_on = true; 478 ifobj->tx_on = false; 479 ifobj->pkt_stream = test->rx_pkt_stream_default; 480 } 481 482 memset(ifobj->umem, 0, sizeof(*ifobj->umem)); 483 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; 484 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 485 if (ifobj->shared_umem && ifobj->rx_on) 486 ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS * 487 XSK_UMEM__DEFAULT_FRAME_SIZE; 488 489 for (j = 0; j < MAX_SOCKETS; j++) { 490 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); 491 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 492 } 493 } 494 495 test->ifobj_tx = ifobj_tx; 496 test->ifobj_rx = ifobj_rx; 497 test->current_step = 0; 498 test->total_steps = 1; 499 test->nb_sockets = 1; 500 test->fail = false; 501 test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; 502 test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; 503 test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; 504 test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; 505 } 506 507 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 508 struct ifobject *ifobj_rx, enum test_mode mode) 509 { 510 struct pkt_stream *tx_pkt_stream; 511 struct pkt_stream *rx_pkt_stream; 512 u32 i; 513 514 tx_pkt_stream = test->tx_pkt_stream_default; 515 rx_pkt_stream = test->rx_pkt_stream_default; 516 memset(test, 0, sizeof(*test)); 517 test->tx_pkt_stream_default = tx_pkt_stream; 518 test->rx_pkt_stream_default = rx_pkt_stream; 519 520 for (i = 0; i < MAX_INTERFACES; i++) { 521 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 522 523 ifobj->bind_flags = XDP_USE_NEED_WAKEUP; 524 if (mode == TEST_MODE_ZC) 525 ifobj->bind_flags |= XDP_ZEROCOPY; 526 else 527 ifobj->bind_flags |= XDP_COPY; 528 } 529 530 test->mode = mode; 531 __test_spec_init(test, ifobj_tx, ifobj_rx); 532 } 533 534 static void test_spec_reset(struct test_spec *test) 535 { 536 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); 537 } 538 539 static void test_spec_set_name(struct test_spec *test, const char *name) 540 { 541 strncpy(test->name, name, MAX_TEST_NAME_SIZE); 542 } 543 544 static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, 545 struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, 546 struct bpf_map *xskmap_tx) 547 { 548 test->xdp_prog_rx = xdp_prog_rx; 549 test->xdp_prog_tx = xdp_prog_tx; 550 test->xskmap_rx = xskmap_rx; 551 test->xskmap_tx = xskmap_tx; 552 } 553 554 static void pkt_stream_reset(struct pkt_stream *pkt_stream) 555 { 556 if (pkt_stream) 557 pkt_stream->rx_pkt_nb = 0; 558 } 559 560 static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb) 561 { 562 if (pkt_nb >= pkt_stream->nb_pkts) 563 return NULL; 564 565 return &pkt_stream->pkts[pkt_nb]; 566 } 567 568 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) 569 { 570 while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) { 571 (*pkts_sent)++; 572 if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid) 573 return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++]; 574 pkt_stream->rx_pkt_nb++; 575 } 576 return NULL; 577 } 578 579 static void pkt_stream_delete(struct pkt_stream *pkt_stream) 580 { 581 free(pkt_stream->pkts); 582 free(pkt_stream); 583 } 584 585 static void pkt_stream_restore_default(struct test_spec *test) 586 { 587 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream; 588 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream; 589 590 if (tx_pkt_stream != test->tx_pkt_stream_default) { 591 pkt_stream_delete(test->ifobj_tx->pkt_stream); 592 test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default; 593 } 594 595 if (rx_pkt_stream != test->rx_pkt_stream_default) { 596 pkt_stream_delete(test->ifobj_rx->pkt_stream); 597 test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default; 598 } 599 } 600 601 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) 602 { 603 struct pkt_stream *pkt_stream; 604 605 pkt_stream = calloc(1, sizeof(*pkt_stream)); 606 if (!pkt_stream) 607 return NULL; 608 609 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); 610 if (!pkt_stream->pkts) { 611 free(pkt_stream); 612 return NULL; 613 } 614 615 pkt_stream->nb_pkts = nb_pkts; 616 return pkt_stream; 617 } 618 619 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len) 620 { 621 pkt->addr = addr + umem->base_addr; 622 pkt->len = len; 623 if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom) 624 pkt->valid = false; 625 else 626 pkt->valid = true; 627 } 628 629 static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len) 630 { 631 struct pkt_stream *pkt_stream; 632 u32 i; 633 634 pkt_stream = __pkt_stream_alloc(nb_pkts); 635 if (!pkt_stream) 636 exit_with_error(ENOMEM); 637 638 for (i = 0; i < nb_pkts; i++) { 639 pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size, 640 pkt_len); 641 pkt_stream->pkts[i].payload = i; 642 } 643 644 return pkt_stream; 645 } 646 647 static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem, 648 struct pkt_stream *pkt_stream) 649 { 650 return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len); 651 } 652 653 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) 654 { 655 struct pkt_stream *pkt_stream; 656 657 pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len); 658 test->ifobj_tx->pkt_stream = pkt_stream; 659 pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len); 660 test->ifobj_rx->pkt_stream = pkt_stream; 661 } 662 663 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, 664 int offset) 665 { 666 struct xsk_umem_info *umem = ifobj->umem; 667 struct pkt_stream *pkt_stream; 668 u32 i; 669 670 pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream); 671 for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2) 672 pkt_set(umem, &pkt_stream->pkts[i], 673 (i % umem->num_frames) * umem->frame_size + offset, pkt_len); 674 675 ifobj->pkt_stream = pkt_stream; 676 } 677 678 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) 679 { 680 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); 681 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); 682 } 683 684 static void pkt_stream_receive_half(struct test_spec *test) 685 { 686 struct xsk_umem_info *umem = test->ifobj_rx->umem; 687 struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream; 688 u32 i; 689 690 test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts, 691 pkt_stream->pkts[0].len); 692 pkt_stream = test->ifobj_rx->pkt_stream; 693 for (i = 1; i < pkt_stream->nb_pkts; i += 2) 694 pkt_stream->pkts[i].valid = false; 695 } 696 697 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb) 698 { 699 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb); 700 struct udphdr *udp_hdr; 701 struct ethhdr *eth_hdr; 702 struct iphdr *ip_hdr; 703 void *data; 704 705 if (!pkt) 706 return NULL; 707 if (!pkt->valid || pkt->len < MIN_PKT_SIZE) 708 return pkt; 709 710 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr); 711 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr)); 712 ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr)); 713 eth_hdr = (struct ethhdr *)data; 714 715 gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr); 716 gen_ip_hdr(ifobject, ip_hdr); 717 gen_udp_csum(udp_hdr, ip_hdr); 718 gen_eth_hdr(ifobject, eth_hdr); 719 720 return pkt; 721 } 722 723 static void __pkt_stream_generate_custom(struct ifobject *ifobj, 724 struct pkt *pkts, u32 nb_pkts) 725 { 726 struct pkt_stream *pkt_stream; 727 u32 i; 728 729 pkt_stream = __pkt_stream_alloc(nb_pkts); 730 if (!pkt_stream) 731 exit_with_error(ENOMEM); 732 733 for (i = 0; i < nb_pkts; i++) { 734 pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr; 735 pkt_stream->pkts[i].len = pkts[i].len; 736 pkt_stream->pkts[i].payload = i; 737 pkt_stream->pkts[i].valid = pkts[i].valid; 738 } 739 740 ifobj->pkt_stream = pkt_stream; 741 } 742 743 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) 744 { 745 __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts); 746 __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts); 747 } 748 749 static void pkt_dump(void *pkt, u32 len) 750 { 751 char s[INET_ADDRSTRLEN]; 752 struct ethhdr *ethhdr; 753 struct udphdr *udphdr; 754 struct iphdr *iphdr; 755 u32 payload, i; 756 757 ethhdr = pkt; 758 iphdr = pkt + sizeof(*ethhdr); 759 udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr); 760 761 /*extract L2 frame */ 762 fprintf(stdout, "DEBUG>> L2: dst mac: "); 763 for (i = 0; i < ETH_ALEN; i++) 764 fprintf(stdout, "%02X", ethhdr->h_dest[i]); 765 766 fprintf(stdout, "\nDEBUG>> L2: src mac: "); 767 for (i = 0; i < ETH_ALEN; i++) 768 fprintf(stdout, "%02X", ethhdr->h_source[i]); 769 770 /*extract L3 frame */ 771 fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl); 772 fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n", 773 inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s))); 774 fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n", 775 inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s))); 776 /*extract L4 frame */ 777 fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source)); 778 fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest)); 779 /*extract L5 frame */ 780 payload = ntohl(*((u32 *)(pkt + PKT_HDR_SIZE))); 781 782 fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload); 783 fprintf(stdout, "---------------------------------------\n"); 784 } 785 786 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr, 787 u64 pkt_stream_addr) 788 { 789 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; 790 u32 offset = addr % umem->frame_size, expected_offset = 0; 791 792 if (!pkt_stream->use_addr_for_fill) 793 pkt_stream_addr = 0; 794 795 expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; 796 797 if (offset == expected_offset) 798 return true; 799 800 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); 801 return false; 802 } 803 804 static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) 805 { 806 void *data = xsk_umem__get_data(buffer, addr); 807 struct xdp_info *meta = data - sizeof(struct xdp_info); 808 809 if (meta->count != pkt->payload) { 810 ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%d]\n", 811 __func__, pkt->payload, meta->count); 812 return false; 813 } 814 815 return true; 816 } 817 818 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) 819 { 820 void *data = xsk_umem__get_data(buffer, addr); 821 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr)); 822 823 if (!pkt) { 824 ksft_print_msg("[%s] too many packets received\n", __func__); 825 return false; 826 } 827 828 if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) { 829 /* Do not try to verify packets that are smaller than minimum size. */ 830 return true; 831 } 832 833 if (pkt->len != len) { 834 ksft_print_msg("[%s] expected length [%d], got length [%d]\n", 835 __func__, pkt->len, len); 836 return false; 837 } 838 839 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) { 840 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE))); 841 842 if (opt_pkt_dump) 843 pkt_dump(data, PKT_SIZE); 844 845 if (pkt->payload != seqnum) { 846 ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n", 847 __func__, pkt->payload, seqnum); 848 return false; 849 } 850 } else { 851 ksft_print_msg("Invalid frame received: "); 852 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version, 853 iphdr->tos); 854 return false; 855 } 856 857 return true; 858 } 859 860 static void kick_tx(struct xsk_socket_info *xsk) 861 { 862 int ret; 863 864 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); 865 if (ret >= 0) 866 return; 867 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { 868 usleep(100); 869 return; 870 } 871 exit_with_error(errno); 872 } 873 874 static void kick_rx(struct xsk_socket_info *xsk) 875 { 876 int ret; 877 878 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); 879 if (ret < 0) 880 exit_with_error(errno); 881 } 882 883 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) 884 { 885 unsigned int rcvd; 886 u32 idx; 887 888 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) 889 kick_tx(xsk); 890 891 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); 892 if (rcvd) { 893 if (rcvd > xsk->outstanding_tx) { 894 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); 895 896 ksft_print_msg("[%s] Too many packets completed\n", __func__); 897 ksft_print_msg("Last completion address: %llx\n", addr); 898 return TEST_FAILURE; 899 } 900 901 xsk_ring_cons__release(&xsk->umem->cq, rcvd); 902 xsk->outstanding_tx -= rcvd; 903 } 904 905 return TEST_PASS; 906 } 907 908 static int receive_pkts(struct test_spec *test, struct pollfd *fds) 909 { 910 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 911 struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream; 912 u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0; 913 struct xsk_socket_info *xsk = test->ifobj_rx->xsk; 914 struct ifobject *ifobj = test->ifobj_rx; 915 struct xsk_umem_info *umem = xsk->umem; 916 struct pkt *pkt; 917 int ret; 918 919 ret = gettimeofday(&tv_now, NULL); 920 if (ret) 921 exit_with_error(errno); 922 timeradd(&tv_now, &tv_timeout, &tv_end); 923 924 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 925 while (pkt) { 926 ret = gettimeofday(&tv_now, NULL); 927 if (ret) 928 exit_with_error(errno); 929 if (timercmp(&tv_now, &tv_end, >)) { 930 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); 931 return TEST_FAILURE; 932 } 933 934 kick_rx(xsk); 935 if (ifobj->use_poll) { 936 ret = poll(fds, 1, POLL_TMOUT); 937 if (ret < 0) 938 exit_with_error(errno); 939 940 if (!ret) { 941 if (!is_umem_valid(test->ifobj_tx)) 942 return TEST_PASS; 943 944 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); 945 return TEST_FAILURE; 946 947 } 948 949 if (!(fds->revents & POLLIN)) 950 continue; 951 } 952 953 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); 954 if (!rcvd) 955 continue; 956 957 if (ifobj->use_fill_ring) { 958 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 959 while (ret != rcvd) { 960 if (ret < 0) 961 exit_with_error(-ret); 962 if (xsk_ring_prod__needs_wakeup(&umem->fq)) { 963 ret = poll(fds, 1, POLL_TMOUT); 964 if (ret < 0) 965 exit_with_error(errno); 966 } 967 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 968 } 969 } 970 971 for (i = 0; i < rcvd; i++) { 972 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); 973 u64 addr = desc->addr, orig; 974 975 orig = xsk_umem__extract_addr(addr); 976 addr = xsk_umem__add_offset_to_addr(addr); 977 978 if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) || 979 !is_offset_correct(umem, pkt_stream, addr, pkt->addr) || 980 (ifobj->use_metadata && !is_metadata_correct(pkt, umem->buffer, addr))) 981 return TEST_FAILURE; 982 983 if (ifobj->use_fill_ring) 984 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; 985 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 986 } 987 988 if (ifobj->use_fill_ring) 989 xsk_ring_prod__submit(&umem->fq, rcvd); 990 if (ifobj->release_rx) 991 xsk_ring_cons__release(&xsk->rx, rcvd); 992 993 pthread_mutex_lock(&pacing_mutex); 994 pkts_in_flight -= pkts_sent; 995 if (pkts_in_flight < umem->num_frames) 996 pthread_cond_signal(&pacing_cond); 997 pthread_mutex_unlock(&pacing_mutex); 998 pkts_sent = 0; 999 } 1000 1001 return TEST_PASS; 1002 } 1003 1004 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds, 1005 bool timeout) 1006 { 1007 struct xsk_socket_info *xsk = ifobject->xsk; 1008 bool use_poll = ifobject->use_poll; 1009 u32 i, idx = 0, valid_pkts = 0; 1010 int ret; 1011 1012 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) { 1013 if (use_poll) { 1014 ret = poll(fds, 1, POLL_TMOUT); 1015 if (timeout) { 1016 if (ret < 0) { 1017 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1018 __func__, errno); 1019 return TEST_FAILURE; 1020 } 1021 if (ret == 0) 1022 return TEST_PASS; 1023 break; 1024 } 1025 if (ret <= 0) { 1026 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1027 __func__, errno); 1028 return TEST_FAILURE; 1029 } 1030 } 1031 1032 complete_pkts(xsk, BATCH_SIZE); 1033 } 1034 1035 for (i = 0; i < BATCH_SIZE; i++) { 1036 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); 1037 struct pkt *pkt = pkt_generate(ifobject, *pkt_nb); 1038 1039 if (!pkt) 1040 break; 1041 1042 tx_desc->addr = pkt->addr; 1043 tx_desc->len = pkt->len; 1044 (*pkt_nb)++; 1045 if (pkt->valid) 1046 valid_pkts++; 1047 } 1048 1049 pthread_mutex_lock(&pacing_mutex); 1050 pkts_in_flight += valid_pkts; 1051 /* pkts_in_flight might be negative if many invalid packets are sent */ 1052 if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) { 1053 kick_tx(xsk); 1054 pthread_cond_wait(&pacing_cond, &pacing_mutex); 1055 } 1056 pthread_mutex_unlock(&pacing_mutex); 1057 1058 xsk_ring_prod__submit(&xsk->tx, i); 1059 xsk->outstanding_tx += valid_pkts; 1060 1061 if (use_poll) { 1062 ret = poll(fds, 1, POLL_TMOUT); 1063 if (ret <= 0) { 1064 if (ret == 0 && timeout) 1065 return TEST_PASS; 1066 1067 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); 1068 return TEST_FAILURE; 1069 } 1070 } 1071 1072 if (!timeout) { 1073 if (complete_pkts(xsk, i)) 1074 return TEST_FAILURE; 1075 1076 usleep(10); 1077 return TEST_PASS; 1078 } 1079 1080 return TEST_CONTINUE; 1081 } 1082 1083 static void wait_for_tx_completion(struct xsk_socket_info *xsk) 1084 { 1085 while (xsk->outstanding_tx) 1086 complete_pkts(xsk, BATCH_SIZE); 1087 } 1088 1089 static int send_pkts(struct test_spec *test, struct ifobject *ifobject) 1090 { 1091 bool timeout = !is_umem_valid(test->ifobj_rx); 1092 struct pollfd fds = { }; 1093 u32 pkt_cnt = 0, ret; 1094 1095 fds.fd = xsk_socket__fd(ifobject->xsk->xsk); 1096 fds.events = POLLOUT; 1097 1098 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) { 1099 ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout); 1100 if ((ret || test->fail) && !timeout) 1101 return TEST_FAILURE; 1102 else if (ret == TEST_PASS && timeout) 1103 return ret; 1104 } 1105 1106 wait_for_tx_completion(ifobject->xsk); 1107 return TEST_PASS; 1108 } 1109 1110 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) 1111 { 1112 int fd = xsk_socket__fd(xsk), err; 1113 socklen_t optlen, expected_len; 1114 1115 optlen = sizeof(*stats); 1116 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); 1117 if (err) { 1118 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1119 __func__, -err, strerror(-err)); 1120 return TEST_FAILURE; 1121 } 1122 1123 expected_len = sizeof(struct xdp_statistics); 1124 if (optlen != expected_len) { 1125 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", 1126 __func__, expected_len, optlen); 1127 return TEST_FAILURE; 1128 } 1129 1130 return TEST_PASS; 1131 } 1132 1133 static int validate_rx_dropped(struct ifobject *ifobject) 1134 { 1135 struct xsk_socket *xsk = ifobject->xsk->xsk; 1136 struct xdp_statistics stats; 1137 int err; 1138 1139 kick_rx(ifobject->xsk); 1140 1141 err = get_xsk_stats(xsk, &stats); 1142 if (err) 1143 return TEST_FAILURE; 1144 1145 /* The receiver calls getsockopt after receiving the last (valid) 1146 * packet which is not the final packet sent in this test (valid and 1147 * invalid packets are sent in alternating fashion with the final 1148 * packet being invalid). Since the last packet may or may not have 1149 * been dropped already, both outcomes must be allowed. 1150 */ 1151 if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 || 1152 stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 - 1) 1153 return TEST_PASS; 1154 1155 return TEST_FAILURE; 1156 } 1157 1158 static int validate_rx_full(struct ifobject *ifobject) 1159 { 1160 struct xsk_socket *xsk = ifobject->xsk->xsk; 1161 struct xdp_statistics stats; 1162 int err; 1163 1164 usleep(1000); 1165 kick_rx(ifobject->xsk); 1166 1167 err = get_xsk_stats(xsk, &stats); 1168 if (err) 1169 return TEST_FAILURE; 1170 1171 if (stats.rx_ring_full) 1172 return TEST_PASS; 1173 1174 return TEST_FAILURE; 1175 } 1176 1177 static int validate_fill_empty(struct ifobject *ifobject) 1178 { 1179 struct xsk_socket *xsk = ifobject->xsk->xsk; 1180 struct xdp_statistics stats; 1181 int err; 1182 1183 usleep(1000); 1184 kick_rx(ifobject->xsk); 1185 1186 err = get_xsk_stats(xsk, &stats); 1187 if (err) 1188 return TEST_FAILURE; 1189 1190 if (stats.rx_fill_ring_empty_descs) 1191 return TEST_PASS; 1192 1193 return TEST_FAILURE; 1194 } 1195 1196 static int validate_tx_invalid_descs(struct ifobject *ifobject) 1197 { 1198 struct xsk_socket *xsk = ifobject->xsk->xsk; 1199 int fd = xsk_socket__fd(xsk); 1200 struct xdp_statistics stats; 1201 socklen_t optlen; 1202 int err; 1203 1204 optlen = sizeof(stats); 1205 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); 1206 if (err) { 1207 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1208 __func__, -err, strerror(-err)); 1209 return TEST_FAILURE; 1210 } 1211 1212 if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) { 1213 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n", 1214 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts); 1215 return TEST_FAILURE; 1216 } 1217 1218 return TEST_PASS; 1219 } 1220 1221 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, 1222 struct xsk_umem_info *umem, bool tx) 1223 { 1224 int i, ret; 1225 1226 for (i = 0; i < test->nb_sockets; i++) { 1227 bool shared = (ifobject->shared_umem && tx) ? true : !!i; 1228 u32 ctr = 0; 1229 1230 while (ctr++ < SOCK_RECONF_CTR) { 1231 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, 1232 ifobject, shared); 1233 if (!ret) 1234 break; 1235 1236 /* Retry if it fails as xsk_socket__create() is asynchronous */ 1237 if (ctr >= SOCK_RECONF_CTR) 1238 exit_with_error(-ret); 1239 usleep(USLEEP_MAX); 1240 } 1241 if (ifobject->busy_poll) 1242 enable_busy_poll(&ifobject->xsk_arr[i]); 1243 } 1244 } 1245 1246 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) 1247 { 1248 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); 1249 ifobject->xsk = &ifobject->xsk_arr[0]; 1250 ifobject->xskmap = test->ifobj_rx->xskmap; 1251 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); 1252 } 1253 1254 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream) 1255 { 1256 u32 idx = 0, i, buffers_to_fill; 1257 int ret; 1258 1259 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) 1260 buffers_to_fill = umem->num_frames; 1261 else 1262 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS; 1263 1264 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); 1265 if (ret != buffers_to_fill) 1266 exit_with_error(ENOSPC); 1267 for (i = 0; i < buffers_to_fill; i++) { 1268 u64 addr; 1269 1270 if (pkt_stream->use_addr_for_fill) { 1271 struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i); 1272 1273 if (!pkt) 1274 break; 1275 addr = pkt->addr; 1276 } else { 1277 addr = i * umem->frame_size; 1278 } 1279 1280 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; 1281 } 1282 xsk_ring_prod__submit(&umem->fq, i); 1283 } 1284 1285 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) 1286 { 1287 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; 1288 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 1289 LIBBPF_OPTS(bpf_xdp_query_opts, opts); 1290 void *bufs; 1291 int ret; 1292 1293 if (ifobject->umem->unaligned_mode) 1294 mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; 1295 1296 if (ifobject->shared_umem) 1297 umem_sz *= 2; 1298 1299 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 1300 if (bufs == MAP_FAILED) 1301 exit_with_error(errno); 1302 1303 ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz); 1304 if (ret) 1305 exit_with_error(-ret); 1306 1307 xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream); 1308 1309 xsk_configure_socket(test, ifobject, ifobject->umem, false); 1310 1311 ifobject->xsk = &ifobject->xsk_arr[0]; 1312 1313 if (!ifobject->rx_on) 1314 return; 1315 1316 ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk); 1317 if (ret) 1318 exit_with_error(errno); 1319 } 1320 1321 static void *worker_testapp_validate_tx(void *arg) 1322 { 1323 struct test_spec *test = (struct test_spec *)arg; 1324 struct ifobject *ifobject = test->ifobj_tx; 1325 int err; 1326 1327 if (test->current_step == 1) { 1328 if (!ifobject->shared_umem) 1329 thread_common_ops(test, ifobject); 1330 else 1331 thread_common_ops_tx(test, ifobject); 1332 } 1333 1334 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts, 1335 ifobject->ifname); 1336 err = send_pkts(test, ifobject); 1337 1338 if (!err && ifobject->validation_func) 1339 err = ifobject->validation_func(ifobject); 1340 if (err) 1341 report_failure(test); 1342 1343 pthread_exit(NULL); 1344 } 1345 1346 static void *worker_testapp_validate_rx(void *arg) 1347 { 1348 struct test_spec *test = (struct test_spec *)arg; 1349 struct ifobject *ifobject = test->ifobj_rx; 1350 struct pollfd fds = { }; 1351 int err; 1352 1353 if (test->current_step == 1) { 1354 thread_common_ops(test, ifobject); 1355 } else { 1356 xsk_clear_xskmap(ifobject->xskmap); 1357 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk); 1358 if (err) { 1359 printf("Error: Failed to update xskmap, error %s\n", strerror(-err)); 1360 exit_with_error(-err); 1361 } 1362 } 1363 1364 fds.fd = xsk_socket__fd(ifobject->xsk->xsk); 1365 fds.events = POLLIN; 1366 1367 pthread_barrier_wait(&barr); 1368 1369 err = receive_pkts(test, &fds); 1370 1371 if (!err && ifobject->validation_func) 1372 err = ifobject->validation_func(ifobject); 1373 if (err) { 1374 report_failure(test); 1375 pthread_mutex_lock(&pacing_mutex); 1376 pthread_cond_signal(&pacing_cond); 1377 pthread_mutex_unlock(&pacing_mutex); 1378 } 1379 1380 pthread_exit(NULL); 1381 } 1382 1383 static u64 ceil_u64(u64 a, u64 b) 1384 { 1385 return (a + b - 1) / b; 1386 } 1387 1388 static void testapp_clean_xsk_umem(struct ifobject *ifobj) 1389 { 1390 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; 1391 1392 if (ifobj->shared_umem) 1393 umem_sz *= 2; 1394 1395 umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; 1396 xsk_umem__delete(ifobj->umem->umem); 1397 munmap(ifobj->umem->buffer, umem_sz); 1398 } 1399 1400 static void handler(int signum) 1401 { 1402 pthread_exit(NULL); 1403 } 1404 1405 static bool xdp_prog_changed(struct test_spec *test, struct ifobject *ifobj) 1406 { 1407 return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; 1408 } 1409 1410 static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, 1411 struct bpf_map *xskmap, enum test_mode mode) 1412 { 1413 int err; 1414 1415 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); 1416 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); 1417 if (err) { 1418 printf("Error attaching XDP program\n"); 1419 exit_with_error(-err); 1420 } 1421 1422 if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) 1423 if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { 1424 ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); 1425 exit_with_error(EINVAL); 1426 } 1427 1428 ifobj->xdp_prog = xdp_prog; 1429 ifobj->xskmap = xskmap; 1430 ifobj->mode = mode; 1431 } 1432 1433 static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, 1434 struct ifobject *ifobj_tx) 1435 { 1436 if (xdp_prog_changed(test, ifobj_rx)) 1437 xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); 1438 1439 if (!ifobj_tx || ifobj_tx->shared_umem) 1440 return; 1441 1442 if (xdp_prog_changed(test, ifobj_tx)) 1443 xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); 1444 } 1445 1446 static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, 1447 struct ifobject *ifobj2) 1448 { 1449 pthread_t t0, t1; 1450 1451 if (ifobj2) 1452 if (pthread_barrier_init(&barr, NULL, 2)) 1453 exit_with_error(errno); 1454 1455 test->current_step++; 1456 pkt_stream_reset(ifobj1->pkt_stream); 1457 pkts_in_flight = 0; 1458 1459 signal(SIGUSR1, handler); 1460 /*Spawn RX thread */ 1461 pthread_create(&t0, NULL, ifobj1->func_ptr, test); 1462 1463 if (ifobj2) { 1464 pthread_barrier_wait(&barr); 1465 if (pthread_barrier_destroy(&barr)) 1466 exit_with_error(errno); 1467 1468 /*Spawn TX thread */ 1469 pthread_create(&t1, NULL, ifobj2->func_ptr, test); 1470 1471 pthread_join(t1, NULL); 1472 } 1473 1474 if (!ifobj2) 1475 pthread_kill(t0, SIGUSR1); 1476 else 1477 pthread_join(t0, NULL); 1478 1479 if (test->total_steps == test->current_step || test->fail) { 1480 if (ifobj2) 1481 xsk_socket__delete(ifobj2->xsk->xsk); 1482 xsk_socket__delete(ifobj1->xsk->xsk); 1483 testapp_clean_xsk_umem(ifobj1); 1484 if (ifobj2 && !ifobj2->shared_umem) 1485 testapp_clean_xsk_umem(ifobj2); 1486 } 1487 1488 return !!test->fail; 1489 } 1490 1491 static int testapp_validate_traffic(struct test_spec *test) 1492 { 1493 struct ifobject *ifobj_rx = test->ifobj_rx; 1494 struct ifobject *ifobj_tx = test->ifobj_tx; 1495 1496 xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); 1497 return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); 1498 } 1499 1500 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) 1501 { 1502 return __testapp_validate_traffic(test, ifobj, NULL); 1503 } 1504 1505 static void testapp_teardown(struct test_spec *test) 1506 { 1507 int i; 1508 1509 test_spec_set_name(test, "TEARDOWN"); 1510 for (i = 0; i < MAX_TEARDOWN_ITER; i++) { 1511 if (testapp_validate_traffic(test)) 1512 return; 1513 test_spec_reset(test); 1514 } 1515 } 1516 1517 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) 1518 { 1519 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; 1520 struct ifobject *tmp_ifobj = (*ifobj1); 1521 1522 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; 1523 (*ifobj2)->func_ptr = tmp_func_ptr; 1524 1525 *ifobj1 = *ifobj2; 1526 *ifobj2 = tmp_ifobj; 1527 } 1528 1529 static void testapp_bidi(struct test_spec *test) 1530 { 1531 test_spec_set_name(test, "BIDIRECTIONAL"); 1532 test->ifobj_tx->rx_on = true; 1533 test->ifobj_rx->tx_on = true; 1534 test->total_steps = 2; 1535 if (testapp_validate_traffic(test)) 1536 return; 1537 1538 print_verbose("Switching Tx/Rx vectors\n"); 1539 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1540 __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); 1541 1542 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1543 } 1544 1545 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx) 1546 { 1547 int ret; 1548 1549 xsk_socket__delete(ifobj_tx->xsk->xsk); 1550 xsk_socket__delete(ifobj_rx->xsk->xsk); 1551 ifobj_tx->xsk = &ifobj_tx->xsk_arr[1]; 1552 ifobj_rx->xsk = &ifobj_rx->xsk_arr[1]; 1553 1554 ret = xsk_update_xskmap(ifobj_rx->xskmap, ifobj_rx->xsk->xsk); 1555 if (ret) 1556 exit_with_error(errno); 1557 } 1558 1559 static void testapp_bpf_res(struct test_spec *test) 1560 { 1561 test_spec_set_name(test, "BPF_RES"); 1562 test->total_steps = 2; 1563 test->nb_sockets = 2; 1564 if (testapp_validate_traffic(test)) 1565 return; 1566 1567 swap_xsk_resources(test->ifobj_tx, test->ifobj_rx); 1568 testapp_validate_traffic(test); 1569 } 1570 1571 static void testapp_headroom(struct test_spec *test) 1572 { 1573 test_spec_set_name(test, "UMEM_HEADROOM"); 1574 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; 1575 testapp_validate_traffic(test); 1576 } 1577 1578 static void testapp_stats_rx_dropped(struct test_spec *test) 1579 { 1580 test_spec_set_name(test, "STAT_RX_DROPPED"); 1581 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); 1582 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - 1583 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; 1584 pkt_stream_receive_half(test); 1585 test->ifobj_rx->validation_func = validate_rx_dropped; 1586 testapp_validate_traffic(test); 1587 } 1588 1589 static void testapp_stats_tx_invalid_descs(struct test_spec *test) 1590 { 1591 test_spec_set_name(test, "STAT_TX_INVALID"); 1592 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); 1593 test->ifobj_tx->validation_func = validate_tx_invalid_descs; 1594 testapp_validate_traffic(test); 1595 } 1596 1597 static void testapp_stats_rx_full(struct test_spec *test) 1598 { 1599 test_spec_set_name(test, "STAT_RX_FULL"); 1600 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE); 1601 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1602 DEFAULT_UMEM_BUFFERS, PKT_SIZE); 1603 if (!test->ifobj_rx->pkt_stream) 1604 exit_with_error(ENOMEM); 1605 1606 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; 1607 test->ifobj_rx->release_rx = false; 1608 test->ifobj_rx->validation_func = validate_rx_full; 1609 testapp_validate_traffic(test); 1610 } 1611 1612 static void testapp_stats_fill_empty(struct test_spec *test) 1613 { 1614 test_spec_set_name(test, "STAT_RX_FILL_EMPTY"); 1615 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE); 1616 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1617 DEFAULT_UMEM_BUFFERS, PKT_SIZE); 1618 if (!test->ifobj_rx->pkt_stream) 1619 exit_with_error(ENOMEM); 1620 1621 test->ifobj_rx->use_fill_ring = false; 1622 test->ifobj_rx->validation_func = validate_fill_empty; 1623 testapp_validate_traffic(test); 1624 } 1625 1626 /* Simple test */ 1627 static bool hugepages_present(struct ifobject *ifobject) 1628 { 1629 size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size; 1630 void *bufs; 1631 1632 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 1633 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, -1, 0); 1634 if (bufs == MAP_FAILED) 1635 return false; 1636 1637 mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; 1638 munmap(bufs, mmap_sz); 1639 return true; 1640 } 1641 1642 static bool testapp_unaligned(struct test_spec *test) 1643 { 1644 if (!hugepages_present(test->ifobj_tx)) { 1645 ksft_test_result_skip("No 2M huge pages present.\n"); 1646 return false; 1647 } 1648 1649 test_spec_set_name(test, "UNALIGNED_MODE"); 1650 test->ifobj_tx->umem->unaligned_mode = true; 1651 test->ifobj_rx->umem->unaligned_mode = true; 1652 /* Let half of the packets straddle a buffer boundrary */ 1653 pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2); 1654 test->ifobj_rx->pkt_stream->use_addr_for_fill = true; 1655 testapp_validate_traffic(test); 1656 1657 return true; 1658 } 1659 1660 static void testapp_single_pkt(struct test_spec *test) 1661 { 1662 struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}}; 1663 1664 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 1665 testapp_validate_traffic(test); 1666 } 1667 1668 static void testapp_invalid_desc(struct test_spec *test) 1669 { 1670 u64 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; 1671 struct pkt pkts[] = { 1672 /* Zero packet address allowed */ 1673 {0, PKT_SIZE, 0, true}, 1674 /* Allowed packet */ 1675 {0x1000, PKT_SIZE, 0, true}, 1676 /* Straddling the start of umem */ 1677 {-2, PKT_SIZE, 0, false}, 1678 /* Packet too large */ 1679 {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, 1680 /* Up to end of umem allowed */ 1681 {umem_size - PKT_SIZE, PKT_SIZE, 0, true}, 1682 /* After umem ends */ 1683 {umem_size, PKT_SIZE, 0, false}, 1684 /* Straddle the end of umem */ 1685 {umem_size - PKT_SIZE / 2, PKT_SIZE, 0, false}, 1686 /* Straddle a page boundrary */ 1687 {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false}, 1688 /* Straddle a 2K boundrary */ 1689 {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true}, 1690 /* Valid packet for synch so that something is received */ 1691 {0x4000, PKT_SIZE, 0, true}}; 1692 1693 if (test->ifobj_tx->umem->unaligned_mode) { 1694 /* Crossing a page boundrary allowed */ 1695 pkts[7].valid = true; 1696 } 1697 if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { 1698 /* Crossing a 2K frame size boundrary not allowed */ 1699 pkts[8].valid = false; 1700 } 1701 1702 if (test->ifobj_tx->shared_umem) { 1703 pkts[4].addr += umem_size; 1704 pkts[5].addr += umem_size; 1705 pkts[6].addr += umem_size; 1706 } 1707 1708 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 1709 testapp_validate_traffic(test); 1710 } 1711 1712 static void testapp_xdp_drop(struct test_spec *test) 1713 { 1714 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 1715 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 1716 1717 test_spec_set_name(test, "XDP_DROP_HALF"); 1718 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, 1719 skel_rx->maps.xsk, skel_tx->maps.xsk); 1720 1721 pkt_stream_receive_half(test); 1722 testapp_validate_traffic(test); 1723 } 1724 1725 static void testapp_xdp_metadata_count(struct test_spec *test) 1726 { 1727 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 1728 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 1729 struct bpf_map *data_map; 1730 int count = 0; 1731 int key = 0; 1732 1733 test_spec_set_name(test, "XDP_METADATA_COUNT"); 1734 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, 1735 skel_tx->progs.xsk_xdp_populate_metadata, 1736 skel_rx->maps.xsk, skel_tx->maps.xsk); 1737 test->ifobj_rx->use_metadata = true; 1738 1739 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); 1740 if (!data_map || !bpf_map__is_internal(data_map)) 1741 exit_with_error(ENOMEM); 1742 1743 if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) 1744 exit_with_error(errno); 1745 1746 testapp_validate_traffic(test); 1747 } 1748 1749 static void testapp_poll_txq_tmout(struct test_spec *test) 1750 { 1751 test_spec_set_name(test, "POLL_TXQ_FULL"); 1752 1753 test->ifobj_tx->use_poll = true; 1754 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ 1755 test->ifobj_tx->umem->frame_size = 2048; 1756 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); 1757 testapp_validate_traffic_single_thread(test, test->ifobj_tx); 1758 } 1759 1760 static void testapp_poll_rxq_tmout(struct test_spec *test) 1761 { 1762 test_spec_set_name(test, "POLL_RXQ_EMPTY"); 1763 test->ifobj_rx->use_poll = true; 1764 testapp_validate_traffic_single_thread(test, test->ifobj_rx); 1765 } 1766 1767 static int xsk_load_xdp_programs(struct ifobject *ifobj) 1768 { 1769 ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); 1770 if (libbpf_get_error(ifobj->xdp_progs)) 1771 return libbpf_get_error(ifobj->xdp_progs); 1772 1773 return 0; 1774 } 1775 1776 static void xsk_unload_xdp_programs(struct ifobject *ifobj) 1777 { 1778 xsk_xdp_progs__destroy(ifobj->xdp_progs); 1779 } 1780 1781 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac, 1782 const char *dst_ip, const char *src_ip, const u16 dst_port, 1783 const u16 src_port, thread_func_t func_ptr) 1784 { 1785 struct in_addr ip; 1786 int err; 1787 1788 memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN); 1789 memcpy(ifobj->src_mac, src_mac, ETH_ALEN); 1790 1791 inet_aton(dst_ip, &ip); 1792 ifobj->dst_ip = ip.s_addr; 1793 1794 inet_aton(src_ip, &ip); 1795 ifobj->src_ip = ip.s_addr; 1796 1797 ifobj->dst_port = dst_port; 1798 ifobj->src_port = src_port; 1799 1800 ifobj->func_ptr = func_ptr; 1801 1802 err = xsk_load_xdp_programs(ifobj); 1803 if (err) { 1804 printf("Error loading XDP program\n"); 1805 exit_with_error(err); 1806 } 1807 } 1808 1809 static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type) 1810 { 1811 switch (type) { 1812 case TEST_TYPE_STATS_RX_DROPPED: 1813 if (mode == TEST_MODE_ZC) { 1814 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); 1815 return; 1816 } 1817 testapp_stats_rx_dropped(test); 1818 break; 1819 case TEST_TYPE_STATS_TX_INVALID_DESCS: 1820 testapp_stats_tx_invalid_descs(test); 1821 break; 1822 case TEST_TYPE_STATS_RX_FULL: 1823 testapp_stats_rx_full(test); 1824 break; 1825 case TEST_TYPE_STATS_FILL_EMPTY: 1826 testapp_stats_fill_empty(test); 1827 break; 1828 case TEST_TYPE_TEARDOWN: 1829 testapp_teardown(test); 1830 break; 1831 case TEST_TYPE_BIDI: 1832 testapp_bidi(test); 1833 break; 1834 case TEST_TYPE_BPF_RES: 1835 testapp_bpf_res(test); 1836 break; 1837 case TEST_TYPE_RUN_TO_COMPLETION: 1838 test_spec_set_name(test, "RUN_TO_COMPLETION"); 1839 testapp_validate_traffic(test); 1840 break; 1841 case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT: 1842 test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT"); 1843 testapp_single_pkt(test); 1844 break; 1845 case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME: 1846 test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE"); 1847 test->ifobj_tx->umem->frame_size = 2048; 1848 test->ifobj_rx->umem->frame_size = 2048; 1849 pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE); 1850 testapp_validate_traffic(test); 1851 break; 1852 case TEST_TYPE_RX_POLL: 1853 test->ifobj_rx->use_poll = true; 1854 test_spec_set_name(test, "POLL_RX"); 1855 testapp_validate_traffic(test); 1856 break; 1857 case TEST_TYPE_TX_POLL: 1858 test->ifobj_tx->use_poll = true; 1859 test_spec_set_name(test, "POLL_TX"); 1860 testapp_validate_traffic(test); 1861 break; 1862 case TEST_TYPE_POLL_TXQ_TMOUT: 1863 testapp_poll_txq_tmout(test); 1864 break; 1865 case TEST_TYPE_POLL_RXQ_TMOUT: 1866 testapp_poll_rxq_tmout(test); 1867 break; 1868 case TEST_TYPE_ALIGNED_INV_DESC: 1869 test_spec_set_name(test, "ALIGNED_INV_DESC"); 1870 testapp_invalid_desc(test); 1871 break; 1872 case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME: 1873 test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE"); 1874 test->ifobj_tx->umem->frame_size = 2048; 1875 test->ifobj_rx->umem->frame_size = 2048; 1876 testapp_invalid_desc(test); 1877 break; 1878 case TEST_TYPE_UNALIGNED_INV_DESC: 1879 if (!hugepages_present(test->ifobj_tx)) { 1880 ksft_test_result_skip("No 2M huge pages present.\n"); 1881 return; 1882 } 1883 test_spec_set_name(test, "UNALIGNED_INV_DESC"); 1884 test->ifobj_tx->umem->unaligned_mode = true; 1885 test->ifobj_rx->umem->unaligned_mode = true; 1886 testapp_invalid_desc(test); 1887 break; 1888 case TEST_TYPE_UNALIGNED_INV_DESC_4K1_FRAME: { 1889 u64 page_size, umem_size; 1890 1891 if (!hugepages_present(test->ifobj_tx)) { 1892 ksft_test_result_skip("No 2M huge pages present.\n"); 1893 return; 1894 } 1895 test_spec_set_name(test, "UNALIGNED_INV_DESC_4K1_FRAME_SIZE"); 1896 /* Odd frame size so the UMEM doesn't end near a page boundary. */ 1897 test->ifobj_tx->umem->frame_size = 4001; 1898 test->ifobj_rx->umem->frame_size = 4001; 1899 test->ifobj_tx->umem->unaligned_mode = true; 1900 test->ifobj_rx->umem->unaligned_mode = true; 1901 /* This test exists to test descriptors that staddle the end of 1902 * the UMEM but not a page. 1903 */ 1904 page_size = sysconf(_SC_PAGESIZE); 1905 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; 1906 assert(umem_size % page_size > PKT_SIZE); 1907 assert(umem_size % page_size < page_size - PKT_SIZE); 1908 testapp_invalid_desc(test); 1909 break; 1910 } 1911 case TEST_TYPE_UNALIGNED: 1912 if (!testapp_unaligned(test)) 1913 return; 1914 break; 1915 case TEST_TYPE_HEADROOM: 1916 testapp_headroom(test); 1917 break; 1918 case TEST_TYPE_XDP_DROP_HALF: 1919 testapp_xdp_drop(test); 1920 break; 1921 case TEST_TYPE_XDP_METADATA_COUNT: 1922 testapp_xdp_metadata_count(test); 1923 break; 1924 default: 1925 break; 1926 } 1927 1928 if (!test->fail) 1929 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), 1930 test->name); 1931 pkt_stream_restore_default(test); 1932 } 1933 1934 static struct ifobject *ifobject_create(void) 1935 { 1936 struct ifobject *ifobj; 1937 1938 ifobj = calloc(1, sizeof(struct ifobject)); 1939 if (!ifobj) 1940 return NULL; 1941 1942 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); 1943 if (!ifobj->xsk_arr) 1944 goto out_xsk_arr; 1945 1946 ifobj->umem = calloc(1, sizeof(*ifobj->umem)); 1947 if (!ifobj->umem) 1948 goto out_umem; 1949 1950 return ifobj; 1951 1952 out_umem: 1953 free(ifobj->xsk_arr); 1954 out_xsk_arr: 1955 free(ifobj); 1956 return NULL; 1957 } 1958 1959 static void ifobject_delete(struct ifobject *ifobj) 1960 { 1961 free(ifobj->umem); 1962 free(ifobj->xsk_arr); 1963 free(ifobj); 1964 } 1965 1966 static bool is_xdp_supported(int ifindex) 1967 { 1968 int flags = XDP_FLAGS_DRV_MODE; 1969 1970 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); 1971 struct bpf_insn insns[2] = { 1972 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), 1973 BPF_EXIT_INSN() 1974 }; 1975 int prog_fd, insn_cnt = ARRAY_SIZE(insns); 1976 int err; 1977 1978 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); 1979 if (prog_fd < 0) 1980 return false; 1981 1982 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); 1983 if (err) { 1984 close(prog_fd); 1985 return false; 1986 } 1987 1988 bpf_xdp_detach(ifindex, flags, NULL); 1989 close(prog_fd); 1990 1991 return true; 1992 } 1993 1994 int main(int argc, char **argv) 1995 { 1996 struct pkt_stream *rx_pkt_stream_default; 1997 struct pkt_stream *tx_pkt_stream_default; 1998 struct ifobject *ifobj_tx, *ifobj_rx; 1999 int modes = TEST_MODE_SKB + 1; 2000 u32 i, j, failed_tests = 0; 2001 struct test_spec test; 2002 bool shared_netdev; 2003 2004 /* Use libbpf 1.0 API mode */ 2005 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2006 2007 ifobj_tx = ifobject_create(); 2008 if (!ifobj_tx) 2009 exit_with_error(ENOMEM); 2010 ifobj_rx = ifobject_create(); 2011 if (!ifobj_rx) 2012 exit_with_error(ENOMEM); 2013 2014 setlocale(LC_ALL, ""); 2015 2016 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 2017 2018 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); 2019 ifobj_tx->shared_umem = shared_netdev; 2020 ifobj_rx->shared_umem = shared_netdev; 2021 2022 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) { 2023 usage(basename(argv[0])); 2024 ksft_exit_xfail(); 2025 } 2026 2027 if (is_xdp_supported(ifobj_tx->ifindex)) { 2028 modes++; 2029 if (ifobj_zc_avail(ifobj_tx)) 2030 modes++; 2031 } 2032 2033 init_iface(ifobj_rx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2, 2034 worker_testapp_validate_rx); 2035 init_iface(ifobj_tx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1, 2036 worker_testapp_validate_tx); 2037 2038 test_spec_init(&test, ifobj_tx, ifobj_rx, 0); 2039 tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE); 2040 rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE); 2041 if (!tx_pkt_stream_default || !rx_pkt_stream_default) 2042 exit_with_error(ENOMEM); 2043 test.tx_pkt_stream_default = tx_pkt_stream_default; 2044 test.rx_pkt_stream_default = rx_pkt_stream_default; 2045 2046 ksft_set_plan(modes * TEST_TYPE_MAX); 2047 2048 for (i = 0; i < modes; i++) { 2049 for (j = 0; j < TEST_TYPE_MAX; j++) { 2050 test_spec_init(&test, ifobj_tx, ifobj_rx, i); 2051 run_pkt_test(&test, i, j); 2052 usleep(USLEEP_MAX); 2053 2054 if (test.fail) 2055 failed_tests++; 2056 } 2057 } 2058 2059 pkt_stream_delete(tx_pkt_stream_default); 2060 pkt_stream_delete(rx_pkt_stream_default); 2061 xsk_unload_xdp_programs(ifobj_tx); 2062 xsk_unload_xdp_programs(ifobj_rx); 2063 ifobject_delete(ifobj_tx); 2064 ifobject_delete(ifobj_rx); 2065 2066 if (failed_tests) 2067 ksft_exit_fail(); 2068 else 2069 ksft_exit_pass(); 2070 } 2071