1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #if HAVE_CONFIG_H 34 # include <config.h> 35 #endif /* HAVE_CONFIG_H */ 36 37 #include <stdio.h> 38 #include <stdlib.h> 39 #include <unistd.h> 40 #include <string.h> 41 #include <sys/types.h> 42 #include <sys/socket.h> 43 #include <sys/time.h> 44 #include <netdb.h> 45 #include <getopt.h> 46 #include <arpa/inet.h> 47 #include <time.h> 48 49 #include "pingpong.h" 50 51 enum { 52 PINGPONG_RECV_WRID = 1, 53 PINGPONG_SEND_WRID = 2, 54 }; 55 56 static int page_size; 57 58 struct pingpong_context { 59 struct ibv_context *context; 60 struct ibv_comp_channel *channel; 61 struct ibv_pd *pd; 62 struct ibv_mr *mr; 63 struct ibv_cq *cq; 64 struct ibv_qp *qp; 65 void *buf; 66 int size; 67 int rx_depth; 68 int pending; 69 struct ibv_port_attr portinfo; 70 }; 71 72 struct pingpong_dest { 73 int lid; 74 int qpn; 75 int psn; 76 union ibv_gid gid; 77 }; 78 79 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 80 enum ibv_mtu mtu, int sl, 81 struct pingpong_dest *dest, int sgid_idx) 82 { 83 struct ibv_qp_attr attr = { 84 .qp_state = IBV_QPS_RTR, 85 .path_mtu = mtu, 86 .dest_qp_num = dest->qpn, 87 .rq_psn = dest->psn, 88 .ah_attr = { 89 .is_global = 0, 90 .dlid = dest->lid, 91 .sl = sl, 92 .src_path_bits = 0, 93 .port_num = port 94 } 95 }; 96 97 if (dest->gid.global.interface_id) { 98 attr.ah_attr.is_global = 1; 99 attr.ah_attr.grh.hop_limit = 1; 100 attr.ah_attr.grh.dgid = dest->gid; 101 attr.ah_attr.grh.sgid_index = sgid_idx; 102 } 103 104 if (ibv_modify_qp(ctx->qp, &attr, 105 IBV_QP_STATE | 106 IBV_QP_AV | 107 IBV_QP_PATH_MTU | 108 IBV_QP_DEST_QPN | 109 IBV_QP_RQ_PSN)) { 110 fprintf(stderr, "Failed to modify QP to RTR\n"); 111 return 1; 112 } 113 114 attr.qp_state = IBV_QPS_RTS; 115 attr.sq_psn = my_psn; 116 if (ibv_modify_qp(ctx->qp, &attr, 117 IBV_QP_STATE | 118 IBV_QP_SQ_PSN)) { 119 fprintf(stderr, "Failed to modify QP to RTS\n"); 120 return 1; 121 } 122 123 return 0; 124 } 125 126 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, 127 const struct pingpong_dest *my_dest) 128 { 129 struct addrinfo *res, *t; 130 struct addrinfo hints = { 131 .ai_family = AF_INET, 132 .ai_socktype = SOCK_STREAM 133 }; 134 char *service; 135 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 136 int n; 137 int sockfd = -1; 138 struct pingpong_dest *rem_dest = NULL; 139 char gid[33]; 140 141 if (asprintf(&service, "%d", port) < 0) 142 return NULL; 143 144 n = getaddrinfo(servername, service, &hints, &res); 145 146 if (n < 0) { 147 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); 148 free(service); 149 return NULL; 150 } 151 152 for (t = res; t; t = t->ai_next) { 153 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 154 if (sockfd >= 0) { 155 if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) 156 break; 157 close(sockfd); 158 sockfd = -1; 159 } 160 } 161 162 freeaddrinfo(res); 163 free(service); 164 165 if (sockfd < 0) { 166 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); 167 return NULL; 168 } 169 170 gid_to_wire_gid(&my_dest->gid, gid); 171 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 172 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 173 fprintf(stderr, "Couldn't send local address\n"); 174 goto out; 175 } 176 177 if (read(sockfd, msg, sizeof msg) != sizeof msg) { 178 perror("client read"); 179 fprintf(stderr, "Couldn't read remote address\n"); 180 goto out; 181 } 182 183 write(sockfd, "done", sizeof "done"); 184 185 rem_dest = malloc(sizeof *rem_dest); 186 if (!rem_dest) 187 goto out; 188 189 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 190 wire_gid_to_gid(gid, &rem_dest->gid); 191 192 out: 193 close(sockfd); 194 return rem_dest; 195 } 196 197 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 198 int ib_port, enum ibv_mtu mtu, 199 int port, int sl, 200 const struct pingpong_dest *my_dest, 201 int sgid_idx) 202 { 203 struct addrinfo *res, *t; 204 struct addrinfo hints = { 205 .ai_flags = AI_PASSIVE, 206 .ai_family = AF_INET, 207 .ai_socktype = SOCK_STREAM 208 }; 209 char *service; 210 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 211 int n; 212 int sockfd = -1, connfd; 213 struct pingpong_dest *rem_dest = NULL; 214 char gid[33]; 215 216 if (asprintf(&service, "%d", port) < 0) 217 return NULL; 218 219 n = getaddrinfo(NULL, service, &hints, &res); 220 221 if (n < 0) { 222 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); 223 free(service); 224 return NULL; 225 } 226 227 for (t = res; t; t = t->ai_next) { 228 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 229 if (sockfd >= 0) { 230 n = 1; 231 232 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); 233 234 if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) 235 break; 236 close(sockfd); 237 sockfd = -1; 238 } 239 } 240 241 freeaddrinfo(res); 242 free(service); 243 244 if (sockfd < 0) { 245 fprintf(stderr, "Couldn't listen to port %d\n", port); 246 return NULL; 247 } 248 249 listen(sockfd, 1); 250 connfd = accept(sockfd, NULL, 0); 251 close(sockfd); 252 if (connfd < 0) { 253 fprintf(stderr, "accept() failed\n"); 254 return NULL; 255 } 256 257 n = read(connfd, msg, sizeof msg); 258 if (n != sizeof msg) { 259 perror("server read"); 260 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); 261 goto out; 262 } 263 264 rem_dest = malloc(sizeof *rem_dest); 265 if (!rem_dest) 266 goto out; 267 268 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 269 wire_gid_to_gid(gid, &rem_dest->gid); 270 271 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { 272 fprintf(stderr, "Couldn't connect to remote QP\n"); 273 free(rem_dest); 274 rem_dest = NULL; 275 goto out; 276 } 277 278 gid_to_wire_gid(&my_dest->gid, gid); 279 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 280 if (write(connfd, msg, sizeof msg) != sizeof msg) { 281 fprintf(stderr, "Couldn't send local address\n"); 282 free(rem_dest); 283 rem_dest = NULL; 284 goto out; 285 } 286 287 read(connfd, msg, sizeof msg); 288 289 out: 290 close(connfd); 291 return rem_dest; 292 } 293 294 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 295 int rx_depth, int port, 296 int use_event) 297 { 298 struct pingpong_context *ctx; 299 300 ctx = calloc(1, sizeof *ctx); 301 if (!ctx) 302 return NULL; 303 304 ctx->size = size; 305 ctx->rx_depth = rx_depth; 306 307 ctx->buf = malloc(roundup(size, page_size)); 308 if (!ctx->buf) { 309 fprintf(stderr, "Couldn't allocate work buf.\n"); 310 return NULL; 311 } 312 313 memset(ctx->buf, 0, size); 314 315 ctx->context = ibv_open_device(ib_dev); 316 if (!ctx->context) { 317 fprintf(stderr, "Couldn't get context for %s\n", 318 ibv_get_device_name(ib_dev)); 319 return NULL; 320 } 321 322 if (use_event) { 323 ctx->channel = ibv_create_comp_channel(ctx->context); 324 if (!ctx->channel) { 325 fprintf(stderr, "Couldn't create completion channel\n"); 326 return NULL; 327 } 328 } else 329 ctx->channel = NULL; 330 331 ctx->pd = ibv_alloc_pd(ctx->context); 332 if (!ctx->pd) { 333 fprintf(stderr, "Couldn't allocate PD\n"); 334 return NULL; 335 } 336 337 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE); 338 if (!ctx->mr) { 339 fprintf(stderr, "Couldn't register MR\n"); 340 return NULL; 341 } 342 343 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, 344 ctx->channel, 0); 345 if (!ctx->cq) { 346 fprintf(stderr, "Couldn't create CQ\n"); 347 return NULL; 348 } 349 350 { 351 struct ibv_qp_init_attr attr = { 352 .send_cq = ctx->cq, 353 .recv_cq = ctx->cq, 354 .cap = { 355 .max_send_wr = 1, 356 .max_recv_wr = rx_depth, 357 .max_send_sge = 1, 358 .max_recv_sge = 1 359 }, 360 .qp_type = IBV_QPT_UC 361 }; 362 363 ctx->qp = ibv_create_qp(ctx->pd, &attr); 364 if (!ctx->qp) { 365 fprintf(stderr, "Couldn't create QP\n"); 366 return NULL; 367 } 368 } 369 370 { 371 struct ibv_qp_attr attr = { 372 .qp_state = IBV_QPS_INIT, 373 .pkey_index = 0, 374 .port_num = port, 375 .qp_access_flags = 0 376 }; 377 378 if (ibv_modify_qp(ctx->qp, &attr, 379 IBV_QP_STATE | 380 IBV_QP_PKEY_INDEX | 381 IBV_QP_PORT | 382 IBV_QP_ACCESS_FLAGS)) { 383 fprintf(stderr, "Failed to modify QP to INIT\n"); 384 return NULL; 385 } 386 } 387 388 return ctx; 389 } 390 391 int pp_close_ctx(struct pingpong_context *ctx) 392 { 393 if (ibv_destroy_qp(ctx->qp)) { 394 fprintf(stderr, "Couldn't destroy QP\n"); 395 return 1; 396 } 397 398 if (ibv_destroy_cq(ctx->cq)) { 399 fprintf(stderr, "Couldn't destroy CQ\n"); 400 return 1; 401 } 402 403 if (ibv_dereg_mr(ctx->mr)) { 404 fprintf(stderr, "Couldn't deregister MR\n"); 405 return 1; 406 } 407 408 if (ibv_dealloc_pd(ctx->pd)) { 409 fprintf(stderr, "Couldn't deallocate PD\n"); 410 return 1; 411 } 412 413 if (ctx->channel) { 414 if (ibv_destroy_comp_channel(ctx->channel)) { 415 fprintf(stderr, "Couldn't destroy completion channel\n"); 416 return 1; 417 } 418 } 419 420 if (ibv_close_device(ctx->context)) { 421 fprintf(stderr, "Couldn't release context\n"); 422 return 1; 423 } 424 425 free(ctx->buf); 426 free(ctx); 427 428 return 0; 429 } 430 431 static int pp_post_recv(struct pingpong_context *ctx, int n) 432 { 433 struct ibv_sge list = { 434 .addr = (uintptr_t) ctx->buf, 435 .length = ctx->size, 436 .lkey = ctx->mr->lkey 437 }; 438 struct ibv_recv_wr wr = { 439 .wr_id = PINGPONG_RECV_WRID, 440 .sg_list = &list, 441 .num_sge = 1, 442 }; 443 struct ibv_recv_wr *bad_wr; 444 int i; 445 446 for (i = 0; i < n; ++i) 447 if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) 448 break; 449 450 return i; 451 } 452 453 static int pp_post_send(struct pingpong_context *ctx) 454 { 455 struct ibv_sge list = { 456 .addr = (uintptr_t) ctx->buf, 457 .length = ctx->size, 458 .lkey = ctx->mr->lkey 459 }; 460 struct ibv_send_wr wr = { 461 .wr_id = PINGPONG_SEND_WRID, 462 .sg_list = &list, 463 .num_sge = 1, 464 .opcode = IBV_WR_SEND, 465 .send_flags = IBV_SEND_SIGNALED, 466 }; 467 struct ibv_send_wr *bad_wr; 468 469 return ibv_post_send(ctx->qp, &wr, &bad_wr); 470 } 471 472 static void usage(const char *argv0) 473 { 474 printf("Usage:\n"); 475 printf(" %s start a server and wait for connection\n", argv0); 476 printf(" %s <host> connect to server at <host>\n", argv0); 477 printf("\n"); 478 printf("Options:\n"); 479 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 480 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 481 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 482 printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); 483 printf(" -m, --mtu=<size> path MTU (default 1024)\n"); 484 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 485 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 486 printf(" -l, --sl=<sl> service level value\n"); 487 printf(" -e, --events sleep on CQ events (default poll)\n"); 488 printf(" -g, --gid-idx=<gid index> local port gid index\n"); 489 } 490 491 int main(int argc, char *argv[]) 492 { 493 struct ibv_device **dev_list; 494 struct ibv_device *ib_dev; 495 struct pingpong_context *ctx; 496 struct pingpong_dest my_dest; 497 struct pingpong_dest *rem_dest; 498 struct timeval start, end; 499 char *ib_devname = NULL; 500 char *servername = NULL; 501 int port = 18515; 502 int ib_port = 1; 503 int size = 4096; 504 enum ibv_mtu mtu = IBV_MTU_1024; 505 int rx_depth = 500; 506 int iters = 1000; 507 int use_event = 0; 508 int routs; 509 int rcnt, scnt; 510 int num_cq_events = 0; 511 int sl = 0; 512 int gidx = -1; 513 char gid[33]; 514 515 srand48(getpid() * time(NULL)); 516 517 while (1) { 518 int c; 519 520 static struct option long_options[] = { 521 { .name = "port", .has_arg = 1, .val = 'p' }, 522 { .name = "ib-dev", .has_arg = 1, .val = 'd' }, 523 { .name = "ib-port", .has_arg = 1, .val = 'i' }, 524 { .name = "size", .has_arg = 1, .val = 's' }, 525 { .name = "mtu", .has_arg = 1, .val = 'm' }, 526 { .name = "rx-depth", .has_arg = 1, .val = 'r' }, 527 { .name = "iters", .has_arg = 1, .val = 'n' }, 528 { .name = "sl", .has_arg = 1, .val = 'l' }, 529 { .name = "events", .has_arg = 0, .val = 'e' }, 530 { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 531 { 0 } 532 }; 533 534 c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); 535 if (c == -1) 536 break; 537 538 switch (c) { 539 case 'p': 540 port = strtol(optarg, NULL, 0); 541 if (port < 0 || port > 65535) { 542 usage(argv[0]); 543 return 1; 544 } 545 break; 546 547 case 'd': 548 ib_devname = strdup(optarg); 549 break; 550 551 case 'i': 552 ib_port = strtol(optarg, NULL, 0); 553 if (ib_port < 0) { 554 usage(argv[0]); 555 return 1; 556 } 557 break; 558 559 case 's': 560 size = strtol(optarg, NULL, 0); 561 break; 562 563 case 'm': 564 mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); 565 if (mtu < 0) { 566 usage(argv[0]); 567 return 1; 568 } 569 break; 570 571 case 'r': 572 rx_depth = strtol(optarg, NULL, 0); 573 break; 574 575 case 'n': 576 iters = strtol(optarg, NULL, 0); 577 break; 578 579 case 'l': 580 sl = strtol(optarg, NULL, 0); 581 break; 582 583 case 'e': 584 ++use_event; 585 break; 586 587 case 'g': 588 gidx = strtol(optarg, NULL, 0); 589 break; 590 591 default: 592 usage(argv[0]); 593 return 1; 594 } 595 } 596 597 if (optind == argc - 1) 598 servername = strdup(argv[optind]); 599 else if (optind < argc) { 600 usage(argv[0]); 601 return 1; 602 } 603 604 page_size = sysconf(_SC_PAGESIZE); 605 606 dev_list = ibv_get_device_list(NULL); 607 if (!dev_list) { 608 perror("Failed to get IB devices list"); 609 return 1; 610 } 611 612 if (!ib_devname) { 613 ib_dev = *dev_list; 614 if (!ib_dev) { 615 fprintf(stderr, "No IB devices found\n"); 616 return 1; 617 } 618 } else { 619 int i; 620 for (i = 0; dev_list[i]; ++i) 621 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) 622 break; 623 ib_dev = dev_list[i]; 624 if (!ib_dev) { 625 fprintf(stderr, "IB device %s not found\n", ib_devname); 626 return 1; 627 } 628 } 629 630 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 631 if (!ctx) 632 return 1; 633 634 routs = pp_post_recv(ctx, ctx->rx_depth); 635 if (routs < ctx->rx_depth) { 636 fprintf(stderr, "Couldn't post receive (%d)\n", routs); 637 return 1; 638 } 639 640 if (use_event) 641 if (ibv_req_notify_cq(ctx->cq, 0)) { 642 fprintf(stderr, "Couldn't request CQ notification\n"); 643 return 1; 644 } 645 646 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 647 fprintf(stderr, "Couldn't get port info\n"); 648 return 1; 649 } 650 651 my_dest.lid = ctx->portinfo.lid; 652 if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { 653 fprintf(stderr, "Couldn't get local LID\n"); 654 return 1; 655 } 656 657 if (gidx >= 0) { 658 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 659 fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 660 return 1; 661 } 662 } else 663 memset(&my_dest.gid, 0, sizeof my_dest.gid); 664 665 my_dest.qpn = ctx->qp->qp_num; 666 my_dest.psn = lrand48() & 0xffffff; 667 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 668 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 669 my_dest.lid, my_dest.qpn, my_dest.psn, gid); 670 671 if (servername) 672 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 673 else 674 rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); 675 676 if (!rem_dest) 677 return 1; 678 679 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 680 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 681 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 682 683 if (servername) 684 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) 685 return 1; 686 687 ctx->pending = PINGPONG_RECV_WRID; 688 689 if (servername) { 690 if (pp_post_send(ctx)) { 691 fprintf(stderr, "Couldn't post send\n"); 692 return 1; 693 } 694 ctx->pending |= PINGPONG_SEND_WRID; 695 } 696 697 if (gettimeofday(&start, NULL)) { 698 perror("gettimeofday"); 699 return 1; 700 } 701 702 rcnt = scnt = 0; 703 while (rcnt < iters || scnt < iters) { 704 if (use_event) { 705 struct ibv_cq *ev_cq; 706 void *ev_ctx; 707 708 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { 709 fprintf(stderr, "Failed to get cq_event\n"); 710 return 1; 711 } 712 713 ++num_cq_events; 714 715 if (ev_cq != ctx->cq) { 716 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); 717 return 1; 718 } 719 720 if (ibv_req_notify_cq(ctx->cq, 0)) { 721 fprintf(stderr, "Couldn't request CQ notification\n"); 722 return 1; 723 } 724 } 725 726 { 727 struct ibv_wc wc[2]; 728 int ne, i; 729 730 do { 731 ne = ibv_poll_cq(ctx->cq, 2, wc); 732 if (ne < 0) { 733 fprintf(stderr, "poll CQ failed %d\n", ne); 734 return 1; 735 } 736 } while (!use_event && ne < 1); 737 738 for (i = 0; i < ne; ++i) { 739 if (wc[i].status != IBV_WC_SUCCESS) { 740 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", 741 ibv_wc_status_str(wc[i].status), 742 wc[i].status, (int) wc[i].wr_id); 743 return 1; 744 } 745 746 switch ((int) wc[i].wr_id) { 747 case PINGPONG_SEND_WRID: 748 ++scnt; 749 break; 750 751 case PINGPONG_RECV_WRID: 752 if (--routs <= 1) { 753 routs += pp_post_recv(ctx, ctx->rx_depth - routs); 754 if (routs < ctx->rx_depth) { 755 fprintf(stderr, 756 "Couldn't post receive (%d)\n", 757 routs); 758 return 1; 759 } 760 } 761 762 ++rcnt; 763 break; 764 765 default: 766 fprintf(stderr, "Completion for unknown wr_id %d\n", 767 (int) wc[i].wr_id); 768 return 1; 769 } 770 771 ctx->pending &= ~(int) wc[i].wr_id; 772 if (scnt < iters && !ctx->pending) { 773 if (pp_post_send(ctx)) { 774 fprintf(stderr, "Couldn't post send\n"); 775 return 1; 776 } 777 ctx->pending = PINGPONG_RECV_WRID | 778 PINGPONG_SEND_WRID; 779 } 780 } 781 } 782 } 783 784 if (gettimeofday(&end, NULL)) { 785 perror("gettimeofday"); 786 return 1; 787 } 788 789 { 790 float usec = (end.tv_sec - start.tv_sec) * 1000000 + 791 (end.tv_usec - start.tv_usec); 792 long long bytes = (long long) size * iters * 2; 793 794 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", 795 bytes, usec / 1000000., bytes * 8. / usec); 796 printf("%d iters in %.2f seconds = %.2f usec/iter\n", 797 iters, usec / 1000000., usec / iters); 798 } 799 800 ibv_ack_cq_events(ctx->cq, num_cq_events); 801 802 if (pp_close_ctx(ctx)) 803 return 1; 804 805 ibv_free_device_list(dev_list); 806 free(rem_dest); 807 808 return 0; 809 } 810