1 // 2 // Copyright 2008, George V. Neville-Neil 3 // All rights reserved. 4 // 5 // 6 // Redistribution and use in source and binary forms, with or without 7 // modification, are permitted provided that the following conditions 8 // are met: 9 // 1. Redistributions of source code must retain the above copyright 10 // notice, this list of conditions and the following disclaimer. 11 // 2. Redistributions in binary form must reproduce the above copyright 12 // notice, this list of conditions and the following disclaimer in the 13 // documentation and/or other materials provided with the distribution. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 // OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 // SUCH DAMAGE. 26 // 27 // This is a relatively simple multicast test which can act as a 28 // source and sink. The purpose of this test is to determine the 29 // latency between two hosts, the source and the sink. The programs 30 // expect to be run somewhat unsynchronized hosts. The source and 31 // the sink both record the time on their own machine and then the 32 // sink will correlate the data at the end of the run. 33 // 34 35 #include <sys/cdefs.h> 36 // C++ STL and other related includes 37 #include <iostream> 38 #include <string> 39 #include <vector> 40 #include <algorithm> 41 42 // Operating System and other C based includes 43 #include <unistd.h> 44 #include <errno.h> 45 #include <sys/types.h> 46 #include <sys/time.h> 47 #include <sys/socket.h> 48 #include <sys/ioctl.h> 49 #include <netinet/in.h> 50 #include <net/if.h> 51 #include <arpa/inet.h> 52 53 // Private include files 54 #include "mctest.h" 55 56 using namespace std; 57 58 // 59 // usage - just the program's usage line 60 // 61 // 62 void usage() 63 { 64 cout << "mctest [-r] -M clients -m client number -i interface -g multicast group -s packet size -n number -t inter-packet gap\n"; 65 exit(-1); 66 } 67 68 // 69 // usage - print out the usage with a possible message and exit 70 // 71 // \param message optional string 72 // 73 // 74 void usage(string message) 75 { 76 77 cerr << message << endl; 78 usage(); 79 } 80 81 82 // 83 // absorb and record packets 84 // 85 // @param interface ///< text name of the interface (em0 etc.) 86 // @param group ///< multicast group 87 // @param pkt_size ///< packet Size 88 // @param number ///< number of packets we're expecting 89 // @param clients ///< total number of clients (N) 90 // @param client ///< our client number (0..N) 91 // 92 // @return 0 for 0K, -1 for error, sets errno 93 // 94 int sink(char *interface, struct in_addr *group, int pkt_size, int number, 95 int clients, int client, short base_port) { 96 97 98 int sock, backchan; 99 socklen_t recvd_len; 100 struct sockaddr_in local, recvd; 101 struct ip_mreq mreq; 102 struct ifreq ifreq; 103 struct in_addr lgroup; 104 struct timeval timeout; 105 106 if (group == NULL) { 107 group = &lgroup; 108 if (inet_pton(AF_INET, DEFAULT_GROUP, group) < 1) 109 return (-1); 110 } 111 112 if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { 113 perror("failed to open datagram socket"); 114 return (-1); 115 } 116 117 if ((backchan = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { 118 perror("failed to open back channel socket"); 119 return (-1); 120 } 121 122 strncpy(ifreq.ifr_name, interface, IFNAMSIZ); 123 if (ioctl(sock, SIOCGIFADDR, &ifreq) < 0) { 124 perror("no such interface"); 125 return (-1); 126 } 127 128 memcpy(&mreq.imr_interface, 129 &((struct sockaddr_in*) &ifreq.ifr_addr)->sin_addr, 130 sizeof(struct in_addr)); 131 132 mreq.imr_multiaddr.s_addr = group->s_addr; 133 if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, 134 sizeof(mreq)) < 0) { 135 perror("failed to add membership"); 136 return (-1); 137 } 138 139 if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF, 140 &((struct sockaddr_in *) &ifreq.ifr_addr)->sin_addr, 141 sizeof(struct in_addr)) < 0) { 142 perror("failed to bind interface"); 143 return (-1); 144 } 145 146 local.sin_family = AF_INET; 147 local.sin_addr.s_addr = group->s_addr; 148 local.sin_port = htons(DEFAULT_PORT); 149 local.sin_len = sizeof(local); 150 151 if (bind(sock, (struct sockaddr *)&local, sizeof(local)) < 0) { 152 perror("could not bind socket"); 153 return (-1); 154 } 155 156 timeval packets[number]; 157 timeval result; 158 char *packet; 159 packet = new char[pkt_size]; 160 int n = 0; 161 162 timerclear(&timeout); 163 timeout.tv_sec = TIMEOUT; 164 165 if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, 166 sizeof(timeout)) < 0) 167 perror("setsockopt failed"); 168 169 while (n < number) { 170 recvd_len = sizeof(recvd); 171 if (recvfrom(sock, packet, pkt_size, 0, (struct sockaddr *)&recvd, 172 &recvd_len) < 0) { 173 if (errno == EWOULDBLOCK) 174 break; 175 perror("recvfrom failed"); 176 return -1; 177 } 178 /* 179 * Bandwidth limiting. If there are N clients then we want 180 * 1/N packets from each, otherwise the clients will overwhelm 181 * the sender. 182 */ 183 if (n % clients == client) { 184 recvd.sin_port = htons(base_port + client); 185 if (sendto(backchan, packet, pkt_size, 0, 186 (struct sockaddr *)&recvd, sizeof(recvd)) < 0) { 187 perror("sendto failed"); 188 return -1; 189 } 190 } 191 gettimeofday(&packets[ntohl(*(int *)packet)], 0); 192 n++; 193 } 194 195 cout << "Packet run complete\n"; 196 if (n < number) 197 cout << "Missed " << number - n << " packets." << endl; 198 long maxgap = 0, mingap= INT_MAX; 199 for (int i = 0; i < number; i++) { 200 cout << "sec: " << packets[i].tv_sec << " usec: " << 201 packets[i].tv_usec << endl; 202 if (i < number - 1) { 203 timersub(&packets[i+1], &packets[i], &result); 204 long gap = (result.tv_sec * 1000000) + result.tv_usec; 205 if (gap > maxgap) 206 maxgap = gap; 207 if (gap < mingap) 208 mingap = gap; 209 } 210 } 211 212 cout << "maximum gap (usecs): " << maxgap << endl; 213 cout << "minimum gap (usecs): " << mingap << endl; 214 return 0; 215 216 } 217 218 // 219 // Structure to hold thread arguments 220 // 221 struct server_args { 222 struct timeval *packets; ///< The timestamps of returning packets 223 int number; ///< Number of packets to expect. 224 int pkt_size; ///< Size of the packets 225 int client; ///< Which client we listen for 226 }; 227 228 // 229 // server receives packets sent back from the sink 230 // 231 // @param passed ///< Arguments passed from the caller 232 // 233 // 0return always NULL 234 void* server(void *passed) { 235 236 int sock, n =0; 237 struct timeval timeout; 238 struct sockaddr_in addr; 239 server_args *args = (server_args *)passed; 240 241 timerclear(&timeout); 242 timeout.tv_sec = TIMEOUT; 243 244 if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { 245 perror("could not open server socket"); 246 return NULL; 247 } 248 249 bzero(&addr, sizeof(addr)); 250 addr.sin_family = AF_INET; 251 addr.sin_addr.s_addr = INADDR_ANY; 252 addr.sin_port = htons(args->client); 253 addr.sin_len = sizeof(addr); 254 255 if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) { 256 perror("could not bind server socket"); 257 return NULL; 258 } 259 260 if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, 261 sizeof(timeout)) < 0) 262 perror("setsockopt failed"); 263 264 char packet[args->pkt_size]; 265 while (n < args->number) { 266 if (recvfrom(sock, &packet, args->pkt_size, 0, NULL, 0) < 0) { 267 if (errno == EWOULDBLOCK) 268 break; 269 perror("recvfrom failed"); 270 return NULL; 271 } 272 gettimeofday(&args->packets[ntohl(*(int *)packet)], 0); 273 n++; 274 } 275 276 cout << "Packet Reflection Complete" << endl; 277 278 if (n < args->number) 279 cout << "Missed " << args->number - n << " packets." << endl; 280 281 return NULL; 282 283 } 284 285 // 286 // transmit packets for the multicast test 287 // 288 // @param interface ///< text name of the interface (em0 etc.) 289 // @param group ///< multicast group 290 // @param pkt_size ///< packet size 291 // @param number ///< number of packets 292 // @param gap ///< inter packet gap in nano-seconds 293 // @param clients ///< number of clients we intend to run 294 // 295 // @return 0 for OK, -1 for error, sets errno 296 // 297 int source(char *interface, struct in_addr *group, int pkt_size, 298 int number, int gap, int clients, short base_port) { 299 300 int sock; 301 struct sockaddr_in addr; 302 struct ip_mreq mreq; 303 struct ifreq ifreq; 304 struct in_addr lgroup; 305 306 if (group == NULL) { 307 group = &lgroup; 308 if (inet_pton(AF_INET, DEFAULT_GROUP, group) < 1) 309 return (-1); 310 } 311 312 if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { 313 perror("could not open dgram socket"); 314 return (-1); 315 } 316 317 bzero(&addr, sizeof(addr)); 318 addr.sin_family = AF_INET; 319 addr.sin_port = htons(DEFAULT_PORT); 320 addr.sin_addr.s_addr = group->s_addr; 321 addr.sin_len = sizeof(addr); 322 323 strncpy(ifreq.ifr_name, interface, IFNAMSIZ); 324 if (ioctl(sock, SIOCGIFADDR, &ifreq) < 0) { 325 perror("no such interface"); 326 return (-1); 327 } 328 329 memcpy(&mreq.imr_interface, 330 &((struct sockaddr_in*) &ifreq.ifr_addr)->sin_addr, 331 sizeof(struct in_addr)); 332 333 mreq.imr_multiaddr.s_addr = group->s_addr; 334 if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, 335 sizeof(mreq)) < 0) { 336 perror("failed to add membership"); 337 return (-1); 338 } 339 340 if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF, 341 &((struct sockaddr_in *) &ifreq.ifr_addr)->sin_addr, 342 sizeof(struct in_addr)) < 0) { 343 perror("failed to bind interface"); 344 return (-1); 345 } 346 347 u_char ttl = 64; 348 349 if (setsockopt(sock, IPPROTO_IP, IP_MULTICAST_TTL, 350 &ttl, sizeof(ttl)) < 0) { 351 perror("failed to set TTL"); 352 return (-1); 353 } 354 355 char *packets[number]; 356 for (int i = 0;i < number; i++) { 357 packets[i] = new char[pkt_size]; 358 *(int *)packets[i] = htonl(i); 359 } 360 361 struct timeval sent[number]; 362 struct timeval received[clients][number]; 363 server_args args[clients]; 364 pthread_t thread[clients]; 365 366 for (int i = 0;i < clients; i++) { 367 args[i].pkt_size = pkt_size; 368 args[i].packets = received[i]; 369 args[i].number = number / clients; 370 args[i].client = base_port + i; 371 if (pthread_create(&thread[i], NULL, server, &args[i]) != 0) { 372 perror("failed to create server thread"); 373 return -1; 374 } 375 } 376 377 struct timespec sleeptime; 378 sleeptime.tv_sec = 0; 379 sleeptime.tv_nsec = gap; 380 381 for (int i = 0;i < number; i++) { 382 if (sendto(sock, (void *)packets[i], pkt_size, 0, 383 (struct sockaddr *)&addr, sizeof(addr)) < 0) { 384 perror("sendto failed"); 385 return -1; 386 } 387 gettimeofday(&sent[i], 0); 388 if (gap > 0) 389 if (nanosleep(&sleeptime, NULL) < 0) { 390 perror("nanosleep failed"); 391 return -1; 392 } 393 } 394 395 for (int i = 0; i < clients; i++) { 396 if (pthread_join(thread[i], NULL) != 0) { 397 perror("failed to join thread"); 398 return -1; 399 } 400 } 401 402 timeval result; 403 vector<int> deltas; 404 double idx[] = { .0001, .001, .01, .1, .5, .9, .99, .999, .9999, 0.0 }; 405 406 for (int client = 0;client < clients; client++) { 407 deltas.clear(); 408 cout << "Results from client #" << client << endl; 409 cout << "in usecs" << endl; 410 for (int i = 0; i < number; i++) { 411 // if (i % clients != client) 412 // continue; 413 if (&args[client].packets[i].tv_sec == 0) 414 continue; 415 timersub(&args[client].packets[i], &sent[i], &result); 416 deltas.push_back(result.tv_usec); 417 // cout << "sec: " << result.tv_sec; 418 // cout << " usecs: " << result.tv_usec << endl; 419 } 420 cout << "comparing " << long(deltas.size()) << " deltas" << endl; 421 cout << "number represents usecs of round-trip time" << endl; 422 sort(deltas.begin(), deltas.end()); 423 for (int i = 0; idx[i] != 0; ++i) { 424 printf("%s% 5d", (i == 0) ? "" : " ", 425 deltas[(int) (idx[i] * deltas.size())]); 426 } 427 printf("\n"); 428 } 429 430 return 0; 431 } 432 433 434 // 435 // main - the main program 436 // 437 // \param -g multicast group address to which to send/recv packets on 438 // \param -n the number of packets to send 439 // \param -s packet size in bytes 440 // \param -t inter-packet gap, in nanoseconds 441 // 442 // 443 int main(int argc, char**argv) 444 { 445 446 const int MAXNSECS = 999999999; ///< Must be < 1.0 x 10**9 nanoseconds 447 448 char ch; ///< character from getopt() 449 extern char* optarg; ///< option argument 450 451 char* interface = 0; ///< Name of the interface 452 struct in_addr *group = NULL; ///< the multicast group address 453 int pkt_size = 0; ///< packet size 454 int gap = 0; ///< inter packet gap (in nanoseconds) 455 int number = 0; ///< number of packets to transmit 456 bool server = false; ///< are we on he receiving end of multicast 457 int client = 0; ///< for receivers which client are we 458 int clients = 1; ///< for senders how many clients are there 459 short base_port = SERVER_PORT; ///< to have multiple copies running at once 460 461 if (argc < 2 || argc > 16) 462 usage(); 463 464 while ((ch = getopt(argc, argv, "M:m:g:i:n:s:t:b:rh")) != -1) { 465 switch (ch) { 466 case 'g': 467 group = new (struct in_addr ); 468 if (inet_pton(AF_INET, optarg, group) < 1) 469 usage(argv[0] + string(" Error: invalid multicast group") + 470 optarg); 471 break; 472 case 'i': 473 interface = optarg; 474 break; 475 case 'n': 476 number = atoi(optarg); 477 if (number < 0 || number > INT_MAX) 478 usage(argv[0] + string(" Error: ") + optarg + 479 string(" number of packets out of range")); 480 break; 481 case 's': 482 pkt_size = atoi(optarg); 483 if (pkt_size < 0 || pkt_size > 65535) 484 usage(argv[0] + string(" Error: ") + optarg + 485 string(" packet size out of range")); 486 break; 487 case 't': 488 gap = atoi(optarg); 489 if (gap < 0 || gap > MAXNSECS) 490 usage(argv[0] + string(" Error: ") + optarg + 491 string(" gap out of range")); 492 break; 493 case 'r': 494 server = true; 495 break; 496 case 'm': 497 client = atoi(optarg); 498 break; 499 case 'M': 500 clients = atoi(optarg); 501 break; 502 case 'b': 503 base_port = atoi(optarg); 504 break; 505 case 'h': 506 usage(string("Help\n")); 507 break; 508 } 509 } 510 511 if (server) { 512 if (clients <= 0 || client < 0) 513 usage("must specify client (-m) and number of clients (-M)"); 514 sink(interface, group, pkt_size, number, clients, client, 515 base_port); 516 } else { 517 if (clients <= 0) 518 usage("must specify number of clients (-M)"); 519 source(interface, group, pkt_size, number, gap, clients, 520 base_port); 521 } 522 523 } 524