1 /*
2  * Copyright (c) 2014, 2015, 2016  Machine Zone, Inc.
3  *
4  * Original author: Lev Walkin <lwalkin@machinezone.com>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14 
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 #define _ISOC99_SOURCE
28 #define _BSD_SOURCE
29 #include <getopt.h>
30 #include <sysexits.h>
31 #include <stdlib.h>
32 #include <unistd.h>
33 #include <stdio.h>
34 #include <ctype.h>
35 #include <string.h>
36 #include <math.h>
37 #include <sys/types.h>
38 #include <sys/socket.h>
39 #include <arpa/inet.h>
40 #include <libgen.h> /* basename(3) */
41 #include <ifaddrs.h>
42 #include <err.h>
43 #include <errno.h>
44 #include <assert.h>
45 #include <time.h>
46 
47 #include <statsd.h>
48 
49 #include "tcpkali.h"
50 #include "tcpkali_run.h"
51 #include "tcpkali_mavg.h"
52 #include "tcpkali_data.h"
53 #include "tcpkali_events.h"
54 #include "tcpkali_signals.h"
55 #include "tcpkali_terminfo.h"
56 #include "tcpkali_websocket.h"
57 #include "tcpkali_transport.h"
58 #include "tcpkali_syslimits.h"
59 #include "tcpkali_logging.h"
60 
61 /*
62  * Describe the command line options.
63  */
64 #define CLI_VERBOSE_OFFSET (1 << 8)
65 #define CLI_STATSD_OFFSET (1 << 9)
66 #define CLI_CHAN_OFFSET (1 << 10)
67 #define CLI_CONN_OFFSET (1 << 11)
68 #define CLI_SOCKET_OPT (1 << 12)
69 #define CLI_LATENCY (1 << 13)
70 #define CLI_DUMP (1 << 14)
71 static struct option cli_long_options[] = {
72     {"channel-lifetime", 1, 0, CLI_CHAN_OFFSET + 't'},
73     {"channel-bandwidth-upstream", 1, 0, 'U'},
74     {"channel-bandwidth-downstream", 1, 0, 'D'},
75     {"connections", 1, 0, 'c'},
76     {"connect-rate", 1, 0, 'R'},
77     {"connect-timeout", 1, 0, CLI_CONN_OFFSET + 't'},
78     {"duration", 1, 0, 'T'},
79     {"dump-one", 0, 0, CLI_DUMP + '1'},
80     {"dump-one-in", 0, 0, CLI_DUMP + 'i'},
81     {"dump-one-out", 0, 0, CLI_DUMP + 'o'},
82     {"dump-all", 0, 0, CLI_DUMP + 'a'},
83     {"dump-all-in", 0, 0, CLI_DUMP + 'I'},
84     {"dump-all-out", 0, 0, CLI_DUMP + 'O'},
85     {"first-message", 1, 0, '1'},
86     {"first-message-file", 1, 0, 'F'},
87     {"help", 0, 0, 'E'},
88     {"header", 1, 0, 'H'},
89     {"latency-connect", 0, 0, CLI_LATENCY + 'c'},
90     {"latency-first-byte", 0, 0, CLI_LATENCY + 'f'},
91     {"latency-marker", 1, 0, CLI_LATENCY + 'm'},
92     {"latency-marker-skip", 1, 0, CLI_LATENCY + 's'},
93     {"latency-percentiles", 1, 0, CLI_LATENCY + 'p'},
94     {"listen-port", 1, 0, 'l'},
95     {"listen-mode", 1, 0, 'L'},
96     {"message", 1, 0, 'm'},
97     {"message-file", 1, 0, 'f'},
98     {"message-rate", 1, 0, 'r'},
99     {"message-stop", 1, 0, 's'},
100     {"nagle", 1, 0, 'N'},
101     {"rcvbuf", 1, 0, CLI_SOCKET_OPT + 'R'},
102     {"sndbuf", 1, 0, CLI_SOCKET_OPT + 'S'},
103     {"source-ip", 1, 0, 'I'},
104     {"statsd", 0, 0, CLI_STATSD_OFFSET + 'e'},
105     {"statsd-host", 1, 0, CLI_STATSD_OFFSET + 'h'},
106     {"statsd-port", 1, 0, CLI_STATSD_OFFSET + 'p'},
107     {"statsd-namespace", 1, 0, CLI_STATSD_OFFSET + 'n'},
108     {"statsd-latency-window", 1, 0, CLI_STATSD_OFFSET + 'w'},
109     {"unescape-message-args", 0, 0, 'e'},
110     {"version", 0, 0, 'V'},
111     {"verbose", 1, 0, CLI_VERBOSE_OFFSET + 'v'},
112     {"workers", 1, 0, 'w'},
113     {"write-combine", 1, 0, 'C'},
114     {"websocket", 0, 0, 'W'},
115     {"ws", 0, 0, 'W'},
116     {"message-marker", 0, 0, 'M'},
117     {0, 0, 0, 0}};
118 
119 static struct tcpkali_config {
120     int max_connections;
121     double connect_rate;  /* New connects per second. */
122     double test_duration; /* Seconds for the full test. */
123     double latency_window;  /* Seconds */
124     int statsd_enable;
125     char *statsd_host;
126     int statsd_port;
127     char *statsd_namespace;
128     char *listen_host;    /* Address on which to listen. Can be NULL */
129     int listen_port;      /* Port on which to listen. */
130     char *first_hostport; /* A single (first) host:port specification */
131     char *first_path;     /* A /path specification from the first host */
132     struct http_headers {
133         size_t offset;
134         char buffer[1024];
135     } http_headers;
136 } default_config = {.max_connections = 1,
137                     .connect_rate = 100.0,
138                     .test_duration = 10.0,
139                     .statsd_enable = 0,
140                     .statsd_host = "127.0.0.1",
141                     .statsd_port = 8125,
142                     .statsd_namespace = "tcpkali"};
143 
144 /*
145  * Bunch of utility functions defined at the end of this file.
146  */
147 static void usage_short(char *argv0);
148 static void usage_long(char *argv0, struct tcpkali_config *);
149 struct multiplier {
150     char *prefix;
151     double mult;
152 };
153 static double parse_with_multipliers(const char *, char *str,
154                                      struct multiplier *, int n);
155 static int parse_percentile_values(const char *option, char *str,
156                                    struct percentile_values *array);
157 
158 /* clang-format off */
159 static struct multiplier km_multiplier[] = { { "k", 1000 }, { "m", 1000000 } };
160 static struct multiplier kb_multiplier[] = { { "k", 1024 }, { "m", 1024*1024 } };
161 static struct multiplier s_multiplier[] = {
162     { "ms", 0.001 }, { "millisecond", 0.001 }, { "milliseconds", 0.001 },
163     { "s", 1 }, { "second", 1 }, { "seconds", 1 },
164     { "m", 60 }, { "min", 60 }, { "minute", 60 }, { "minutes", 60 },
165     { "h", 3600 }, { "hr", 3600 }, { "hour", 3600 }, { "hours", 3600 },
166     { "d", 86400 }, { "day", 86400 }, { "days", 86400 },
167     { "y", 31536000 }, { "year", 31536000 }, { "years", 31536000 }
168 };
169 static struct multiplier bw_multiplier[] = {
170     /* bits per second */
171     { "bps", 1.0/8 },
172     { "kbps", 1000/8 },
173     { "Kbps", 1000/8 },
174     { "mbps", 1000000/8 }, { "Mbps", 1000000/8 },
175     { "gbps", 1000000000/8 }, { "Gbps", 1000000000/8 },
176     /* Bytes per second */
177     { "Bps", 1 },
178     { "kBps", 1000 }, { "KBps", 1000 },
179     { "mBps", 1000000 }, { "MBps", 1000000 },
180     { "gBps", 1000000000 }, { "GBps", 1000000000 }
181 };
182 /* clang-format on */
183 
184 #ifndef  HAVE_SRANDOMDEV
portable_srandomdev()185 void portable_srandomdev() {
186     FILE *f = fopen("/dev/urandom", "rb");
187     if(f) {
188         unsigned int r;
189         if(fread(&r, sizeof(r), 1, f) == 1) {
190             fclose(f);
191             srandom(r);
192             return;
193         }
194         fclose(f);
195     }
196     /* No usable /dev/urandom, too bad. */
197     srandom(getpid() ^ time(NULL) ^ atol(getenv("TCPKALI_RANDOM")?:"0"));
198 }
199 #endif
200 
201 /*
202  * Parse command line options and kick off the engine.
203  */
204 int
main(int argc,char ** argv)205 main(int argc, char **argv) {
206     struct tcpkali_config conf = default_config;
207     struct engine_params engine_params = {.verbosity_level = DBG_ERROR,
208                                           .connect_timeout = 1.0,
209                                           .channel_lifetime = INFINITY,
210                                           .nagle_setting = NSET_UNSET,
211                                           .write_combine = WRCOMB_ON};
212     struct rate_modulator rate_modulator = {.state = RM_UNMODULATED};
213     int unescape_message_data = 0;
214 
215 #ifdef HAVE_SRANDOMDEV
216     srandomdev();
217 #else
218     portable_srandomdev();
219 #endif
220 
221     struct percentile_values latency_percentiles = {
222         .size = 0, .values = NULL,
223     };
224 
225     while(1) {
226         char *option = argv[optind];
227         int longindex = -1;
228         int c;
229         c = getopt_long(argc, argv, "dhc:e1:m:f:r:l:vw:T:H:", cli_long_options,
230                         &longindex);
231         if(c == -1) break;
232         switch(c) {
233         case 'V':
234             printf(PACKAGE_NAME " version " VERSION
235 #ifdef USE_LIBUV
236                                 " (libuv)"
237 #else
238                                 " (libev)"
239 #endif
240                                 "\n");
241             exit(0);
242         case 'h':
243             usage_short(argv[0]);
244             exit(EX_USAGE);
245         case 'E':
246             usage_long(argv[0], &default_config);
247             exit(EX_USAGE);
248         case 'v': /* -v */
249             engine_params.verbosity_level++;
250             if(engine_params.verbosity_level >= _DBG_MAX) {
251                 engine_params.verbosity_level = (_DBG_MAX - 1);
252             }
253             break;
254         case 'H': {
255             char *hdrbuf = strdup(optarg);
256             size_t hdrlen = strlen(optarg);
257             if(unescape_message_data) unescape_data(hdrbuf, &hdrlen);
258             char buffer[PRINTABLE_DATA_SUGGESTED_BUFFER_SIZE(hdrlen)];
259             if(hdrlen == 0) {
260                 warning("--header=\"\" value is empty, ignored\n");
261                 break;
262             }
263             if(!isalpha(hdrbuf[0])) {
264                 warning(
265                     "--header=%s starts with '%c', are you sure?\n",
266                     printable_data(buffer, sizeof(buffer), hdrbuf, hdrlen, 1),
267                     hdrbuf[0]);
268             }
269             char *lf = memchr(hdrbuf, '\n', hdrlen);
270             if(lf) {
271                 fprintf(stderr,
272                     "--header=%s should not contain '\\n'\n",
273                     printable_data(buffer, sizeof(buffer), hdrbuf, hdrlen, 1));
274                 exit(EX_USAGE);
275             }
276 
277             if(conf.http_headers.offset + hdrlen + 2
278                >= sizeof(conf.http_headers.buffer)) {
279                 fprintf(stderr, "--header adds too many HTTP headers\n");
280                 exit(EX_USAGE);
281             } else {
282                 memcpy(conf.http_headers.buffer + conf.http_headers.offset,
283                        hdrbuf, hdrlen);
284                 conf.http_headers.offset += hdrlen;
285                 memcpy(conf.http_headers.buffer + conf.http_headers.offset,
286                        "\r\n", sizeof("\r\n"));
287                 conf.http_headers.offset += sizeof("\r\n") - 1;
288                 free(hdrbuf);
289             }
290             } break;
291         case CLI_VERBOSE_OFFSET + 'v': /* --verbose <level> */
292             engine_params.verbosity_level = atoi(optarg);
293             if((int)engine_params.verbosity_level < 0
294                || engine_params.verbosity_level >= _DBG_MAX) {
295                 fprintf(stderr, "Expecting --verbose=[0..%d]\n", _DBG_MAX - 1);
296                 exit(EX_USAGE);
297             }
298             break;
299         case 'd': /* -d */
300         /* FALL THROUGH */
301         case CLI_DUMP + '1': /* --dump-one */
302             engine_params.dump_setting |= DS_DUMP_ONE;
303             break;
304         case CLI_DUMP + 'i': /* --dump-one-in */
305             engine_params.dump_setting |= DS_DUMP_ONE_IN;
306             break;
307         case CLI_DUMP + 'o': /* --dump-one-out */
308             engine_params.dump_setting |= DS_DUMP_ONE_OUT;
309             break;
310         case CLI_DUMP + 'a': /* --dump-all */
311             engine_params.dump_setting |= DS_DUMP_ALL;
312             break;
313         case CLI_DUMP + 'I': /* --dump-all-in */
314             engine_params.dump_setting |= DS_DUMP_ALL_IN;
315             break;
316         case CLI_DUMP + 'O': /* --dump-all-out */
317             engine_params.dump_setting |= DS_DUMP_ALL_OUT;
318             break;
319         case 'c':
320             conf.max_connections = parse_with_multipliers(
321                 option, optarg, km_multiplier,
322                 sizeof(km_multiplier) / sizeof(km_multiplier[0]));
323             if(conf.max_connections < 0) {
324                 fprintf(stderr, "Expecting --connections > 0\n");
325                 exit(EX_USAGE);
326             }
327             break;
328         case 'R':
329             conf.connect_rate = parse_with_multipliers(
330                 option, optarg, km_multiplier,
331                 sizeof(km_multiplier) / sizeof(km_multiplier[0]));
332             if(conf.connect_rate <= 0) {
333                 fprintf(stderr, "Expected positive --connect-rate=%s\n",
334                         optarg);
335                 exit(EX_USAGE);
336             }
337             break;
338         case 'T':
339             conf.test_duration = parse_with_multipliers(
340                 option, optarg, s_multiplier,
341                 sizeof(s_multiplier) / sizeof(s_multiplier[0]));
342             if(conf.test_duration <= 0) {
343                 fprintf(stderr, "Expected positive --duration=%s\n", optarg);
344                 exit(EX_USAGE);
345             }
346             break;
347         case 'e':
348             unescape_message_data = 1;
349             break;
350         case 'M': /* --message-marker */
351             if(!engine_params.message_marker
352                 && (engine_params.latency_setting & SLT_MARKER)) {
353                 fprintf(stderr,
354                         "--message-marker: --latency-marker is already specified, use one or the other\n");
355                 exit(EX_USAGE);
356             }
357             engine_params.message_marker = 1;
358             break;
359         case 'm': /* --message */
360             message_collection_add(&engine_params.message_collection,
361                                    MSK_PURPOSE_MESSAGE, optarg, strlen(optarg),
362                                    unescape_message_data, 1);
363             break;
364         case '1': /* --first-message */
365             message_collection_add(&engine_params.message_collection,
366                                    MSK_PURPOSE_FIRST_MSG, optarg,
367                                    strlen(optarg), unescape_message_data, 1);
368             break;
369         case 'f': { /* --message-file */
370             char *data;
371             size_t size;
372             if(read_in_file(optarg, &data, &size) != 0) exit(EX_DATAERR);
373             message_collection_add(&engine_params.message_collection,
374                                    MSK_PURPOSE_MESSAGE, data, size,
375                                    unescape_message_data, 1);
376             free(data);
377         } break;
378         case 'F': { /* --first-message-file */
379             char *data;
380             size_t size;
381             if(read_in_file(optarg, &data, &size) != 0) exit(EX_DATAERR);
382             message_collection_add(&engine_params.message_collection,
383                                    MSK_PURPOSE_FIRST_MSG, data, size,
384                                    unescape_message_data, 1);
385         } break;
386         case 'w': {
387             int n = atoi(optarg);
388             if(n <= 0) {
389                 if(optarg[0] >= '0' && optarg[0] <= '9') {
390                     fprintf(stderr, "Expected --workers > 1\n");
391                 } else if(optarg[0] == 's') {
392                     fprintf(stderr,
393                             "Expected -w <N> (--workers) or --ws "
394                             "(--websocket), but not -ws.\n");
395                 } else {
396                     fprintf(stderr,
397                             "Expected --workers <N> (-w <N>), or --websocket "
398                             "(--ws)\n");
399                 }
400                 exit(EX_USAGE);
401             }
402             if(n > number_of_cpus()) {
403                 fprintf(stderr,
404                         "Value --workers=%d is unreasonably large,"
405                         " only %ld CPU%s detected\n",
406                         n, number_of_cpus(), number_of_cpus() == 1 ? "" : "s");
407                 exit(EX_USAGE);
408             }
409             engine_params.requested_workers = n;
410             break;
411         }
412         case 'U': { /* --channel-bandwidth-upstream <Bw> */
413             double Bps = parse_with_multipliers(
414                 option, optarg, bw_multiplier,
415                 sizeof(bw_multiplier) / sizeof(bw_multiplier[0]));
416             if(Bps <= 0) {
417                 fprintf(stderr, "Expecting --channel-bandwidth-upstream > 0\n");
418                 exit(EX_USAGE);
419             }
420             engine_params.channel_send_rate = RATE_BPS(Bps);
421             break;
422         }
423         case 'D': { /* --channel-bandwidth-downstream <Bw> */
424             double Bps = parse_with_multipliers(
425                 option, optarg, bw_multiplier,
426                 sizeof(bw_multiplier) / sizeof(bw_multiplier[0]));
427             if(Bps < 0) {
428                 fprintf(stderr,
429                         "Expecting --channel-bandwidth-downstream > 0\n");
430                 exit(EX_USAGE);
431             }
432             engine_params.channel_recv_rate = RATE_BPS(Bps);
433             break;
434         }
435         case 'r': { /* --message-rate <Rate> */
436             if(optarg[0] == '@') {
437                 double latency = parse_with_multipliers(
438                     option, optarg + 1, s_multiplier,
439                     sizeof(s_multiplier) / sizeof(s_multiplier[0]));
440                 if(latency <= 0) {
441                     fprintf(stderr, "Expecting --message-rate @<Latency>\n");
442                     exit(EX_USAGE);
443                 }
444 
445                 /* Override -r<Rate> spec. */
446                 if(rate_modulator.mode == RM_UNMODULATED
447                    && engine_params.channel_send_rate.value_base
448                           != RS_UNLIMITED) {
449                     warning(
450                         "--message-rate %s overrides previous fixed "
451                         "--message-rate specification.\n",
452                         optarg);
453                 }
454                 engine_params.channel_send_rate = RATE_MPS(100); /* Initial */
455                 rate_modulator.mode = RM_MAX_RATE_AT_TARGET_LATENCY;
456                 rate_modulator.latency_target = latency;
457                 rate_modulator.latency_target_s = strdup(optarg + 1);
458                 conf.test_duration = INFINITY;
459             } else {
460                 double rate = parse_with_multipliers(
461                     option, optarg, km_multiplier,
462                     sizeof(km_multiplier) / sizeof(km_multiplier[0]));
463                 if(rate <= 0) {
464                     fprintf(stderr, "Expecting --message-rate > 0\n");
465                     exit(EX_USAGE);
466                 }
467                 engine_params.channel_send_rate = RATE_MPS(rate);
468                 /* Override -r@<Time> spec. */
469                 if(rate_modulator.mode != RM_UNMODULATED) {
470                     warning(
471                         "--message-rate %s overrides previous dynamic "
472                         "--message-rate specification.\n",
473                         optarg);
474                     rate_modulator.mode = RM_UNMODULATED;
475                 }
476             }
477             break;
478         }
479         case 's': { /* --message-stop */
480             char *data = strdup(optarg);
481             size_t size = strlen(optarg);
482             if(unescape_message_data) unescape_data(data, &size);
483             if(size == 0) {
484                 fprintf(stderr,
485                         "--message-stop: Non-empty message expected\n");
486                 exit(EX_USAGE);
487             }
488             if(parse_expression(&engine_params.message_stop_expr, data, size,
489                                 0)
490                == -1) {
491                 fprintf(stderr,
492                         "--message-stop: Failed to parse expression\n");
493                 exit(EX_USAGE);
494             } else if(!EXPR_IS_TRIVIAL(engine_params.message_stop_expr)) {
495                 fprintf(stderr,
496                         "--message-stop: Non-trivial expressions are not "
497                         "supported\n");
498                 exit(EX_USAGE);
499             }
500         } break;
501         case 'N': /* --nagle {on|off} */
502             /* Enabling Nagle toggles off NODELAY */
503             if(strcmp(optarg, "on") == 0)
504                 engine_params.nagle_setting = NSET_NODELAY_OFF;
505             else if(strcmp(optarg, "off") == 0)
506                 engine_params.nagle_setting = NSET_NODELAY_ON;
507             else {
508                 fprintf(stderr, "Expecting --nagle \"on\" or \"off\"\n");
509                 exit(EX_USAGE);
510             }
511             break;
512         case 'C': /* --write-combine {on|off} */
513             if(strcmp(optarg, "on") == 0) {
514                 fprintf(stderr,
515                         "NOTE: --write-combine on is a default setting\n");
516                 engine_params.write_combine = WRCOMB_ON;
517             } else if(strcmp(optarg, "off") == 0) {
518                 engine_params.write_combine = WRCOMB_OFF;
519             } else {
520                 fprintf(stderr, "Expecting --write-combine off\n");
521                 exit(EX_USAGE);
522             }
523             break;
524         case CLI_SOCKET_OPT + 'R': { /* --rcvbuf */
525             long size = parse_with_multipliers(
526                 option, optarg, kb_multiplier,
527                 sizeof(kb_multiplier) / sizeof(kb_multiplier[0]));
528             if(size <= 0) {
529                 fprintf(stderr, "Expecting --rcvbuf > 0\n");
530                 exit(EX_USAGE);
531             }
532             engine_params.sock_rcvbuf_size = size;
533         } break;
534         case CLI_SOCKET_OPT + 'S': { /* --sndbuf */
535             long size = parse_with_multipliers(
536                 option, optarg, kb_multiplier,
537                 sizeof(kb_multiplier) / sizeof(kb_multiplier[0]));
538             if(size <= 0) {
539                 fprintf(stderr, "Expecting --sndbuf > 0\n");
540                 exit(EX_USAGE);
541             }
542             engine_params.sock_sndbuf_size = size;
543         } break;
544         case CLI_STATSD_OFFSET + 'e':
545             conf.statsd_enable = 1;
546             break;
547         case CLI_STATSD_OFFSET + 'h':
548             conf.statsd_host = strdup(optarg);
549             break;
550         case CLI_STATSD_OFFSET + 'n':
551             conf.statsd_namespace = strdup(optarg);
552             break;
553         case CLI_STATSD_OFFSET + 'p':
554             conf.statsd_port = atoi(optarg);
555             if(conf.statsd_port <= 0 || conf.statsd_port >= 65535) {
556                 fprintf(stderr, "--statsd-port=%d is not in [1..65535]\n",
557                         conf.statsd_port);
558                 exit(EX_USAGE);
559             }
560             break;
561         case CLI_STATSD_OFFSET + 'w':
562             conf.latency_window = parse_with_multipliers(
563                 option, optarg, s_multiplier,
564                 sizeof(s_multiplier) / sizeof(s_multiplier[0]));
565             if(conf.latency_window <= 0) {
566                 fprintf(stderr, "Expected positive --statsd-latency-window=%s\n", optarg);
567                 exit(EX_USAGE);
568             }
569             break;
570         case 'l': {
571             const char *port = optarg;
572             const char *colon = strchr(optarg, ':');
573             if(colon) {
574                 port = colon + 1;
575                 free(conf.listen_host);
576                 conf.listen_host = strdup(optarg);
577                 assert(conf.listen_host);
578                 conf.listen_host[colon - optarg] = '\0';
579             }
580             conf.listen_port = atoi(port);
581             if(conf.listen_port <= 0 || conf.listen_port >= 65535) {
582                 fprintf(stderr, "--listen-port=%d is not in [1..65535]\n",
583                         conf.listen_port);
584                 exit(EX_USAGE);
585             }
586             }
587             break;
588         case 'L': /* --listen-mode={silent|active} */
589             if(strcmp(optarg, "silent") == 0) {
590                 engine_params.listen_mode = LMODE_DEFAULT;
591             }
592             if(strcmp(optarg, "active") == 0) {
593                 engine_params.listen_mode &= ~_LMODE_SND_MASK;
594                 engine_params.listen_mode |= LMODE_ACTIVE;
595             } else {
596                 fprintf(stderr,
597                         "--listen-mode=%s is not one of {silent|active}\n",
598                         optarg);
599                 exit(EX_USAGE);
600             }
601             break;
602         case CLI_CONN_OFFSET + 't':
603             engine_params.connect_timeout = parse_with_multipliers(
604                 option, optarg, s_multiplier,
605                 sizeof(s_multiplier) / sizeof(s_multiplier[0]));
606             if(engine_params.connect_timeout <= 0.0) {
607                 fprintf(stderr, "Expected positive --connect-timeout=%s\n",
608                         optarg);
609                 exit(EX_USAGE);
610             }
611             break;
612         case CLI_CHAN_OFFSET + 't':
613             engine_params.channel_lifetime = parse_with_multipliers(
614                 option, optarg, s_multiplier,
615                 sizeof(s_multiplier) / sizeof(s_multiplier[0]));
616             if(engine_params.channel_lifetime < 0.0) {
617                 fprintf(stderr, "Expected non-negative --channel-lifetime=%s\n",
618                         optarg);
619                 exit(EX_USAGE);
620             }
621             break;
622         case 'W': /* --websocket: Enable WebSocket framing */
623             engine_params.websocket_enable = 1;
624             break;
625         case CLI_LATENCY + 'c': /* --latency-connect */
626             engine_params.latency_setting |= SLT_CONNECT;
627             break;
628         case CLI_LATENCY + 'f': /* --latency-first-byte */
629             engine_params.latency_setting |= SLT_FIRSTBYTE;
630             break;
631         case CLI_LATENCY + 'm': { /* --latency-marker */
632             if(engine_params.message_marker) {
633                 fprintf(stderr,
634                         "--latency-marker: --message-marker is already specified, use one or the other\n");
635                 exit(EX_USAGE);
636             }
637             engine_params.latency_setting |= SLT_MARKER;
638             char *data = strdup(optarg);
639             size_t size = strlen(optarg);
640             if(unescape_message_data) unescape_data(data, &size);
641             if(size == 0) {
642                 fprintf(stderr,
643                         "--latency-marker: Non-empty marker expected\n");
644                 exit(EX_USAGE);
645             }
646             if(parse_expression(&engine_params.latency_marker_expr, data, size,
647                                 0)
648                == -1) {
649                 fprintf(stderr,
650                         "--latency-marker: Failed to parse expression\n");
651                 exit(EX_USAGE);
652             }
653         } break;
654         case CLI_LATENCY + 's': { /* --latency-marker-skip */
655             engine_params.latency_marker_skip = parse_with_multipliers(
656                 option, optarg, km_multiplier,
657                 sizeof(km_multiplier) / sizeof(km_multiplier[0]));
658             if(engine_params.latency_marker_skip < 0) {
659                 fprintf(stderr,
660                         "--latency-marker-skip: "
661                         "Failed to parse or out of range expression\n");
662                 exit(EX_USAGE);
663             }
664         } break;
665         case CLI_LATENCY + 'p': { /* --latency-percentiles */
666             if(parse_percentile_values(cli_long_options[longindex].name,
667                                        optarg, &latency_percentiles))
668                 exit(EX_USAGE);
669         } break;
670         case 'I': { /* --source-ip */
671             if(add_source_ip(&engine_params.source_addresses, optarg) < 0) {
672                 fprintf(stderr, "--source-ip=%s: local IP address expected\n",
673                         optarg);
674                 exit(EX_USAGE);
675             }
676         } break;
677         default:
678             fprintf(stderr, "%s: unknown option\n", option);
679             usage_long(argv[0], &default_config);
680             exit(EX_USAGE);
681         }
682     }
683 
684     int print_stats = isatty(1);
685     if(print_stats) {
686         if(tcpkali_init_terminal() == -1) {
687             warning("Dumb terminal, expect unglorified output.\n");
688             print_stats = 0;
689         }
690     }
691 
692     /* Check that -H,--header is not given without --ws,--websocket */
693     if(conf.http_headers.offset > 0 && !engine_params.websocket_enable) {
694         fprintf(stderr, "--header option ignored without --websocket\n");
695         exit(EX_USAGE);
696     }
697 
698     if(rate_modulator.latency_target > 1) {
699         char *end = 0;
700         strtod(rate_modulator.latency_target_s, &end);
701         if(*end == '\0') {
702             /* No explicit unit ending, such as "ms" */
703             warning(
704                 "--message-rate @%s: target latency is greater than one "
705                 "second, which is probably not what you want\n",
706                 rate_modulator.latency_target_s);
707             warning("Suggesting using --message-rate @%gms\n",
708                     rate_modulator.latency_target);
709         }
710     }
711 
712     /*
713      * Avoid spawning more threads than connections.
714      */
715     if(engine_params.requested_workers == 0
716        && conf.max_connections < number_of_cpus() && conf.listen_port == 0) {
717         engine_params.requested_workers = conf.max_connections;
718     }
719     if(!engine_params.requested_workers)
720         engine_params.requested_workers = number_of_cpus();
721 
722 
723     /*
724      * Check that we'll have a chance to report latency
725      */
726     if(conf.latency_window) {
727         if(conf.latency_window > conf.test_duration) {
728             fprintf(stderr, "--statsd-latency-window=%gs exceeds --duration=%gs.\n",
729                 conf.latency_window, conf.test_duration);
730             exit(EX_USAGE);
731         }
732         if(conf.latency_window >= conf.test_duration / 2) {
733             warning("--statsd-latency-window=%gs might result in too few latency reports.\n", conf.latency_window);
734         }
735         if(conf.latency_window < 0.5) {
736             fprintf(stderr, "--statsd-latency-window=%gs is too small. Try 0.5s.\n",
737                 conf.latency_window);
738             exit(EX_USAGE);
739         }
740     }
741 
742     /*
743      * Check that the system environment is prepared to handle high load.
744      */
745     if(adjust_system_limits_for_highload(conf.max_connections,
746                                          engine_params.requested_workers)
747        == -1) {
748         /* Print the full set of problems with system limits. */
749         check_system_limits_sanity(conf.max_connections,
750                                    engine_params.requested_workers);
751         fprintf(stderr, "System limits will not support the expected load.\n");
752         exit(EX_SOFTWARE);
753     } else {
754         /* Check other system limits and print out if they might be too low. */
755         check_system_limits_sanity(conf.max_connections,
756                                    engine_params.requested_workers);
757     }
758 
759     /*
760      * Check whether --rcvbuf and --sndbuf options mean something.
761      * Some sysctl variables may make these options ineffectful.
762      */
763     if(engine_params.sock_rcvbuf_size
764        && check_setsockopt_effect(SO_RCVBUF) == 0) {
765         /* The check_setsockopt_effect() function yelled already. */
766         warning("--rcvbuf option makes no effect.\n");
767     }
768     if(engine_params.sock_sndbuf_size
769        && check_setsockopt_effect(SO_SNDBUF) == 0) {
770         /* The check_setsockopt_effect() function yelled already. */
771         warning("--sndbuf option makes no effect.\n");
772     }
773 
774     /*
775      * Pick multiple destinations from the command line, resolve them.
776      */
777     if(argc - optind > 0) {
778         engine_params.remote_addresses =
779             resolve_remote_addresses(&argv[optind], argc - optind);
780         if(engine_params.remote_addresses.n_addrs == 0) {
781             errx(EX_NOHOST,
782                  "DNS did not return usable addresses for given host(s)");
783         } else {
784             fprint_addresses(stderr, "Destination: ", "\nDestination: ", "\n",
785                              engine_params.remote_addresses);
786         }
787         /* Figure out the host and port for HTTP "Host:" header. */
788         conf.first_hostport = strdup(argv[optind]);
789         conf.first_path = strchr(conf.first_hostport, '/');
790         if(conf.first_path) {
791             *conf.first_path++ = '\0';
792         } else {
793             conf.first_path = ""; /* "GET / HTTP/1.1" */
794         }
795 
796         /* Figure out source IPs */
797         if(engine_params.source_addresses.n_addrs == 0) {
798             if(detect_source_ips(&engine_params.remote_addresses,
799                                  &engine_params.source_addresses)
800                < 0) {
801                 exit(EX_SOFTWARE);
802             }
803         } else {
804             fprint_addresses(stderr, "Source IP: ", "\nSource IP: ", "\n",
805                              engine_params.source_addresses);
806         }
807     } else {
808         conf.max_connections = 0;
809     }
810     if(conf.listen_port > 0) {
811         engine_params.listen_addresses =
812             detect_listen_addresses(conf.listen_host, conf.listen_port);
813     }
814 
815     /*
816      * Add final touches to the collection:
817      * add websocket headers if needed, etc.
818      */
819     message_collection_finalize(
820         &engine_params.message_collection, engine_params.websocket_enable,
821         conf.first_hostport, conf.first_path, conf.http_headers.buffer);
822 
823     int no_message_to_send =
824         (0 == message_collection_estimate_size(
825                   &engine_params.message_collection, MSK_PURPOSE_MESSAGE,
826                   MSK_PURPOSE_MESSAGE, MCE_MINIMUM_SIZE));
827 
828     /*
829      * Message marker mode can be explicitly enabled via --message-marker,
830      * or implicitly via \{message.marker} in the messages to sent.
831      */
832     engine_params.message_marker |= message_collection_has(&engine_params.message_collection, EXPR_MESSAGE_MARKER);
833     if(engine_params.message_marker) {
834         engine_params.latency_setting |= SLT_MARKER;
835         int res = parse_expression(&engine_params.latency_marker_expr, MESSAGE_MARKER_TOKEN, sizeof(MESSAGE_MARKER_TOKEN) - 1, 0);
836         assert(res != -1);
837         assert(EXPR_IS_TRIVIAL(engine_params.latency_marker_expr));
838     }
839 
840     /*
841      * Check that we will actually send messages
842      * if we are also told to measure latency.
843      */
844     if(engine_params.latency_marker_expr && !engine_params.message_marker) {
845         const char *optname = engine_params.message_marker ?
846                             "--message-marker" : "--latency-marker";
847         if(no_message_to_send) {
848             fprintf(stderr,
849                     "%s is given, but no messages "
850                     "are supposed to be sent. Specify --message?\n", optname);
851             exit(EX_USAGE);
852         } else if(argc - optind == 0) {
853             fprintf(stderr,
854                     "%s is given, but no connections "
855                     "are supposed to be initiated. Specify <host:port>?\n",
856                     optname);
857             exit(EX_USAGE);
858         }
859     } else if(rate_modulator.mode != RM_UNMODULATED) {
860         fprintf(stderr,
861                 "--message-rate @<Latency> requires specifying "
862                 "--latency-marker as well.\n");
863         exit(EX_USAGE);
864     }
865 
866     /*
867      * Make sure the message rate makes sense (e.g. the -m param is there).
868      */
869     if((engine_params.channel_send_rate.value_base == RS_MESSAGES_PER_SECOND
870         || rate_modulator.mode != RM_UNMODULATED)
871        && no_message_to_send) {
872         if(message_collection_estimate_size(
873                &engine_params.message_collection, MSK_PURPOSE_MESSAGE,
874                MSK_PURPOSE_MESSAGE, MCE_MAXIMUM_SIZE)
875            > 0) {
876             fprintf(stderr,
877                     "--message may resolve "
878                     "to zero length, double-check regular expression\n");
879         } else {
880             fprintf(stderr,
881                     "--message-rate parameter makes no sense "
882                     "without --message or --message-file\n");
883         }
884         exit(EX_USAGE);
885     }
886 
887     /*
888      * --write-combine=off makes little sense with Nagle on.
889      * Disable Nagle or complain.
890      */
891     if(engine_params.write_combine == WRCOMB_OFF) {
892         switch(engine_params.nagle_setting) {
893         case NSET_UNSET:
894             fprintf(stderr,
895                     "NOTE: --write-combine=off presumes --nagle=off.\n");
896             engine_params.nagle_setting = NSET_NODELAY_OFF;
897             break;
898         case NSET_NODELAY_OFF: /* --nagle=on */
899             warning(
900                 "--write-combine=off makes little sense "
901                 "with --nagle=on.\n");
902             break;
903         case NSET_NODELAY_ON: /* --nagle=off */
904             /* This is the proper setting when --write-combine=off */
905             break;
906         }
907     }
908 
909     if(optind == argc && conf.listen_port == 0) {
910         fprintf(stderr,
911                 "Expecting target <host:port> or --listen-port. See -h or "
912                 "--help.\n");
913         usage_short(argv[0]);
914         exit(EX_USAGE);
915     }
916 
917     /*
918      * Check if the number of connections can be opened in time.
919      */
920     if(conf.max_connections / conf.connect_rate > conf.test_duration / 10) {
921         if(conf.max_connections / conf.connect_rate > conf.test_duration) {
922             fprintf(stderr,
923                     "%d connections can not be opened "
924                     "at a rate %g within test duration %g.\n"
925                     "Decrease --connections=%d, or increase --duration=%g or "
926                     "--connect-rate=%g.\n",
927                     conf.max_connections, conf.connect_rate, conf.test_duration,
928                     conf.max_connections, conf.test_duration,
929                     conf.connect_rate);
930             exit(EX_USAGE);
931         } else {
932             warning(
933                 "%d connections might not be opened "
934                 "at a rate %g within test duration %g.\n"
935                 "Decrease --connections=%d, or increase --duration=%g or "
936                 "--connect-rate=%g.\n",
937                 conf.max_connections, conf.connect_rate, conf.test_duration,
938                 conf.max_connections, conf.test_duration, conf.connect_rate);
939         }
940     }
941 
942     /* Which latency types to report to statsd */
943     statsd_report_latency_types requested_latency_types = engine_params.latency_setting;
944 
945     if(requested_latency_types && !latency_percentiles.size) {
946         static struct percentile_value percentile_values[] = {
947             { 95, "95" }, { 99, "99" }, { 99.5, "99.5" } };
948         static struct percentile_values pvs = {
949                 .size = sizeof(percentile_values) / sizeof(percentile_values[1]),
950                 .values = percentile_values
951             };
952         latency_percentiles = pvs;
953     }
954 
955     /*
956      * Initialize statsd library and push initial (empty) metrics.
957      */
958     Statsd *statsd;
959     if(conf.statsd_enable) {
960         statsd_new(&statsd, conf.statsd_host, conf.statsd_port,
961                    conf.statsd_namespace, NULL);
962         /* Clear up traffic numbers, for better graphing. */
963         report_to_statsd(statsd, 0, requested_latency_types, &latency_percentiles);
964     } else {
965         statsd = 0;
966     }
967 
968     /* Stop flashing cursor in the middle of status reporting. */
969     if(print_stats)
970         tcpkali_disable_cursor();
971 
972     /* Block term signals so they're not scheduled in the worker threads. */
973     block_term_signals();
974 
975     struct engine *eng = engine_start(engine_params);
976 
977     /*
978      * Traffic in/out moving average, smoothing period is 3 seconds.
979      */
980     struct oc_args oc_args = {
981         .eng = eng,
982         .max_connections = conf.max_connections,
983         .connect_rate = conf.connect_rate,
984         .latency_window = conf.latency_window,
985         .statsd = statsd,
986         .rate_modulator = &rate_modulator,
987         .latency_percentiles = &latency_percentiles,
988         .print_stats = print_stats
989     };
990     mavg_init(&oc_args.traffic_mavgs[0], tk_now(TK_DEFAULT), 1.0 / 8, 3.0);
991     mavg_init(&oc_args.traffic_mavgs[1], tk_now(TK_DEFAULT), 1.0 / 8, 3.0);
992     mavg_init(&oc_args.count_mavgs[0], tk_now(TK_DEFAULT), 1.0 / 8, 3.0);
993     mavg_init(&oc_args.count_mavgs[1], tk_now(TK_DEFAULT), 1.0 / 8, 3.0);
994 
995     /*
996      * Convert SIGINT into change of a flag.
997      * Has to be run after all other threads are run, otherwise
998      * a signal can be delivered to a wrong thread.
999      */
1000     flagify_term_signals(&oc_args.term_flag);
1001 
1002     /*
1003      * Ramp up to the specified number of connections by opening them at a
1004      * specifed --connect-rate.
1005      */
1006     if(conf.max_connections) {
1007         oc_args.epoch_end = tk_now(TK_DEFAULT) + conf.test_duration;
1008         if(open_connections_until_maxed_out(PHASE_ESTABLISHING_CONNECTIONS,
1009                 &oc_args) == OC_CONNECTED) {
1010             fprintf(stderr, "%s", tcpkali_clear_eol());
1011             fprintf(stderr, "Ramped up to %d connections.\n",
1012                     conf.max_connections);
1013         } else {
1014             fprintf(stderr, "%s", tcpkali_clear_eol());
1015             fprintf(stderr,
1016                     "Could not create %d connection%s"
1017                     " in allotted time (%gs)\n",
1018                     conf.max_connections, conf.max_connections == 1 ? "" : "s",
1019                     conf.test_duration);
1020             /* Level down graphs/charts. */
1021             report_to_statsd(statsd, 0, requested_latency_types, &latency_percentiles);
1022             exit(1);
1023         }
1024     }
1025 
1026     /*
1027      * Start measuring the steady-state performance, as opposed to
1028      * ramping up and waiting for the connections to be established.
1029      * (initial_traffic_stats) contain traffic numbers accumulated duing
1030      * ramp-up time.
1031      */
1032     oc_args.checkpoint.initial_traffic_stats = engine_traffic(eng);
1033     oc_args.checkpoint.epoch_start = tk_now(TK_DEFAULT);
1034 
1035     /* Reset the test duration after ramp-up. */
1036     oc_args.epoch_end = tk_now(TK_DEFAULT) + conf.test_duration;
1037     enum oc_return_value orv = open_connections_until_maxed_out(
1038                                     PHASE_STEADY_STATE, &oc_args);
1039 
1040     fprintf(stderr, "%s", tcpkali_clear_eol());
1041     engine_terminate(eng, oc_args.checkpoint.epoch_start,
1042                      oc_args.checkpoint.initial_traffic_stats, &latency_percentiles);
1043 
1044     /* Send zeroes, otherwise graphs would continue showing non-zeroes... */
1045     report_to_statsd(statsd, 0, requested_latency_types, &latency_percentiles);
1046 
1047     switch(orv) {
1048     case OC_CONNECTED:
1049         assert(orv != OC_CONNECTED);
1050     /* Fall through */
1051     case OC_INTERRUPT:
1052         exit(EX_USAGE);
1053         break;
1054     case OC_TIMEOUT:
1055         if(rate_modulator.mode != RM_UNMODULATED) {
1056             fprintf(stderr,
1057                     "Failed to find the best --message-rate for latency %s in "
1058                     "-T%g seconds\n",
1059                     rate_modulator.latency_target_s, conf.test_duration);
1060             exit(EX_UNAVAILABLE);
1061         }
1062         break;
1063     case OC_RATE_GOAL_MET:
1064         printf("Best --message-rate for latency %s⁹⁵ᵖ is %g\n",
1065                rate_modulator.latency_target_s,
1066                rate_modulator.suggested_rate_value);
1067         break;
1068     case OC_RATE_GOAL_FAILED:
1069         fprintf(stderr,
1070                 "Best --message-rate for latency %s can not be determined in "
1071                 "time due to unstable target system behavior\n",
1072                 rate_modulator.latency_target_s);
1073         exit(EX_UNAVAILABLE);
1074         break;
1075     }
1076 
1077     return 0;
1078 }
1079 
1080 static double
parse_with_multipliers(const char * option,char * str,struct multiplier * ms,int n)1081 parse_with_multipliers(const char *option, char *str, struct multiplier *ms,
1082                        int n) {
1083     char *endptr;
1084     double value = strtod(str, &endptr);
1085     if(endptr == str) {
1086         return -1;
1087     }
1088     for(; n > 0; n--, ms++) {
1089         if(strcmp(endptr, ms->prefix) == 0) {
1090             value *= ms->mult;
1091             endptr += strlen(endptr);
1092             break;
1093         }
1094     }
1095     if(*endptr) {
1096         fprintf(stderr, "Unknown prefix \"%s\" in %s\n", endptr, str);
1097         return -1;
1098     }
1099     if(!isfinite(value)) {
1100         fprintf(stderr, "Option %s parses to infinite value\n", option);
1101         return -1;
1102     }
1103     return value;
1104 }
1105 
1106 static int
parse_percentile_values(const char * option,char * str,struct percentile_values * array)1107 parse_percentile_values(const char *option, char *str,
1108                        struct percentile_values *array) {
1109     struct percentile_value *values = array->values;
1110     size_t size = array->size;
1111 
1112     for(char *pos = str; *pos; pos++) {
1113         const char *start = pos;
1114         char *endpos;
1115 
1116         double value_d = strtod(start, &endpos);
1117 
1118         if(start == endpos) {
1119             fprintf(stderr, "--%s: %s: Failed to parse: bad number\n", option, str);
1120             return -1;
1121         }
1122         if(value_d < 0 || !isfinite(value_d) || value_d > 100) {
1123             fprintf(stderr, "--%s: %s: Failed to parse: bad latency percentile specification"
1124                             ", expected range is [0..100]\n", option, str);
1125             return -1;
1126         }
1127         for(; pos < endpos; pos++) {
1128             switch(*pos) {
1129             case '+': case ' ':
1130                 assert(pos == start);
1131                 start++;
1132                 continue;
1133             case '.':
1134                 if(endpos - pos == 1) {
1135                     fprintf(stderr, "--%s: %s: Fractional part is missing\n", option, str);
1136                     return -1;
1137                 }
1138                 continue;
1139             case '0' ... '9':
1140                 continue;
1141             default:
1142                 fprintf(stderr, "--%s: %s: Number should contain only digits\n", option, str);
1143                 return -1;
1144             }
1145             break;
1146         }
1147         if(endpos - start >= (ssize_t)sizeof(values[0].value_s)) {
1148             fprintf(stderr, "--%s: %s: Unreasonably precise percentile specification\n", option, str);
1149             return -1;
1150         }
1151 
1152         if(*endpos != 0 && *endpos != ',' && *endpos != '/') {
1153             fprintf(stderr,
1154                     "--%s: %s: Failed to parse: "
1155                     "invalid separator, use ',' or '/'\n",
1156                     option, str);
1157             return -1;
1158         }
1159 
1160         values = realloc(values, ++size * sizeof(values[0]));
1161         assert(values);
1162         values[size - 1].value_d = value_d;
1163         memcpy(values[size - 1].value_s, start, endpos-start);
1164         values[size - 1].value_s[endpos - start] = '\0';
1165 
1166         pos = endpos;
1167         if(*pos == 0) break;
1168     }
1169 
1170     array->values = values;
1171     array->size = size;
1172     return 0;
1173 }
1174 
1175 /*
1176  * Display the Usage screen.
1177  */
1178 static void
usage_long(char * argv0,struct tcpkali_config * conf)1179 usage_long(char *argv0, struct tcpkali_config *conf) {
1180     fprintf(stdout, "Usage: %s [OPTIONS] [-l <port>] [<host:port>...]\n",
1181             basename(argv0));
1182     /* clang-format off */
1183     fprintf(stdout,
1184     "Where OPTIONS are:\n"
1185     "  -h                           Print short help screen, then exit\n"
1186     "  --help                       Print this help screen, then exit\n"
1187     "  --version                    Print version number, then exit\n"
1188     "  -v, --verbose <level=1>      Increase (-v) or set verbosity level [0..%d]\n"
1189     "  -d, --dump-one               Dump i/o data for a single connection\n"
1190     "  --dump-one-in                Dump incoming data for a single connection\n"
1191     "  --dump-one-out               Dump outgoing data for a single connection\n"
1192     "  --dump-{all,all-in,all-out}  Dump i/o data for all connections\n"
1193     "  --nagle {on|off}             Control Nagle algorithm (set TCP_NODELAY)\n"
1194     "  --rcvbuf <SizeBytes>         Set TCP receive buffers (set SO_RCVBUF)\n"
1195     "  --sndbuf <SizeBytes>         Set TCP send buffers (set SO_SNDBUF)\n"
1196     "  --source-ip <IP>             Use the specified IP address to connect\n"
1197     "  --write-combine off          Disable batching adjacent writes\n"
1198     "  -w, --workers <N=%ld>%s         Number of parallel threads to use\n"
1199     "\n"
1200     "  --ws, --websocket            Use RFC6455 WebSocket transport\n"
1201     "  -H, --header <string>        Add HTTP header into WebSocket handshake\n"
1202     "  -c, --connections <N=%d>      Connections to keep open to the destinations\n"
1203     "  --connect-rate <Rate=%g>    Limit number of new connections per second\n"
1204     "  --connect-timeout <Time=1s>  Limit time spent in a connection attempt\n"
1205     "  --channel-lifetime <Time>    Shut down each connection after Time seconds\n"
1206     "  --channel-bandwidth-upstream <Bandwidth>     Limit upstream bandwidth\n"
1207     "  --channel-bandwidth-downstream <Bandwidth>   Limit downstream bandwidth\n"
1208     "  -l, --listen-port <port>     Listen on the specified port\n"
1209     "  --listen-mode=<mode>         What to do upon client connect, where <mode> is:\n"
1210     "               \"silent\"        Do not send data, ignore received data (default)\n"
1211     "               \"active\"        Actively send messages\n"
1212     "  -T, --duration <Time=10s>    Exit after the specified amount of time\n"
1213     "\n"
1214     "  -e, --unescape-message-args  Unescape the message data arguments\n"
1215     "  -1, --first-message <string> Send this message first, once\n"
1216     "  --first-message-file <name>  Read the first message from a file\n"
1217     "  -m, --message <string>       Message to repeatedly send to the remote\n"
1218     "  -f, --message-file <name>    Read message to send from a file\n"
1219     "  -r, --message-rate <Rate>    Messages per second to send in a connection\n"
1220     "  -r, --message-rate @<Latency> Measure a message rate at a given latency\n"
1221     "  --message-stop <string>      Abort if this string is found in received data\n"
1222     "\n"
1223     "  --latency-connect            Measure TCP connection establishment latency\n"
1224     "  --latency-first-byte         Measure time to first byte latency\n"
1225     "  --latency-marker <string>    Measure latency using a per-message marker\n"
1226     "  --latency-marker-skip <N>    Ignore the first N occurrences of a marker\n"
1227     "  --latency-percentiles <list> Report latency at specified percentiles\n"
1228     "  --message-marker             Parse markers to calculate latency\n"
1229     "\n"
1230     "  --statsd                     Enable StatsD output (default %s)\n"
1231     "  --statsd-host <host>         StatsD host to send data (default is localhost)\n"
1232     "  --statsd-port <port>         StatsD port to use (default is %d)\n"
1233     "  --statsd-namespace <string>  Metric namespace (default is \"%s\")\n"
1234     "  --statsd-latency-window <T>  Aggregate latencies in discrete windows\n"
1235     "\n"
1236     "Variable units and recognized multipliers:\n"
1237     "  <N>, <Rate>:  k (1000, as in \"5k\" is 5000), m (1000000)\n"
1238     "  <SizeBytes>:  k (1024, as in \"5k\" is 5120), m (1024*1024)\n"
1239     "  <Bandwidth>:  kbps, Mbps (bits per second), kBps, MBps (bytes per second)\n"
1240     "  <Time>, <Latency>:  ms, s, m, h, d (milliseconds, seconds, minutes, etc)\n"
1241     "  <Rate>, <Time> and <Latency> can be fractional values, such as 0.25.\n",
1242         /* clang-format on */
1243 
1244         (_DBG_MAX - 1), number_of_cpus(), number_of_cpus() < 10 ? " " : "",
1245         conf->max_connections, conf->connect_rate,
1246         conf->statsd_enable ? "enabled" : "disabled", conf->statsd_port,
1247         conf->statsd_namespace);
1248 }
1249 
1250 static void
usage_short(char * argv0)1251 usage_short(char *argv0) {
1252     if(isatty(fileno(stdout))) {
1253         tcpkali_init_terminal();
1254     }
1255 
1256     fprintf(stdout, "Usage: %s [OPTIONS] [-l <port>] [<host:port>...]\n",
1257             basename(argv0));
1258     fprintf(stdout,
1259         /* clang-format off */
1260     "Where some OPTIONS are:\n"
1261     "  -h                   Print this help screen, then exit\n"
1262     "  %s--help%s               Print long help screen, then exit\n"
1263     "  -d                   Dump i/o data for a single connection\n"
1264     "\n"
1265     "  -c <N>               Connections to keep open to the destinations\n"
1266     "  -l <port>            Listen on the specified port\n"
1267     "  --ws, --websocket    Use RFC6455 WebSocket transport\n"
1268     "  -T <Time=10s>        Exit after the specified amount of time\n"
1269     "\n"
1270     "  -e                   Unescape backslash-escaping in a message string\n"
1271     "  -1 <string>          Message to send to the remote host once\n"
1272     "  -m <string>          Message to repeatedly send to the remote\n"
1273     "  -r <Rate>            Messages per second to send in a connection\n"
1274     "\n"
1275     "Variable units and recognized multipliers:\n"
1276     "  <N>, <Rate>:  k (1000, as in \"5k\" is 5000), m (1000000)\n"
1277     "  <Time>:       ms, s, m, h, d (milliseconds, seconds, minutes, hours, days)\n"
1278     "  <Rate> and <Time> can be fractional values, such as 0.25.\n"
1279     "\n"
1280     "Use `%s %s--help%s` or `man tcpkali` for a %sfull set%s of supported options.\n",
1281     tk_attr(TKA_HIGHLIGHT),
1282     tk_attr(TKA_NORMAL),
1283     basename(argv0),
1284     tk_attr(TKA_HIGHLIGHT),
1285     tk_attr(TKA_NORMAL),
1286     tk_attr(TKA_HIGHLIGHT),
1287     tk_attr(TKA_NORMAL)
1288     );
1289     /* clang-format on */
1290 }
1291