1 /*
2 ** Copyright (C) 2001-2020 by Carnegie Mellon University.
3 **
4 ** @OPENSOURCE_LICENSE_START@
5 ** See license information in ../../LICENSE.txt
6 ** @OPENSOURCE_LICENSE_END@
7 */
8
9 /*
10 * addrcount.c
11 *
12 * this is the last major rwset tool I want to write, it takes in two streams
13 * of data, an rwset and an rwfilter stream. From this data, it then generates
14 * a result - one of three outputs:
15 * totals (default) - outputs to screen a table containing the
16 * ip address, bytes, packets, records
17 * print-ips - outputs to screen the ip addresses
18 * set-file - outputs to screen the set data.
19 *
20 * So the reason for the second two is because I'm including three thresholds
21 * here - bytes, packets & records.
22 *
23 * 12/2 notes. Often, the best is the enemy of the good, I am implementing
24 * a simple version of this application for now with the long-term plan being
25 * that I am going to write a better, faster, set-friendly version later.
26 */
27
28 #include <silk/silk.h>
29
30 RCSIDENT("$SiLK: rwaddrcount.c ef14e54179be 2020-04-14 21:57:45Z mthomas $");
31
32 #include <silk/iptree.h>
33 #include <silk/rwrec.h>
34 #include <silk/skipaddr.h>
35 #include <silk/sksite.h>
36 #include <silk/skstream.h>
37 #include <silk/skstringmap.h>
38 #include <silk/skvector.h>
39 #include <silk/utils.h>
40
41
42 /* LOCAL DEFINES AND TYPEDEFS */
43
44 /* where to write output from --help */
45 #define USAGE_FH stdout
46
47 /* number of buckets in hash table */
48 #define RWAC_ARRAYSIZE 50865917
49
50 /* number of countRecord_t to allocate at one time */
51 #define RWAC_BLOCK_SIZE 4096
52
53 /*
54 * Get the IP address from the record 'r' to use as the key. Uses the
55 * global variable 'use_dest'
56 */
57 #define GETIP(r) \
58 ((use_dest) ? rwRecGetDIPv4(r) : rwRecGetSIPv4(r))
59
60 /*
61 * One of the problems I've had recently is a tendency to focus on
62 * doing the perfect hash. The perfect hash is nice, but what we need
63 * RIGHT HERE RIGHT NOW is a tool that'll actually do the job. Enough
64 * mathematical wanking.
65 *
66 * So, this is a hash whose collision compensation algorithm is linear
67 * chaining. I'm not happy about it, but it'll do for now.
68 */
69 #define HASHFUNC(value) \
70 ((value ^ (value >> 7) ^ (value << 23)) % RWAC_ARRAYSIZE)
71
72 /*
73 * CMPFNC: return TRUE if IP on the rwRec 'r' matches the IP stored in
74 * the countRecord_t 'b'.
75 */
76 #define CMPFNC(r, cr) \
77 ((cr)->cr_key == GETIP(r))
78
79 /*
80 * when generating output, this macro will evaluate to TRUE if the
81 * record is within the limits given by the user and should be
82 * printed/counted/used-to-genrate-output. This macro uses the global
83 * limit variables.
84 */
85 #define IS_RECORD_WITHIN_LIMITS(count_rec) \
86 ((count_rec)->cr_bytes >= min_bytes && \
87 (count_rec)->cr_packets >= min_packets && \
88 (count_rec)->cr_records >= min_records && \
89 (count_rec)->cr_bytes <= max_bytes && \
90 (count_rec)->cr_packets <= max_packets && \
91 (count_rec)->cr_records <= max_records)
92
93
94 /* formats for printing records */
95 #define FMT_REC_VALUE \
96 "%*s%c%*" PRIu64 "%c%*" PRIu64 "%c%*" PRIu64 "%c%*s%c%*s%s\n"
97 #define FMT_REC_TITLE "%*s%c%*s%c%*s%c%*s%c%*s%c%*s%s\n"
98 #define FMT_REC_WIDTH {15, 20, 10, 10, 20, 20}
99
100 /* formats for printing statistics */
101 #define FMT_STAT_VALUE \
102 "%*s%c%*" PRIu32 "%c%*" PRIu64 "%c%*" PRIu64 "%c%*" PRIu64 "%s\n"
103 #define FMT_STAT_TITLE "%*s%c%*s%c%*s%c%*s%c%*s%s\n"
104 #define FMT_STAT_WIDTH {10, 10, 20, 15, 15}
105
106 typedef struct countRecord_st countRecord_t;
107 struct countRecord_st {
108 /* total number of bytes */
109 uint64_t cr_bytes;
110 /* total number of packets */
111 uint64_t cr_packets;
112 /* total number of records */
113 uint64_t cr_records;
114 /* Pointer to the next record for collision */
115 countRecord_t *cr_next;
116 /* IP address; source or dest does not matter here */
117 uint32_t cr_key;
118 /* start time */
119 uint32_t cr_start;
120 /* end time */
121 uint32_t cr_end;
122 };
123
124 typedef enum {
125 RWAC_PMODE_NONE=0,
126 RWAC_PMODE_IPS,
127 RWAC_PMODE_RECORDS,
128 RWAC_PMODE_STAT,
129 RWAC_PMODE_IPSETFILE,
130 RWAC_PMODE_SORTED_RECORDS,
131 RWAC_PMODE_SORTED_IPS
132 } rwac_print_mode_t;
133
134
135 /* LOCAL VARIABLES */
136
137 /* for looping over files on the command line */
138 static sk_options_ctx_t *optctx;
139
140 /* output mode */
141 static rwac_print_mode_t print_mode = RWAC_PMODE_NONE;
142
143 /* user-specified limits of bins to print */
144 static uint64_t min_bytes = 0;
145 static uint64_t max_bytes = UINT64_MAX;
146 static uint64_t min_packets = 0;
147 static uint64_t max_packets = UINT64_MAX;
148 static uint64_t min_records = 0;
149 static uint64_t max_records = UINT64_MAX;
150
151 /* the hash table */
152 static countRecord_t *hash_bins[RWAC_ARRAYSIZE];
153
154 /* IPset file for output when --set-file is specified */
155 static const char *ipset_file = NULL;
156
157 /* entries are allocated in blocks; the blocks are stored in a
158 * vector */
159 static sk_vector_t *mem_vector = NULL;
160
161 /* whether to use the source(==0) or destination(==1) IPs */
162 static uint8_t use_dest = 0;
163
164 /* output mode for IPs */
165 static uint32_t ip_format = SKIPADDR_CANONICAL;
166 static uint8_t sort_ips_flag = 0;
167
168 /* flags when registering --ip-format */
169 static const unsigned int ip_format_register_flags =
170 (SK_OPTION_IP_FORMAT_INTEGER_IPS | SK_OPTION_IP_FORMAT_ZERO_PAD_IPS);
171
172 /* whether to suppress column titles; default no (i.e. print titles) */
173 static uint8_t no_titles = 0;
174
175 /* whether to suppress columnar output; default no (i.e. columnar) */
176 static uint8_t no_columns = 0;
177
178 /* whether to suppress the final delimiter; default no (i.e. end with '|') */
179 static uint8_t no_final_delimiter = 0;
180
181 /* column separator */
182 static char delimiter = '|';
183
184 /* what to print at the end of the line */
185 static char final_delim[] = {'\0', '\0'};
186
187 /* flags to pass to sktimestamp_r() */
188 static uint32_t time_flags = 0;
189
190 /* flags when registering --timestamp-format */
191 static const uint32_t time_register_flags =
192 (SK_OPTION_TIMESTAMP_NEVER_MSEC | SK_OPTION_TIMESTAMP_OPTION_LEGACY);
193
194 /* where to write output */
195 static sk_fileptr_t output;
196
197 /* name of program to run to page output */
198 static char *pager = NULL;
199
200
201 /* OPTIONS */
202
203 /* Names of options; keep the order in sync with appOptions[] */
204 typedef enum {
205 OPT_PRINT_RECORDS,
206 OPT_PRINT_STAT,
207 OPT_PRINT_IPS,
208 OPT_USE_DEST,
209 OPT_MIN_BYTES,
210 OPT_MIN_PACKETS,
211 OPT_MIN_RECORDS,
212 OPT_MAX_BYTES,
213 OPT_MAX_PACKETS,
214 OPT_MAX_RECORDS,
215 OPT_SET_FILE,
216 OPT_SORT_IPS,
217 OPT_NO_TITLES,
218 OPT_NO_COLUMNS,
219 OPT_COLUMN_SEPARATOR,
220 OPT_NO_FINAL_DELIMITER,
221 OPT_DELIMITED,
222 OPT_OUTPUT_PATH,
223 OPT_PAGER
224 } appOptionsEnum;
225
226 static struct option appOptions[] = {
227 {"print-recs", NO_ARG, 0, OPT_PRINT_RECORDS},
228 {"print-stat", NO_ARG, 0, OPT_PRINT_STAT},
229 {"print-ips", NO_ARG, 0, OPT_PRINT_IPS},
230 {"use-dest", NO_ARG, 0, OPT_USE_DEST},
231 {"min-bytes", REQUIRED_ARG, 0, OPT_MIN_BYTES},
232 {"min-packets", REQUIRED_ARG, 0, OPT_MIN_PACKETS},
233 {"min-records", REQUIRED_ARG, 0, OPT_MIN_RECORDS},
234 {"max-bytes", REQUIRED_ARG, 0, OPT_MAX_BYTES},
235 {"max-packets", REQUIRED_ARG, 0, OPT_MAX_PACKETS},
236 {"max-records", REQUIRED_ARG, 0, OPT_MAX_RECORDS},
237 {"set-file", REQUIRED_ARG, 0, OPT_SET_FILE},
238 {"sort-ips", NO_ARG, 0, OPT_SORT_IPS},
239 {"no-titles", NO_ARG, 0, OPT_NO_TITLES},
240 {"no-columns", NO_ARG, 0, OPT_NO_COLUMNS},
241 {"column-separator", REQUIRED_ARG, 0, OPT_COLUMN_SEPARATOR},
242 {"no-final-delimiter", NO_ARG, 0, OPT_NO_FINAL_DELIMITER},
243 {"delimited", OPTIONAL_ARG, 0, OPT_DELIMITED},
244 {"output-path", REQUIRED_ARG, 0, OPT_OUTPUT_PATH},
245 {"pager", REQUIRED_ARG, 0, OPT_PAGER},
246 {0,0,0,0} /* sentinel entry */
247 };
248
249 static const char *appHelp[] = {
250 "Print summary byte, packet, flow counts per IP bin",
251 "Print statistics (total bytes, packets, flows, unique IPs)",
252 "Print IP addresses only to stdout",
253 "Use destination IP address as key. Def. Source address",
254 ("Do not print IPs when sum has less than this many total\n"
255 "\tbytes. Def. 1"),
256 ("Do not print IPs when sum has less than this many total\n"
257 "\tpackets. Def. 1"),
258 ("Do not print IPs when sum has less than this many total\n"
259 "\trecords. Def. 1"),
260 ("Do not print IPs when sum has more than this many total\n"
261 "\tbytes. Def. 18446744073709551615"),
262 ("Do not print IPs when sum has more than this many total\n"
263 "\tpackets. Def. 4294967295"),
264 ("Do not print IPs when sum has more than this many total\n"
265 "\trecords. Def. 4294967295"),
266 "Write IPs to specified binary IPset file. Def. No",
267 "When printing results, sort by IP address. Def. No",
268 "Do not print column titles. Def. Print titles",
269 "Disable fixed-width columnar output. Def. Columnar",
270 "Use specified character between columns. Def. '|'",
271 "Suppress column delimiter at end of line. Def. No",
272 "Shortcut for --no-columns --no-final-del --column-sep=CHAR",
273 "Write the output to this stream or file. Def. stdout",
274 "Invoke this program to page output. Def. $SILK_PAGER or $PAGER",
275 (char *)NULL
276 };
277
278 static struct option legacyOptions[] = {
279 {"byte-min", REQUIRED_ARG, 0, OPT_MIN_BYTES},
280 {"packet-min", REQUIRED_ARG, 0, OPT_MIN_PACKETS},
281 {"rec-min", REQUIRED_ARG, 0, OPT_MIN_RECORDS},
282 {"byte-max", REQUIRED_ARG, 0, OPT_MAX_BYTES},
283 {"packet-max", REQUIRED_ARG, 0, OPT_MAX_PACKETS},
284 {"rec-max", REQUIRED_ARG, 0, OPT_MAX_RECORDS},
285 {0,0,0,0} /* sentinel entry */
286 };
287
288
289 /* LOCAL FUNCTION PROTOTYPES */
290
291 static int appOptionsHandler(clientData cData, int opt_index, char *opt_arg);
292
293
294 /* FUNCTION DEFINITIONS */
295
296 /*
297 * appUsageLong();
298 *
299 * Print complete usage information to USAGE_FH. Pass this
300 * function to skOptionsSetUsageCallback(); skOptionsParse() will
301 * call this funciton and then exit the program when the --help
302 * option is given.
303 */
304 static void
appUsageLong(void)305 appUsageLong(
306 void)
307 {
308 #define USAGE_MSG \
309 ("{--print-recs|--print-stat|--print-ips} [SWITCHES] [FILES]\n" \
310 "\tSummarize SiLK Flow records by source or destination IP; with\n" \
311 "\tthe --print-recs option will produce textual output with counts of\n" \
312 "\tbytes, packets, and flow records for each IP, and the time range\n" \
313 "\twhen the IP was active. When no files are given on command line,\n" \
314 "\tflows are read from STDIN.\n")
315
316 FILE *fh = USAGE_FH;
317 int i;
318
319 fprintf(fh, "%s %s", skAppName(), USAGE_MSG);
320 fprintf(fh, "\nSWITCHES:\n");
321 skOptionsDefaultUsage(fh);
322
323 for (i = 0; appOptions[i].name; ++i) {
324 fprintf(fh, "--%s %s. ", appOptions[i].name,
325 SK_OPTION_HAS_ARG(appOptions[i]));
326 switch (appOptions[i].val) {
327 case OPT_SET_FILE:
328 fprintf(fh, "%s\n", appHelp[i]);
329 /* print help for --timestamp-format */
330 skOptionsTimestampFormatUsage(fh);
331 /* print help for --ip-format */
332 skOptionsIPFormatUsage(fh);
333 break;
334 default:
335 /* Simple static help text from the appHelp array */
336 fprintf(fh, "%s\n", appHelp[i]);
337 break;
338 }
339 }
340 skOptionsCtxOptionsUsage(optctx, fh);
341 sksiteOptionsUsage(fh);
342
343 fprintf(fh, "\nDEPRECATED SWITCHES:\n");
344 for (i = 0; legacyOptions[i].name; ++i) {
345 fprintf(fh, "--%s %s. Deprecated alias for --%s\n",
346 legacyOptions[i].name, SK_OPTION_HAS_ARG(legacyOptions[i]),
347 appOptions[legacyOptions[i].val].name);
348 }
349 }
350
351
352 /*
353 * appTeardown()
354 *
355 * Teardown all modules, close all files, and tidy up all
356 * application state.
357 *
358 * This function is idempotent.
359 */
360 static void
appTeardown(void)361 appTeardown(
362 void)
363 {
364 /* #define RWAC_MAX_CHAIN 64 */
365
366 static int teardownFlag = 0;
367 countRecord_t *bin;
368 uint32_t i;
369 #ifdef RWAC_MAX_CHAIN
370 uint32_t chained[RWAC_MAX_CHAIN];
371 uint32_t chain_len;
372 #endif
373
374 if (teardownFlag) {
375 return;
376 }
377 teardownFlag = 1;
378
379 /* close the output file or process */
380 if (output.of_name) {
381 skFileptrClose(&output, &skAppPrintErr);
382 }
383
384 /* close the copy-stream */
385 skOptionsCtxCopyStreamClose(optctx, &skAppPrintErr);
386
387 #ifdef RWAC_MAX_CHAIN
388 /* print chain length of each bucket */
389 memset(chained, 0, sizeof(chained));
390 chain_len = 0;
391
392 for (i = 0; i < RWAC_ARRAYSIZE; ++i) {
393 if (NULL != hash_bins[i]) {
394 bin = hash_bins[i];
395 chain_len = 0;
396 do {
397 ++chain_len;
398 bin = bin->cr_next;
399 } while (bin != hash_bins[i]);
400 if (chain_len < RWAC_MAX_CHAIN) {
401 ++chained[chain_len];
402 } else {
403 ++chained[RWAC_MAX_CHAIN-1];
404 }
405 }
406 }
407
408 if (chain_len) {
409 fprintf(stderr, "Hash Chaining Information\n%10s|%10s|\n",
410 "Length", "Count");
411 }
412 for (i = 0; i < RWAC_MAX_CHAIN; ++i) {
413 if (chained[i]) {
414 fprintf(stderr, ("%10" PRIu32 "|%10" PRIu32 "|\n"),
415 i, chained[i]);
416 }
417 }
418 #endif /* RWAC_MAX_CHAIN */
419
420 if (mem_vector) {
421 for (i = 0; 0 == skVectorGetValue(&bin, mem_vector, i); ++i) {
422 free(bin);
423 }
424 skVectorDestroy(mem_vector);
425 }
426
427 skOptionsCtxDestroy(&optctx);
428 skAppUnregister();
429 }
430
431
432 /*
433 * appSetup(argc, argv);
434 *
435 * Perform all the setup for this application include setting up
436 * required modules, parsing options, etc. This function should be
437 * passed the same arguments that were passed into main().
438 *
439 * Returns to the caller if all setup succeeds. If anything fails,
440 * this function will cause the application to exit with a FAILURE
441 * exit status.
442 */
443 static void
appSetup(int argc,char ** argv)444 appSetup(
445 int argc,
446 char **argv)
447 {
448 SILK_FEATURES_DEFINE_STRUCT(features);
449 unsigned int optctx_flags;
450 int rv;
451
452 /* verify same number of options and help strings */
453 assert((sizeof(appHelp)/sizeof(char *)) ==
454 (sizeof(appOptions)/sizeof(struct option)));
455
456 /* register the application */
457 skAppRegister(argv[0]);
458 skAppVerifyFeatures(&features, NULL);
459 skOptionsSetUsageCallback(&appUsageLong);
460
461 /* initialize globals */
462 use_dest = 0;
463 memset(&output, 0, sizeof(output));
464 output.of_fp = stdout;
465
466 optctx_flags = (SK_OPTIONS_CTX_INPUT_SILK_FLOW | SK_OPTIONS_CTX_ALLOW_STDIN
467 | SK_OPTIONS_CTX_XARGS | SK_OPTIONS_CTX_PRINT_FILENAMES
468 | SK_OPTIONS_CTX_COPY_INPUT);
469
470 /* register the options */
471 if (skOptionsCtxCreate(&optctx, optctx_flags)
472 || skOptionsCtxOptionsRegister(optctx)
473 || skOptionsRegister(appOptions, &appOptionsHandler, NULL)
474 || skOptionsRegister(legacyOptions, &appOptionsHandler, NULL)
475 || skOptionsTimestampFormatRegister(&time_flags, time_register_flags)
476 || skOptionsIPFormatRegister(&ip_format, ip_format_register_flags)
477 || sksiteOptionsRegister(SK_SITE_FLAG_CONFIG_FILE))
478 {
479 skAppPrintErr("Unable to register options");
480 exit(EXIT_FAILURE);
481 }
482
483 /* register the teardown handler */
484 if (atexit(appTeardown) < 0) {
485 skAppPrintErr("Unable to register appTeardown() with atexit()");
486 appTeardown();
487 exit(EXIT_FAILURE);
488 }
489
490 /* parse options */
491 rv = skOptionsCtxOptionsParse(optctx, argc, argv);
492 if (rv < 0) {
493 skAppUsage();/* never returns */
494 }
495
496 /* try to load site config file; if it fails, we will not be able
497 * to resolve flowtype and sensor from input file names */
498 sksiteConfigure(0);
499
500 /* handle the final delimiter */
501 if (!no_final_delimiter) {
502 final_delim[0] = delimiter;
503 }
504
505 if (print_mode == RWAC_PMODE_NONE) {
506 skAppPrintErr("Must specify --%s, --%s, --%s, or --%s",
507 appOptions[OPT_PRINT_RECORDS].name,
508 appOptions[OPT_PRINT_STAT].name,
509 appOptions[OPT_PRINT_IPS].name,
510 appOptions[OPT_SET_FILE].name);
511 skAppUsage();
512 }
513
514 /* verify that the bounds make sense */
515 if (min_bytes > max_bytes) {
516 skAppPrintErr(("The %s value is greater than %s:"
517 " %" PRIu64 " > %" PRIu64),
518 appOptions[OPT_MIN_BYTES].name,
519 appOptions[OPT_MAX_BYTES].name,
520 min_bytes, max_bytes);
521 exit(EXIT_FAILURE);
522 }
523 if (min_packets > max_packets) {
524 skAppPrintErr(("The %s value is greater than %s:"
525 " %" PRIu64 " > %" PRIu64),
526 appOptions[OPT_MIN_PACKETS].name,
527 appOptions[OPT_MAX_PACKETS].name,
528 min_packets, max_packets);
529 exit(EXIT_FAILURE);
530 }
531 if (min_records > max_records) {
532 skAppPrintErr(("The %s value is greater than %s:"
533 " %" PRIu64 " > %" PRIu64),
534 appOptions[OPT_MIN_RECORDS].name,
535 appOptions[OPT_MAX_RECORDS].name,
536 min_records, max_records);
537 exit(EXIT_FAILURE);
538 }
539
540 /* Do they want the IPs in sorted order? */
541 if (sort_ips_flag) {
542 switch (print_mode) {
543 case RWAC_PMODE_IPS:
544 print_mode = RWAC_PMODE_SORTED_IPS;
545 break;
546 case RWAC_PMODE_RECORDS:
547 print_mode = RWAC_PMODE_SORTED_RECORDS;
548 break;
549 default:
550 /* skAppPrintErr("--sort-ips switch ignored"); */
551 break;
552 }
553 }
554
555 /* make certain stdout is not being used for multiple outputs */
556 if (skOptionsCtxCopyStreamIsStdout(optctx)) {
557 if ((NULL == output.of_name)
558 || (0 == strcmp(output.of_name, "-"))
559 || (0 == strcmp(output.of_name, "stdout")))
560 {
561 skAppPrintErr("May not use stdout for multiple output streams");
562 exit(EXIT_FAILURE);
563 }
564 }
565
566 /* create vector */
567 mem_vector = skVectorNew(sizeof(countRecord_t*));
568 if (NULL == mem_vector) {
569 skAppPrintErr("Cannot create vector");
570 exit(EXIT_FAILURE);
571 }
572
573 /* open the --output-path. the 'of_name' member is NULL if user
574 * didn't get an output-path. */
575 if (output.of_name) {
576 rv = skFileptrOpen(&output, SK_IO_WRITE);
577 if (rv) {
578 skAppPrintErr("Cannot open '%s': %s",
579 output.of_name, skFileptrStrerror(rv));
580 exit(EXIT_FAILURE);
581 }
582 }
583
584 /* looks good, open the --copy-input destination */
585 if (skOptionsCtxOpenStreams(optctx, &skAppPrintErr)) {
586 exit(EXIT_FAILURE);
587 }
588
589 return; /* OK */
590 }
591
592
593 /*
594 * status = appOptionsHandler(cData, opt_index, opt_arg);
595 *
596 * Called by skOptionsParse(), this handles a user-specified switch
597 * that the application has registered, typically by setting global
598 * variables. Returns 1 if the switch processing failed or 0 if it
599 * succeeded. Returning a non-zero from from the handler causes
600 * skOptionsParse() to return a negative value.
601 *
602 * The clientData in 'cData' is typically ignored; 'opt_index' is
603 * the index number that was specified as the last value for each
604 * struct option in appOptions[]; 'opt_arg' is the user's argument
605 * to the switch for options that have a REQUIRED_ARG or an
606 * OPTIONAL_ARG.
607 */
608 static int
appOptionsHandler(clientData UNUSED (cData),int opt_index,char * opt_arg)609 appOptionsHandler(
610 clientData UNUSED(cData),
611 int opt_index,
612 char *opt_arg)
613 {
614 int rv;
615
616 switch ((appOptionsEnum)opt_index) {
617 case OPT_USE_DEST:
618 use_dest = 1;
619 break;
620
621 case OPT_MIN_BYTES:
622 rv = skStringParseUint64(&min_bytes, opt_arg, 0, 0);
623 if (rv) {
624 goto PARSE_ERROR;
625 }
626 break;
627
628 case OPT_MAX_BYTES:
629 rv = skStringParseUint64(&max_bytes, opt_arg, 0, 0);
630 if (rv) {
631 goto PARSE_ERROR;
632 }
633 break;
634
635 case OPT_MIN_PACKETS:
636 rv = skStringParseUint64(&min_packets, opt_arg, 0, 0);
637 if (rv) {
638 goto PARSE_ERROR;
639 }
640 break;
641
642 case OPT_MAX_PACKETS:
643 rv = skStringParseUint64(&max_packets, opt_arg, 0, 0);
644 if (rv) {
645 goto PARSE_ERROR;
646 }
647 break;
648
649 case OPT_MIN_RECORDS:
650 rv = skStringParseUint64(&min_records, opt_arg, 0, 0);
651 if (rv) {
652 goto PARSE_ERROR;
653 }
654 break;
655
656 case OPT_MAX_RECORDS:
657 rv = skStringParseUint64(&max_records, opt_arg, 0, 0);
658 if (rv) {
659 goto PARSE_ERROR;
660 }
661 break;
662
663 case OPT_PRINT_STAT:
664 print_mode = RWAC_PMODE_STAT;
665 break;
666
667 case OPT_PRINT_IPS:
668 print_mode = RWAC_PMODE_IPS;
669 break;
670
671 case OPT_PRINT_RECORDS:
672 print_mode = RWAC_PMODE_RECORDS;
673 break;
674
675 case OPT_SET_FILE:
676 print_mode = RWAC_PMODE_IPSETFILE;
677 ipset_file = opt_arg;
678 break;
679
680 case OPT_SORT_IPS:
681 sort_ips_flag = 1;
682 break;
683
684 case OPT_NO_TITLES:
685 no_titles = 1;
686 break;
687
688 case OPT_NO_COLUMNS:
689 no_columns = 1;
690 break;
691
692 case OPT_COLUMN_SEPARATOR:
693 delimiter = opt_arg[0];
694 break;
695
696 case OPT_NO_FINAL_DELIMITER:
697 no_final_delimiter = 1;
698 break;
699
700 case OPT_DELIMITED:
701 no_columns = 1;
702 no_final_delimiter = 1;
703 if (opt_arg) {
704 delimiter = opt_arg[0];
705 }
706 break;
707
708 case OPT_OUTPUT_PATH:
709 if (output.of_name) {
710 skAppPrintErr("Invalid %s: Switch used multiple times",
711 appOptions[opt_index].name);
712 return 1;
713 }
714 output.of_name = opt_arg;
715 break;
716
717 case OPT_PAGER:
718 pager = opt_arg;
719 break;
720 }
721
722 return 0; /* OK */
723
724 PARSE_ERROR:
725 skAppPrintErr("Invalid %s '%s': %s",
726 appOptions[opt_index].name, opt_arg,
727 skStringParseStrerror(rv));
728 return 1;
729 }
730
731
732 /*
733 * void addToBin(countRecord_t *bin, rwRec *rwrec)
734 *
735 * Adds the contents of a record to the values stored in a bin.
736 */
737 static void
addToBin(countRecord_t * bin,const rwRec * rwrec)738 addToBin(
739 countRecord_t *bin,
740 const rwRec *rwrec)
741 {
742 assert(bin->cr_key == GETIP(rwrec));
743 bin->cr_bytes += rwRecGetBytes(rwrec);
744 bin->cr_packets += rwRecGetPkts(rwrec);
745 ++bin->cr_records;
746 if (rwRecGetStartSeconds(rwrec) < bin->cr_start) {
747 bin->cr_start = rwRecGetStartSeconds(rwrec);
748 }
749 if (bin->cr_end < rwRecGetEndSeconds(rwrec)) {
750 bin->cr_end = rwRecGetEndSeconds(rwrec);
751 }
752 }
753
754
755 /*
756 * int newBin(rwRec *rwrec)
757 *
758 * Creates a new countRecord and initializes it with the values from
759 * the record.
760 */
761 static countRecord_t *
newBin(const rwRec * rwrec)762 newBin(
763 const rwRec *rwrec)
764 {
765 static countRecord_t *current_block = NULL;
766 static uint32_t available = 0;
767 countRecord_t *bin;
768
769 if (available) {
770 --available;
771 bin = current_block;
772 ++current_block;
773 } else {
774 current_block
775 = (countRecord_t*)malloc(RWAC_BLOCK_SIZE * sizeof(countRecord_t));
776 if (NULL == current_block) {
777 skAppPrintErr("Error allocating memory for bin");
778 exit(EXIT_FAILURE);
779 }
780 if (skVectorAppendValue(mem_vector, ¤t_block)) {
781 skAppPrintErr("Error allocating memory for bin");
782 exit(EXIT_FAILURE);
783 }
784 bin = current_block;
785 ++current_block;
786 available = RWAC_BLOCK_SIZE - 1;
787 }
788
789 bin->cr_bytes = rwRecGetBytes(rwrec);
790 bin->cr_packets = rwRecGetPkts(rwrec);
791 bin->cr_records = 1;
792 bin->cr_key = GETIP(rwrec);
793 bin->cr_start = rwRecGetStartSeconds(rwrec);
794 bin->cr_end = rwRecGetEndSeconds(rwrec);
795 bin->cr_next = NULL;
796
797 return bin;
798 }
799
800
801 /*
802 * SECTION: Dumping
803 *
804 * All the output routines are in this section of the text.
805 */
806
807
808 /*
809 * hashToIPTree(&iptree);
810 *
811 * Fills in the 'iptree' with all the IPs in global hash_bins array.
812 */
813 static void
hashToIPTree(skIPTree_t ** ip_tree)814 hashToIPTree(
815 skIPTree_t **ip_tree)
816 {
817 countRecord_t *bin;
818 uint32_t i;
819
820 skIPTreeCreate(ip_tree);
821
822 for (i = 0; i < RWAC_ARRAYSIZE; i++) {
823 if (NULL != hash_bins[i]) {
824 bin = hash_bins[i];
825 do {
826 if (IS_RECORD_WITHIN_LIMITS(bin)) {
827 skIPTreeAddAddress((*ip_tree), (bin->cr_key));
828 }
829 bin = bin->cr_next;
830 } while (bin != hash_bins[i]);
831 }
832 }
833 }
834
835
836 /*
837 * int dumpRecords(outfp)
838 *
839 * Dumps the addrcount contents as a record of bytes, packets,
840 * times &c to 'outfp'
841 *
842 * This is the typical text output from addrcount.
843 *
844 */
845 static int
dumpRecords(FILE * outfp)846 dumpRecords(
847 FILE *outfp)
848 {
849 int w[] = FMT_REC_WIDTH;
850 uint32_t i;
851 countRecord_t *bin;
852 char ip_st[SKIPADDR_STRLEN];
853 char start_st[SKTIMESTAMP_STRLEN];
854 char end_st[SKTIMESTAMP_STRLEN];
855 skipaddr_t ipaddr;
856
857 if (no_columns) {
858 memset(w, 0, sizeof(w));
859 } else {
860 w[0] = skipaddrStringMaxlen(0, ip_format);
861 }
862
863 if ( !no_titles) {
864 fprintf(outfp, FMT_REC_TITLE,
865 w[0], (use_dest ? "dIP" : "sIP"), delimiter,
866 w[1], "Bytes", delimiter,
867 w[2], "Packets", delimiter,
868 w[3], "Records", delimiter,
869 w[4], "Start_Time", delimiter,
870 w[5], "End_Time", final_delim);
871 }
872
873 for (i = 0; i < RWAC_ARRAYSIZE; i++) {
874 if (NULL != hash_bins[i]) {
875 bin = hash_bins[i];
876 do {
877 if (IS_RECORD_WITHIN_LIMITS(bin)) {
878 skipaddrSetV4(&ipaddr, &bin->cr_key);
879 fprintf(outfp, FMT_REC_VALUE,
880 w[0], skipaddrString(ip_st, &ipaddr, ip_format),
881 delimiter,
882 w[1], bin->cr_bytes, delimiter,
883 w[2], bin->cr_packets, delimiter,
884 w[3], bin->cr_records, delimiter,
885 w[4], sktimestamp_r(start_st,
886 sktimeCreate(bin->cr_start, 0),
887 time_flags),
888 delimiter,
889 w[5], sktimestamp_r(end_st,
890 sktimeCreate(bin->cr_end, 0),
891 time_flags),
892 final_delim);
893 }
894 bin = bin->cr_next;
895 } while (bin != hash_bins[i]);
896 }
897 }
898 return 0;
899 }
900
901
902 /*
903 * int dumpRecordsSorted(outfp)
904 *
905 * Dumps the addrcount contents as a record of bytes, packets,
906 * times &c to 'outfp', sorted by the IP address.
907 *
908 */
909 static int
dumpRecordsSorted(FILE * outfp)910 dumpRecordsSorted(
911 FILE *outfp)
912 {
913 int w[] = FMT_REC_WIDTH;
914 uint32_t ip, key;
915 countRecord_t *bin;
916 skIPTree_t *ipset;
917 skIPTreeIterator_t iter;
918 char ip_st[SKIPADDR_STRLEN];
919 char start_st[SKTIMESTAMP_STRLEN];
920 char end_st[SKTIMESTAMP_STRLEN];
921 skipaddr_t ipaddr;
922
923 if (no_columns) {
924 memset(w, 0, sizeof(w));
925 } else {
926 w[0] = skipaddrStringMaxlen(0, ip_format);
927 }
928
929 hashToIPTree(&ipset);
930 if (skIPTreeIteratorBind(&iter, ipset)) {
931 skAppPrintErr("Unable to bind IPTree iterator");
932 skIPTreeDelete(&ipset);
933 return 1;
934 }
935
936 if ( !no_titles) {
937 fprintf(outfp, FMT_REC_TITLE,
938 w[0], (use_dest ? "dIP" : "sIP"), delimiter,
939 w[1], "Bytes", delimiter,
940 w[2], "Packets", delimiter,
941 w[3], "Records", delimiter,
942 w[4], "Start_Time", delimiter,
943 w[5], "End_Time", final_delim);
944 }
945
946 while (skIPTreeIteratorNext(&ip, &iter) == SK_ITERATOR_OK) {
947
948 /* find the ip's entry in the hash table */
949 key = HASHFUNC(ip);
950
951 /* loop through the list of records at this hash table entry
952 * until we find the one that has the IP we want */
953 bin = hash_bins[key];
954 while (ip != bin->cr_key) {
955 /* not the ip we wanted; goto next */
956 bin = bin->cr_next;
957 /* if we've looped all the around, we missed the
958 * IP we wanted, and things are horked */
959 assert(bin != hash_bins[key]);
960 }
961
962 skipaddrSetV4(&ipaddr, &bin->cr_key);
963 fprintf(outfp, FMT_REC_VALUE,
964 w[0], skipaddrString(ip_st, &ipaddr, ip_format), delimiter,
965 w[1], bin->cr_bytes, delimiter,
966 w[2], bin->cr_packets, delimiter,
967 w[3], bin->cr_records, delimiter,
968 w[4], sktimestamp_r(start_st,
969 sktimeCreate(bin->cr_start, 0),
970 time_flags),
971 delimiter,
972 w[5], sktimestamp_r(end_st,
973 sktimeCreate(bin->cr_end, 0),
974 time_flags),
975 final_delim);
976 }
977
978 skIPTreeDelete(&ipset);
979 return 0;
980 }
981
982
983 /*
984 * int dumpIPs(outfp)
985 *
986 * writes IP addresses to the stream 'outfp' in hash-table order
987 * (unsorted).
988 *
989 * Returns 0 on success.
990 */
991 static int
dumpIPs(FILE * outfp)992 dumpIPs(
993 FILE *outfp)
994 {
995 uint32_t i;
996 countRecord_t *bin;
997 char ip_st[SKIPADDR_STRLEN];
998 skipaddr_t ipaddr;
999 int w;
1000
1001 w = ((no_columns) ? 0 : skipaddrStringMaxlen(0, ip_format));
1002
1003 if ( !no_titles) {
1004 fprintf(outfp, "%*s\n", w, (use_dest ? "dIP" : "sIP"));
1005 }
1006
1007 for (i = 0; i < RWAC_ARRAYSIZE; i++) {
1008 if (NULL != hash_bins[i]) {
1009 bin = hash_bins[i];
1010 do {
1011 if (IS_RECORD_WITHIN_LIMITS(bin)) {
1012 skipaddrSetV4(&ipaddr, &bin->cr_key);
1013 fprintf(outfp, "%*s\n",
1014 w, skipaddrString(ip_st, &ipaddr, ip_format));
1015 }
1016 bin = bin->cr_next;
1017 } while (bin != hash_bins[i]);
1018 }
1019 }
1020 return 0;
1021 }
1022
1023
1024 /*
1025 * int dumpIPsSorted(outfp)
1026 *
1027 * Writes the IPs to the stream 'outfp' in sorted order.
1028 *
1029 */
1030 static int
dumpIPsSorted(FILE * outfp)1031 dumpIPsSorted(
1032 FILE *outfp)
1033 {
1034 uint32_t ip;
1035 skIPTree_t *ipset;
1036 skIPTreeIterator_t iter;
1037 char ip_st[SKIPADDR_STRLEN];
1038 skipaddr_t ipaddr;
1039 int w;
1040
1041 hashToIPTree(&ipset);
1042 if (skIPTreeIteratorBind(&iter, ipset)) {
1043 skAppPrintErr("Unable to create IPTree iterator");
1044 skIPTreeDelete(&ipset);
1045 return 1;
1046 }
1047
1048 w = ((no_columns) ? 0 : skipaddrStringMaxlen(0, ip_format));
1049
1050 if ( !no_titles) {
1051 fprintf(outfp, "%*s\n", w, (use_dest ? "dIP" : "sIP"));
1052 }
1053 while (skIPTreeIteratorNext(&ip, &iter) == SK_ITERATOR_OK) {
1054 skipaddrSetV4(&ipaddr, &ip);
1055 fprintf(outfp, "%*s\n", w, skipaddrString(ip_st, &ipaddr, ip_format));
1056 }
1057
1058 skIPTreeDelete(&ipset);
1059 return 0;
1060 }
1061
1062
1063 /*
1064 * int dumpStats(outfp)
1065 *
1066 * dumps a text string describing rwaddrcont results to the stream
1067 * 'outfp'.
1068 */
1069 static int
dumpStats(FILE * outfp)1070 dumpStats(
1071 FILE *outfp)
1072 {
1073 int fmt_width[] = FMT_STAT_WIDTH;
1074 uint32_t i;
1075 uint32_t qual_ips;
1076 uint32_t tot_ips;
1077 uint64_t qual_bytes, qual_packets, qual_records;
1078 uint64_t tot_bytes, tot_packets, tot_records;
1079 countRecord_t *bin;
1080
1081 qual_ips = 0;
1082 tot_ips = 0;
1083 qual_bytes = qual_packets = qual_records = 0;
1084 tot_bytes = tot_packets = tot_records = 0;
1085
1086 if (no_columns) {
1087 memset(fmt_width, 0, sizeof(fmt_width));
1088 }
1089 for (i = 0; i < RWAC_ARRAYSIZE; i++) {
1090 if (NULL != hash_bins[i]) {
1091 bin = hash_bins[i];
1092 do {
1093 ++tot_ips;
1094 tot_bytes += bin->cr_bytes;
1095 tot_packets += bin->cr_packets;
1096 tot_records += bin->cr_records;
1097
1098 if (IS_RECORD_WITHIN_LIMITS(bin)) {
1099 ++qual_ips;
1100 qual_bytes += bin->cr_bytes;
1101 qual_packets += bin->cr_packets;
1102 qual_records += bin->cr_records;
1103 }
1104 bin = bin->cr_next;
1105 } while (bin != hash_bins[i]);
1106 }
1107 }
1108
1109 /* title */
1110 if ( !no_titles) {
1111 fprintf(outfp, FMT_STAT_TITLE,
1112 fmt_width[0], "", delimiter,
1113 fmt_width[1], (use_dest ? "dIP_Uniq" : "sIP_Uniq"), delimiter,
1114 fmt_width[2], "Bytes", delimiter,
1115 fmt_width[3], "Packets", delimiter,
1116 fmt_width[4], "Records", final_delim);
1117 }
1118
1119 fprintf(outfp, FMT_STAT_VALUE,
1120 fmt_width[0], "Total", delimiter,
1121 fmt_width[1], tot_ips, delimiter,
1122 fmt_width[2], tot_bytes, delimiter,
1123 fmt_width[3], tot_packets, delimiter,
1124 fmt_width[4], tot_records, final_delim);
1125
1126 /* print qualifying records if limits were given */
1127 if (0 < min_bytes || max_bytes < UINT64_MAX
1128 || 0 < min_packets || max_packets < UINT64_MAX
1129 || 0 < min_records || max_records < UINT64_MAX)
1130 {
1131 fprintf(outfp, FMT_STAT_VALUE,
1132 fmt_width[0], "Qualifying", delimiter,
1133 fmt_width[1], qual_ips, delimiter,
1134 fmt_width[2], qual_bytes, delimiter,
1135 fmt_width[3], qual_packets, delimiter,
1136 fmt_width[4], qual_records, final_delim);
1137 }
1138
1139 return 0;
1140 }
1141
1142
1143 /*
1144 * int dumpIPSet (char *path)
1145 *
1146 * Dumps the ip addresses counted during normal operation to disk
1147 * in IPSet format.
1148 */
1149 static int
dumpIPSet(const char * path)1150 dumpIPSet(
1151 const char *path)
1152 {
1153 skIPTree_t *ipset;
1154
1155 /* Create the ip tree */
1156 hashToIPTree(&ipset);
1157
1158 /*
1159 * Okay, now we write to disk.
1160 */
1161 if (skIPTreeSave(ipset, path)) {
1162 exit(EXIT_FAILURE);
1163 }
1164
1165 skIPTreeDelete(&ipset);
1166 return 0;
1167 }
1168
1169
1170 /*
1171 * void countFile(stream)
1172 *
1173 * Read the flow records from stream and fill the hash table with
1174 * countRecord_t's.
1175 */
1176 static void
countFile(skstream_t * stream)1177 countFile(
1178 skstream_t *stream)
1179 {
1180 uint32_t hash_idx;
1181 countRecord_t *bin;
1182 rwRec rwrec;
1183 int rv;
1184
1185 /* Read records */
1186 while ((rv = skStreamReadRecord(stream, &rwrec)) == SKSTREAM_OK) {
1187 hash_idx = HASHFUNC(GETIP(&rwrec));
1188
1189 /*
1190 * standard hash foo - check to see if we've got a value.
1191 * If not, create and stuff. If so, check to see if they
1192 * match - if so, stuff. If not, move down until you do -
1193 * if you find nothing, stuff.
1194 */
1195 if (hash_bins[hash_idx] == NULL) {
1196 /* new bin */
1197 hash_bins[hash_idx] = newBin(&rwrec);
1198 hash_bins[hash_idx]->cr_next = hash_bins[hash_idx];
1199 } else {
1200 /* hash collision */
1201 bin = hash_bins[hash_idx];
1202 while ((bin->cr_next != hash_bins[hash_idx]) &&
1203 !CMPFNC(&rwrec, bin))
1204 {
1205 bin = bin->cr_next;
1206 }
1207 /*
1208 * Alright, we've either hit the end of the linked
1209 * list or we've found the value (or both). Check if
1210 * we found the value. */
1211 if (CMPFNC(&rwrec, bin)) {
1212 addToBin(bin, &rwrec);
1213 } else {
1214 assert(bin->cr_next == hash_bins[hash_idx]);
1215 bin->cr_next = newBin(&rwrec);
1216 bin = bin->cr_next;
1217 bin->cr_next = hash_bins[hash_idx]; /* Restore the loop */
1218 }
1219 hash_bins[hash_idx] = bin;
1220 }
1221 }
1222 if (rv != SKSTREAM_ERR_EOF) {
1223 skStreamPrintLastErr(stream, rv, &skAppPrintErr);
1224 }
1225 }
1226
1227
main(int argc,char ** argv)1228 int main(int argc, char **argv)
1229 {
1230 skstream_t *stream;
1231 int rv;
1232
1233 appSetup(argc, argv); /* never returns on error */
1234
1235 /* Read in records from all input files */
1236 while ((rv = skOptionsCtxNextSilkFile(optctx, &stream, &skAppPrintErr))
1237 == 0)
1238 {
1239 skStreamSetIPv6Policy(stream, SK_IPV6POLICY_ASV4);
1240 countFile(stream);
1241 skStreamDestroy(&stream);
1242 }
1243 if (rv < 0) {
1244 exit(EXIT_FAILURE);
1245 }
1246
1247 /* Invoke the pager when appropriate. */
1248 switch (print_mode) {
1249 case RWAC_PMODE_STAT:
1250 case RWAC_PMODE_IPSETFILE:
1251 break;
1252 case RWAC_PMODE_RECORDS:
1253 case RWAC_PMODE_IPS:
1254 case RWAC_PMODE_SORTED_RECORDS:
1255 case RWAC_PMODE_SORTED_IPS:
1256 if (NULL == output.of_name) {
1257 rv = skFileptrOpenPager(&output, pager);
1258 if (rv && rv != SK_FILEPTR_PAGER_IGNORED) {
1259 skAppPrintErr("Unable to invoke pager");
1260 }
1261 }
1262 break;
1263 case RWAC_PMODE_NONE:
1264 skAbortBadCase(print_mode);
1265 }
1266
1267 /* Produce the output */
1268 switch (print_mode) {
1269 case RWAC_PMODE_STAT:
1270 dumpStats(output.of_fp);
1271 break;
1272 case RWAC_PMODE_IPSETFILE:
1273 dumpIPSet(ipset_file);
1274 break;
1275 case RWAC_PMODE_RECORDS:
1276 dumpRecords(output.of_fp);
1277 break;
1278 case RWAC_PMODE_IPS:
1279 dumpIPs(output.of_fp);
1280 break;
1281 case RWAC_PMODE_SORTED_RECORDS:
1282 dumpRecordsSorted(output.of_fp);
1283 break;
1284 case RWAC_PMODE_SORTED_IPS:
1285 dumpIPsSorted(output.of_fp);
1286 break;
1287 case RWAC_PMODE_NONE:
1288 skAbortBadCase(print_mode);
1289 }
1290
1291 /* output */
1292 return (0);
1293 }
1294
1295
1296 /*
1297 ** Local Variables:
1298 ** mode:c
1299 ** indent-tabs-mode:nil
1300 ** c-basic-offset:4
1301 ** End:
1302 */
1303