1 /*
2 ** Copyright (C) 2005-2020 by Carnegie Mellon University.
3 **
4 ** @OPENSOURCE_LICENSE_START@
5 ** See license information in ../../LICENSE.txt
6 ** @OPENSOURCE_LICENSE_END@
7 */
8
9 /*
10 * Read rwrec input and write the output to every known SiLK Flow
11 * record file format.
12 *
13 */
14
15
16 #include <silk/silk.h>
17
18 RCSIDENT("$SiLK: rwallformats.c ef14e54179be 2020-04-14 21:57:45Z mthomas $");
19
20 #include <silk/rwrec.h>
21 #include <silk/sksite.h>
22 #include <silk/skstream.h>
23 #include <silk/utils.h>
24
25
26 /* LOCAL DEFINES AND TYPEDEFS */
27
28 /* where to write --help output */
29 #define USAGE_FH stdout
30
31
32 /* LOCAL VARIABLES */
33
34 /* basename of outputs. Will append -{B,L}-<type>-<version>.dat to this. */
35 static const char *base_name = NULL;
36
37 /* if non-zero, do not include invocation in the output file. I added
38 * this to deal with libtool since, when running within the build
39 * directory, the command name may or may not include an "lt-" prefix,
40 * and this makes comparing complete files impossible. */
41 static int no_invocation = 0;
42
43 /* where to write the records as they are read to avoid having to open
44 * many file handles */
45 static const char *temp_directory = NULL;
46
47 /* file handle to temporary file */
48 static FILE *tmpf = NULL;
49
50 /* support for input files */
51 static sk_options_ctx_t *optctx = NULL;
52
53 /* reference to argc and argv for filter output */
54 static int g_argc;
55 static char **g_argv;
56
57 /* file formats to output */
58 static unsigned int stream_format[] = {
59 FT_FLOWCAP,
60 FT_RWAUGMENTED,
61 FT_RWAUGROUTING,
62 FT_RWAUGWEB,
63 FT_RWAUGSNMPOUT,
64 FT_RWFILTER,
65 FT_RWGENERIC,
66 FT_RWIPV6,
67 FT_RWIPV6ROUTING,
68 FT_RWNOTROUTED,
69 FT_RWROUTED,
70 FT_RWSPLIT,
71 FT_RWWWW
72 };
73
74
75 /* OPTIONS SETUP */
76
77 typedef enum {
78 OPT_BASENAME, OPT_NO_INVOCATION
79 } appOptionsEnum;
80
81 static struct option appOptions[] = {
82 {"basename", REQUIRED_ARG, 0, OPT_BASENAME},
83 {"no-invocation", NO_ARG, 0, OPT_NO_INVOCATION},
84 {0,0,0,0} /* sentinel entry */
85 };
86
87 static const char *appHelp[] = {
88 "Begin each output file with this text",
89 "Do not include command line invocation in output",
90 (char *)NULL
91 };
92
93
94 /* LOCAL FUNCTION PROTOTYPES */
95
96 static int appOptionsHandler(clientData cData, int opt_index, char *opt_arg);
97 static FILE *openTempFile(void);
98 static size_t openOutput(skstream_t **out_stream, const rwRec *rwrec);
99
100
101 /* FUNCTION DEFINITIONS */
102
103 /*
104 * appUsageLong();
105 *
106 * Print complete usage information to USAGE_FH. Pass this
107 * function to skOptionsSetUsageCallback(); skOptionsParse() will
108 * call this funciton and then exit the program when the --help
109 * option is given.
110 */
111 static void
appUsageLong(void)112 appUsageLong(
113 void)
114 {
115 #define USAGE_MSG \
116 ("[SWITCHES] [FILES]\n" \
117 "\tRead SiLK Flow records as input and write them to files using\n" \
118 "\tevery known SiLK Flow file format and byte order. Files are\n" \
119 "\tnamed FT_<format>-v<version>-c<compmethod>-{B,L}.dat, where\n" \
120 "\t<version> is file version, <compmethod> is the compression\n" \
121 "\tmethod, and {B,L} is the byte order (big,little). The names will\n" \
122 "\tbe prefixed by \"<basename>-\" when --basename is given.\n")
123
124 FILE *fh = USAGE_FH;
125
126 skAppStandardUsage(fh, USAGE_MSG, appOptions, appHelp);
127 skOptionsCtxOptionsUsage(optctx, fh);
128 skOptionsTempDirUsage(fh);
129 sksiteOptionsUsage(fh);
130 }
131
132
133 /*
134 * appTeardown()
135 *
136 * Teardown all modules, close all files, and tidy up all
137 * application state.
138 *
139 * This function is idempotent.
140 */
141 static void
appTeardown(void)142 appTeardown(
143 void)
144 {
145 static int teardownFlag = 0;
146
147 if (teardownFlag) {
148 return;
149 }
150 teardownFlag = 1;
151
152 if (tmpf) {
153 fclose(tmpf);
154 tmpf = NULL;
155 }
156
157 skOptionsCtxDestroy(&optctx);
158 skAppUnregister();
159 }
160
161
162 /*
163 * appSetup(argc, argv);
164 *
165 * Perform all the setup for this application include setting up
166 * required modules, parsing options, etc. This function should be
167 * passed the same arguments that were passed into main().
168 *
169 * Returns to the caller if all setup succeeds. If anything fails,
170 * this function will cause the application to exit with a FAILURE
171 * exit status.
172 */
173 static void
appSetup(int argc,char ** argv)174 appSetup(
175 int argc,
176 char **argv)
177 {
178 SILK_FEATURES_DEFINE_STRUCT(features);
179 unsigned int optctx_flags;
180 int rv;
181
182 /* verify same number of options and help strings */
183 assert((sizeof(appHelp)/sizeof(char *)) ==
184 (sizeof(appOptions)/sizeof(struct option)));
185
186 /* register the application */
187 skAppRegister(argv[0]);
188 skAppVerifyFeatures(&features, NULL);
189 skOptionsSetUsageCallback(&appUsageLong);
190
191 optctx_flags = (SK_OPTIONS_CTX_INPUT_SILK_FLOW
192 | SK_OPTIONS_CTX_ALLOW_STDIN);
193
194 /* register the options */
195 if (skOptionsCtxCreate(&optctx, optctx_flags)
196 || skOptionsCtxOptionsRegister(optctx)
197 || skOptionsRegister(appOptions, &appOptionsHandler, NULL)
198 || skOptionsTempDirRegister(&temp_directory)
199 || sksiteOptionsRegister(SK_SITE_FLAG_CONFIG_FILE))
200 {
201 skAppPrintErr("Unable to register options");
202 exit(EXIT_FAILURE);
203 }
204
205 /* register the teardown handler */
206 if (atexit(appTeardown) < 0) {
207 skAppPrintErr("Unable to register appTeardown() with atexit()");
208 appTeardown();
209 exit(EXIT_FAILURE);
210 }
211
212 /* parse the options */
213 rv = skOptionsCtxOptionsParse(optctx, argc, argv);
214 if (rv < 0) {
215 skAppUsage();
216 }
217
218 /* ensure the site config is available */
219 if (sksiteConfigure(1)) {
220 exit(EXIT_FAILURE);
221 }
222
223 /* open a temporary file */
224 tmpf = openTempFile();
225 if (tmpf == NULL) {
226 exit(EXIT_FAILURE);
227 }
228
229 return; /* OK */
230 }
231
232
233 /*
234 * status = appOptionsHandler(cData, opt_index, opt_arg);
235 *
236 * This function is passed to skOptionsRegister(); it will be called
237 * by skOptionsParse() for each user-specified switch that the
238 * application has registered; it should handle the switch as
239 * required---typically by setting global variables---and return 1
240 * if the switch processing failed or 0 if it succeeded. Returning
241 * a non-zero from from the handler causes skOptionsParse() to return
242 * a negative value.
243 *
244 * The clientData in 'cData' is typically ignored; 'opt_index' is
245 * the index number that was specified as the last value for each
246 * struct option in appOptions[]; 'opt_arg' is the user's argument
247 * to the switch for options that have a REQUIRED_ARG or an
248 * OPTIONAL_ARG.
249 */
250 static int
appOptionsHandler(clientData UNUSED (cData),int opt_index,char * opt_arg)251 appOptionsHandler(
252 clientData UNUSED(cData),
253 int opt_index,
254 char *opt_arg)
255 {
256 switch ((appOptionsEnum)opt_index) {
257 case OPT_BASENAME:
258 base_name = opt_arg;
259 break;
260
261 case OPT_NO_INVOCATION:
262 no_invocation = 1;
263 break;
264 }
265
266 return 0; /* OK */
267 }
268
269
270 /*
271 * fp = openTempFile();
272 *
273 * Open a temporary file; the file is immediately unlinked, so it
274 * will be removed when the program exits.
275 */
276 static FILE *
openTempFile(void)277 openTempFile(
278 void)
279 {
280 static char temp_name[PATH_MAX];
281 int len;
282 int fd;
283 FILE *fp;
284
285 temp_directory = skTempDir(temp_directory, &skAppPrintErr);
286 if (temp_directory == NULL) {
287 return NULL;
288 }
289
290 /* attempt to open a temp file, then remove it */
291 len = snprintf(temp_name, sizeof(temp_name), "%s/%s.XXXXXXXX",
292 temp_directory, skAppName());
293 if (len < 0 || (size_t)len > sizeof(temp_name)) {
294 skAppPrintErr("Error creating temp file name");
295 return NULL;
296 }
297 fd = mkstemp(temp_name);
298 if (fd == -1) {
299 skAppPrintSyserror("Cannot create temp file %s", temp_name);
300 return NULL;
301 }
302 fp = fdopen(fd, "wb+");
303 if (fp == NULL) {
304 skAppPrintSyserror("Cannot reopen temp file %s", temp_name);
305 return NULL;
306 }
307 if (unlink(temp_name)) {
308 skAppPrintSyserror("Cannot remove temp file '%s'", temp_name);
309 fclose(fp);
310 return NULL;
311 }
312
313 return fp;
314 }
315
316
317 /*
318 * status = openOutput(&stream, rwrec);
319 *
320 * Open an output stream for every known type-version-endian
321 * combination. The files are opened one at a time; a different
322 * file is opened each time this function is called. The caller
323 * should skStreamClose() the file when finished with it. The newly
324 * opened file is placed into the memory pointed at by 'stream'.
325 * The 'rwrec' is used to write headers to the file.
326 */
327 static size_t
openOutput(skstream_t ** out_stream,const rwRec * first_rec)328 openOutput(
329 skstream_t **out_stream,
330 const rwRec *first_rec)
331 {
332 const size_t num_formats = (sizeof(stream_format)/sizeof(unsigned int));
333 const silk_endian_t endians[] = {SILK_ENDIAN_BIG, SILK_ENDIAN_LITTLE};
334 const char *endian_name[] = {"B", "L"};
335
336 static unsigned int num_compmethod = 0;
337 static unsigned int f, e, c;
338 static sk_file_version_t v;
339
340 silk_endian_t byte_order;
341 skstream_t *stream;
342 sk_file_header_t *hdr;
343 int rv = SKSTREAM_OK;
344 char path[PATH_MAX];
345 char format_name[SK_MAX_STRLEN_FILE_FORMAT+1];
346
347 if (num_compmethod == 0) {
348 /* need to initialize */
349
350 /* find the number of compression methods */
351 for (num_compmethod = 0;
352 skCompMethodCheck(num_compmethod);
353 ++num_compmethod)
354 ; /* no-op */
355
356 c = 0;
357 f = 0;
358 v = 0;
359 e = 0;
360 }
361
362 /* loop over compression methods */
363 for ( ; c < num_compmethod; ++c) {
364 if (SK_COMPMETHOD_IS_AVAIL != skCompMethodCheck(c)) {
365 continue;
366 }
367
368 /* loop over formats */
369 for ( ; f < num_formats; ++f) {
370 /* loop over versions of that format */
371 for (;;) {
372 /* only RWGENERIC supports v=0 */
373 if (v == 0) {
374 if (stream_format[f] != FT_RWGENERIC) {
375 goto NEXT_VERSION;
376 }
377 }
378 if (stream_format[f] == FT_FLOWCAP && v < 2) {
379 goto NEXT_VERSION;
380 }
381
382 /* loop over byte-orders */
383 while (e < 2) {
384 byte_order = endians[e];
385
386 skFileFormatGetName(format_name, sizeof(format_name),
387 stream_format[f]);
388 if ((size_t)snprintf(path, sizeof(path),
389 "%s%s%s-v%d-c%u-%s.dat",
390 ((base_name != NULL) ? base_name :""),
391 ((base_name != NULL) ? "-" : ""),
392 format_name, v, c, endian_name[e])
393 >= sizeof(path))
394 {
395 skAppPrintErr("File name overflow");
396 return -1;
397 }
398
399 /* create and open the file */
400 hdr = NULL;
401 rv = SKSTREAM_OK;
402 if ( !rv) {
403 rv = skStreamCreate(&stream, SK_IO_WRITE,
404 SK_CONTENT_SILK_FLOW);
405 }
406 if ( !rv) {
407 rv = skStreamBind(stream, path);
408 }
409 if ( !rv) {
410 hdr = skStreamGetSilkHeader(stream);
411 rv = skHeaderSetFileFormat(hdr, stream_format[f]);
412 }
413 if ( !rv) {
414 rv = skHeaderSetRecordVersion(hdr, v);
415 }
416 if ( !rv) {
417 rv = skHeaderSetByteOrder(hdr, byte_order);
418 }
419 if ( !rv) {
420 rv = skHeaderSetCompressionMethod(hdr, c);
421 }
422 #if 0
423 if ( !rv) {
424 /* force output to be in IPv4 for comparison
425 * with formats that do not support IPv6 */
426 rv = skStreamSetIPv6Policy(stream, SK_IPV6POLICY_ASV4);
427 }
428 #endif /* 0 */
429 if ( !rv) {
430 rv = skHeaderAddProbename(hdr, "DUMMY_PROBE");
431 }
432 if ( !rv) {
433 rv=skHeaderAddPackedfile(hdr,
434 rwRecGetStartTime(first_rec),
435 rwRecGetFlowType(first_rec),
436 rwRecGetSensor(first_rec));
437 }
438 if ( !rv && !no_invocation) {
439 rv = skHeaderAddInvocation(hdr, 1, g_argc, g_argv);
440 }
441 if ( !rv) {
442 rv = skStreamOpen(stream);
443 }
444 if ( !rv) {
445 rv = skStreamWriteSilkHeader(stream);
446 }
447
448 if (rv) {
449 if (rv == SKSTREAM_ERR_UNSUPPORT_VERSION) {
450 /* Reached max version for this type. Try
451 * next type. */
452 skStreamDestroy(&stream);
453 if (skFileExists(path)) {
454 unlink(path);
455 }
456 goto NEXT_TYPE;
457 }
458
459 /* Unexpected error. Bail */
460 skStreamPrintLastErr(stream, rv, &skAppPrintErr);
461 skAppPrintErr("Error opening '%s'\n", path);
462 skStreamDestroy(&stream);
463 return -1;
464 }
465
466 /* increment 'e' for the next run */
467 ++e;
468
469 /* and return */
470 *out_stream = stream;
471 return 0;
472
473 } /* for e */
474
475 NEXT_VERSION:
476 ++v;
477 e = 0;
478 } /* while 1 */
479 NEXT_TYPE:
480 v = 0;
481 e = 0;
482 } /* for f */
483 f = 0;
484 v = 0;
485 e = 0;
486 } /* for c */
487
488 return 1;
489 }
490
491
492 /*
493 * writeOutputs();
494 *
495 * Writes the records stored in the global 'tmpf' to a file in
496 * every know file format, version, compression method, and byte
497 * order. The openOutput() function is called to open the next
498 * file, then the records in tmpf are written there.
499 */
500 static void
writeOutputs(void)501 writeOutputs(
502 void)
503 {
504 skstream_t *stream;
505 rwRec rwrec;
506 int rv;
507
508 /* write each real output file */
509 for (;;) {
510 if (fseek(tmpf, 0, SEEK_SET) == -1) {
511 skAppPrintSyserror("Cannot seek in temp file");
512 exit(EXIT_FAILURE);
513 }
514 if (!fread(&rwrec, sizeof(rwRec), 1, tmpf)) {
515 skAppPrintErr("Cannot read from temp file");
516 exit(EXIT_FAILURE);
517 }
518
519 rv = openOutput(&stream, &rwrec);
520 if (rv != 0) {
521 if (rv == 1) {
522 /* done */
523 break;
524 }
525 /* error */
526 exit(EXIT_FAILURE);
527 }
528
529 do {
530 rv = skStreamWriteRecord(stream, &rwrec);
531 if (SKSTREAM_OK != rv) {
532 skStreamPrintLastErr(stream, rv, &skAppPrintErr);
533 if (SKSTREAM_ERROR_IS_FATAL(rv)) {
534 break;
535 }
536 }
537 } while (fread(&rwrec, sizeof(rwRec), 1, tmpf));
538
539 rv = skStreamClose(stream);
540 if (SKSTREAM_OK != rv) {
541 skStreamPrintLastErr(stream, rv, &skAppPrintErr);
542 }
543 skStreamDestroy(&stream);
544 }
545 }
546
547
548 /*
549 * status = readFileToTemp(input_path);
550 *
551 * Append the contents of input_path to the temporary file.
552 */
553 static int
readFileToTemp(skstream_t * in_stream)554 readFileToTemp(
555 skstream_t *in_stream)
556 {
557 rwRec rwrec;
558 int rv;
559
560 while ((rv = skStreamReadRecord(in_stream, &rwrec)) == SKSTREAM_OK) {
561 if (!fwrite(&rwrec, sizeof(rwRec), 1, tmpf)) {
562 skAppPrintSyserror("Cannot write to temp file");
563 return -1;
564 }
565 }
566 if (SKSTREAM_ERR_EOF != rv) {
567 skStreamPrintLastErr(in_stream, rv, &skAppPrintErr);
568 }
569
570 return 0;
571 }
572
573
main(int argc,char ** argv)574 int main(int argc, char **argv)
575 {
576 skstream_t *stream;
577 int rv = 0;
578
579 g_argc = argc;
580 g_argv = argv;
581
582 appSetup(argc, argv); /* never returns on error */
583
584 /* process input */
585 while ((rv = skOptionsCtxNextSilkFile(optctx, &stream, &skAppPrintErr))
586 == 0)
587 {
588 if (readFileToTemp(stream)) {
589 skStreamDestroy(&stream);
590 exit(EXIT_FAILURE);
591 }
592 skStreamDestroy(&stream);
593 }
594 if (rv < 0) {
595 exit(EXIT_FAILURE);
596 }
597
598 /* Write the records from the temp file to each output file */
599 writeOutputs();
600
601 /* done */
602 return 0;
603 }
604
605
606 /*
607 ** Local Variables:
608 ** mode:c
609 ** indent-tabs-mode:nil
610 ** c-basic-offset:4
611 ** End:
612 */
613