1 /**
2 * @file mediator_export.c
3 *
4 * All exporting related functions, bulk of the code.
5 *
6 * ------------------------------------------------------------------------
7 * Copyright (C) 2012-2018 Carnegie Mellon University. All Rights Reserved.
8 * ------------------------------------------------------------------------
9 * Authors: Emily Sarneso, Matt Coates
10 * -----------------------------------------------------------------------
11 * @OPENSOURCE_HEADER_START@
12 * Use of this (and related) source code is subject to the terms
13 * of the following licenses:
14 *
15 * GNU Public License (GPL) Rights pursuant to Version 2, June 1991
16 * Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013
17 *
18 *
19 * This material is based upon work funded and supported by
20 * the Department of Defense under Contract FA8721-05-C-0003 with
21 * Carnegie Mellon University for the operation of the Software Engineering
22 * Institue, a federally funded research and development center. Any opinions,
23 * findings and conclusions or recommendations expressed in this
24 * material are those of the author(s) and do not
25 * necessarily reflect the views of the United States
26 * Department of Defense.
27 *
28 * NO WARRANTY
29 *
30 * THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE
31 * MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY
32 * MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED
33 * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
34 * FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
35 * OBTAINED FROM THE USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY
36 * DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM
37 * PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
38 *
39 * This material has been approved for public release and unlimited
40 * distribution.
41 *
42 * Carnegie Mellon®, CERT® and CERT Coordination Center® are
43 * registered marks of Carnegie Mellon University.
44 *
45 * DM-0001877
46 *
47 * Carnegie Mellon University retains
48 * copyrights in all material produced under this contract. The U.S.
49 * Government retains a non-exclusive, royalty-free license to publish or
50 * reproduce these documents, or allow others to do so, for U.S.
51 * Government purposes only pursuant to the copyright license under the
52 * contract clause at 252.227.7013.
53 *
54 * Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
55 * Mellon University, its trustees, officers, employees, and agents from
56 * all claims or demands made against them (and any related losses,
57 * expenses, or attorney's fees) arising out of, or relating to Licensee's
58 * and/or its sub licensees' negligent use or willful misuse of or
59 * negligent conduct or willful misconduct regarding the Software,
60 * facilities, or other rights or assistance granted by Carnegie Mellon
61 * University under this License, including, but not limited to, any
62 * claims of product liability, personal injury, death, damage to
63 * property, or violation of any laws or regulations.
64 *
65 * @OPENSOURCE_HEADER_END@
66 * -----------------------------------------------------------
67 */
68
69 #include <mediator/mediator_inf.h>
70 #include <mediator/mediator_core.h>
71 #include <mediator/config.h>
72 #include <mediator/mediator_filter.h>
73 #include <mediator/mediator_util.h>
74 #include "mediator_dns.h"
75 #include "mediator_dedup.h"
76 #include "mediator_ssl.h"
77 #include "mediator_stat.h"
78 #include "mediator_print.h"
79 #include "mediator_json.h"
80
81 #if HAVE_MYSQL
82 #include <mysql.h>
83 #endif
84
85 #define TIME_FMT "%04u%02u%02u%02u%02u%02u"
86 #define PRINT_SHORT_TIME_FMT "%02u:%02u:%02u"
87
88 #define FBBLNP(a, b) fbBasicListGetNextPtr(a, b)
89
90 #define MD_REM_MSG(_buf_) (_buf_->buflen - (_buf_->cp - _buf_->buf))
91 #define MD_MSG_LEN(_buf_) (_buf_->cp - _buf_->buf)
92 #define MD_CHECK_RET(_buf_, _ret_, _size_) \
93 if (_ret_ < 0) return 0; \
94 if ((size_t)_ret_ >= _size_) return 0; \
95 _size_ -= _ret_; \
96 _buf_->cp += _ret_;
97 #define MD_APPEND_CHAR(_buf_, _ch_) \
98 *(_buf_->cp) = _ch_; \
99 ++(_buf_->cp);
100 #define MD_APPEND_CHAR_CHECK(_rem_, _buf_, _ch_) \
101 if (_rem_ > 1) { \
102 MD_APPEND_CHAR(_buf_, _ch_); \
103 _rem_ -= 1; \
104 } else { \
105 return FALSE; \
106 }
107
108 #define MD_RET0(_rv_) \
109 if (!_rv_) { \
110 return 0; \
111 }
112
113
114 /* a struct to keep track of table/file names for DPI output */
115 typedef struct mdTableInfo_st {
116 char *table_name;
117 FILE *table_file;
118 char *file_name;
119 uint64_t last_rotate_ms;
120 uint8_t serial;
121 } mdTableInfo_t;
122
123 mdTableInfo_t **table_info = NULL;
124 static int num_tables = 0;
125 static GHashTable *table_hash = NULL;
126
127 static int num_exporters = 0;
128
129 typedef struct mdMySQLInfo_st {
130 char *user;
131 char *password;
132 char *db_name;
133 char *db_host;
134 char *table;
135 #if HAVE_MYSQL
136 MYSQL *conn;
137 #endif
138 } mdMySQLInfo_t;
139
140 typedef struct mdSSLConfig_st {
141 int *issuer;
142 int *subject;
143 int *other;
144 int *extensions;
145 } mdSSLConfig_t;
146
147 typedef gboolean (*mdBLPrint_fn)(mdFlowExporter_t *, fbBasicList_t *,
148 char *, size_t, char *, gboolean);
149 typedef gboolean (*mdVLPrint_fn)(mdFlowExporter_t *,uint8_t *, char *,
150 char *, size_t, uint16_t, size_t, gboolean);
151
152 struct mdFlowExporter_st {
153 fbExporter_t *exporter;
154 FILE *lfp;
155 char *outspec;
156 char *current_fname;
157 char *mv_path;
158 fBuf_t *fbuf;
159 mdMySQLInfo_t *mysql;
160 mdFieldList_t *custom_list;
161 mdSSLConfig_t *ssl_config;
162 GHashTable *dpi_field_table;
163 char *name;
164 mdBuf_t *buf;
165 mdBLPrint_fn BLprint_fn;
166 mdVLPrint_fn VLprint_fn;
167 md_sess_init_fn sess_init;
168 fbConnSpec_t spec;
169 uint64_t rotate;
170 uint64_t last_rotate_ms;
171 uint64_t last_restart_ms;
172 uint64_t lastUdpTempTime;
173 uint64_t exp_flows;
174 uint64_t exp_stats;
175 uint64_t exp_bytes;
176 uint64_t time_started;
177 mdTransportType_t type;
178 char delimiter;
179 char dpi_delimiter;
180 gboolean lock;
181 uint8_t no_stats;
182 uint8_t id;
183 uint8_t dns_rr_only;
184 gboolean gzip;
185 gboolean custom_list_dpi;
186 gboolean basic_list_dpi;
187 gboolean flowonly;
188 gboolean dpionly;
189 gboolean dnsdedup;
190 gboolean dnsdeduponly;
191 gboolean print_header;
192 gboolean remove_empty;
193 gboolean multi_files;
194 gboolean no_index;
195 gboolean timestamp_files;
196 gboolean no_flow_stats;
197 gboolean escape_chars;
198 gboolean remove_uploaded;
199 gboolean active;
200 gboolean json;
201 gboolean dns_resp_only;
202 gboolean dedup_per_flow;
203 gboolean dedupconfig;
204 gboolean deduponly;
205 gboolean ssldedup;
206 gboolean ssldeduponly;
207 gboolean md5_hash;
208 gboolean sha1_hash;
209 gboolean no_flow;
210 gboolean metadata_export;
211 };
212
213 static gboolean mdJsonifyNewSSLCertRecord(
214 mdFlowExporter_t *exporter,
215 yaf_newssl_cert_t *cert,
216 uint8_t cert_no);
217
218 static gboolean mdExporterTextNewSSLCertPrint(
219 mdFlowExporter_t *exporter,
220 yaf_newssl_cert_t *cert,
221 char *index_str,
222 size_t index_len,
223 uint8_t cert_no);
224
225 static gboolean mdExporterCheckSSLConfig(
226 mdFlowExporter_t *exporter,
227 int obj_id,
228 uint8_t type);
229
230 static void mdCloseAndUnlock(
231 mdFlowExporter_t *exporter,
232 FILE *fp,
233 char *filename,
234 char *table);
235
236 /**
237 * mdNewTable
238 *
239 *
240 * create a new table to keep track of the table or file names
241 * for DPI to CSV output
242 *
243 * @param table name of table
244 */
mdNewTable(char * table)245 void *mdNewTable(
246 char *table)
247 {
248
249 if (!table_info) {
250 table_info =
251 (mdTableInfo_t **)g_malloc(MAX_VALUE_LIST*sizeof(mdTableInfo_t *));
252 }
253
254 if (num_tables > 0 && (num_tables % MAX_VALUE_LIST)) {
255 table_info =
256 (mdTableInfo_t **)g_realloc(table_info,
257 ((MAX_VALUE_LIST + num_tables) *sizeof(mdTableInfo_t *)));
258 }
259
260 table_info[num_tables] = g_slice_new0(mdTableInfo_t);
261 table_info[num_tables]->table_name = g_strdup(table);
262 table_info[num_tables]->serial = 0;
263 num_tables++;
264
265 return (void *)table_info[num_tables-1];
266 }
267
268
mdGetTable(int id)269 void *mdGetTable(
270 int id)
271 {
272
273 mdTableInfo_t *ret = NULL;
274
275 /* associate app label with known info element */
276 switch (id) {
277 case 80:
278 id = 110;
279 break;
280 case 21:
281 id = 131;
282 break;
283 case 25:
284 id = 162;
285 break;
286 case 53:
287 id = 1;
288 break;
289 case 143:
290 id = 136;
291 break;
292 case 554:
293 id = 143;
294 break;
295 case 5060:
296 id = 155;
297 break;
298 case 22:
299 id = 171;
300 break;
301 default:
302 return NULL;
303 }
304
305 ret = g_hash_table_lookup(table_hash, GUINT_TO_POINTER((unsigned int)id));
306
307 return (void *)ret;
308
309 }
310
311 /**
312 * mdInsertTableItem
313 *
314 *
315 * Insert an Info Element ID/mdTableInfo struct into the hash table
316 * for quick lookup.
317 *
318 */
mdInsertTableItem(void * table_name,int val)319 gboolean mdInsertTableItem(
320 void *table_name,
321 int val)
322 {
323
324 void *key = NULL;
325 gpointer value = NULL;
326 gboolean rc;
327
328 if (!table_hash) {
329 table_hash = g_hash_table_new((GHashFunc)g_direct_hash,
330 (GEqualFunc)g_direct_equal);
331 if (table_hash == NULL) {
332 return FALSE;
333 }
334 }
335
336 rc = g_hash_table_lookup_extended(table_hash,
337 GUINT_TO_POINTER((unsigned int)val),
338 key, &value);
339 if (rc) {
340 return FALSE;
341 }
342
343 g_hash_table_insert(table_hash, GUINT_TO_POINTER(val), table_name);
344
345 return TRUE;
346 }
347
348
349 /**
350 * mdBuildDefaultTableHash
351 *
352 * if the user doesn't give us names for the files,
353 * we need to create the hash table with all the default ones.
354 *
355 */
mdBuildDefaultTableHash()356 void mdBuildDefaultTableHash()
357 {
358 mdTableInfo_t *tab = NULL;
359
360 tab = mdNewTable(FLOW_STATS_DEFAULT);
361 mdInsertTableItem(tab, 500);
362
363 tab = mdNewTable(FTP_DEFAULT);
364 mdInsertTableItem(tab, 131);
365 mdInsertTableItem(tab, 132);
366 mdInsertTableItem(tab, 133);
367 mdInsertTableItem(tab, 134);
368 mdInsertTableItem(tab, 135);
369
370 tab = mdNewTable(SSH_DEFAULT);
371 mdInsertTableItem(tab, 171);
372
373 tab = mdNewTable(SMTP_DEFAULT);
374 mdInsertTableItem(tab, 162);
375 mdInsertTableItem(tab, 163);
376 mdInsertTableItem(tab, 164);
377 mdInsertTableItem(tab, 165);
378 mdInsertTableItem(tab, 166);
379 mdInsertTableItem(tab, 167);
380 mdInsertTableItem(tab, 168);
381 mdInsertTableItem(tab, 169);
382 mdInsertTableItem(tab, 170);
383 mdInsertTableItem(tab, 222);
384 mdInsertTableItem(tab, 251);
385
386 tab = mdNewTable(DNS_DEFAULT);
387 mdInsertTableItem(tab, 1);
388 mdInsertTableItem(tab, 2);
389 mdInsertTableItem(tab, 5);
390 mdInsertTableItem(tab, 6);
391 mdInsertTableItem(tab, 12);
392 mdInsertTableItem(tab, 15);
393 mdInsertTableItem(tab, 16);
394 mdInsertTableItem(tab, 28);
395 mdInsertTableItem(tab, 33);
396 mdInsertTableItem(tab, 43);
397 mdInsertTableItem(tab, 47);
398 mdInsertTableItem(tab, 48);
399 mdInsertTableItem(tab, 50);
400 mdInsertTableItem(tab, 51);
401 mdInsertTableItem(tab, 53);
402
403
404 tab = mdNewTable(TFTP_DEFAULT);
405 mdInsertTableItem(tab, 126);
406 mdInsertTableItem(tab, 127);
407
408 tab = mdNewTable(HTTP_DEFAULT);
409 mdInsertTableItem(tab, 110);
410 mdInsertTableItem(tab, 111);
411 mdInsertTableItem(tab, 112);
412 mdInsertTableItem(tab, 113);
413 mdInsertTableItem(tab, 114);
414 mdInsertTableItem(tab, 115);
415 mdInsertTableItem(tab, 116);
416 mdInsertTableItem(tab, 117);
417 mdInsertTableItem(tab, 118);
418 mdInsertTableItem(tab, 119);
419 mdInsertTableItem(tab, 120);
420 mdInsertTableItem(tab, 121);
421 mdInsertTableItem(tab, 122);
422 mdInsertTableItem(tab, 123);
423 mdInsertTableItem(tab, 220);
424 mdInsertTableItem(tab, 221);
425 mdInsertTableItem(tab, 252);
426 mdInsertTableItem(tab, 253);
427 mdInsertTableItem(tab, 254);
428 mdInsertTableItem(tab, 255);
429 mdInsertTableItem(tab, 256);
430 mdInsertTableItem(tab, 257);
431 mdInsertTableItem(tab, 258);
432 mdInsertTableItem(tab, 259);
433 mdInsertTableItem(tab, 260);
434 mdInsertTableItem(tab, 261);
435 mdInsertTableItem(tab, 262);
436 mdInsertTableItem(tab, 263);
437 mdInsertTableItem(tab, 264);
438 mdInsertTableItem(tab, 265);
439 mdInsertTableItem(tab, 266);
440 mdInsertTableItem(tab, 267);
441 mdInsertTableItem(tab, 268);
442 mdInsertTableItem(tab, 269);
443 mdInsertTableItem(tab, 270);
444 mdInsertTableItem(tab, 271);
445 mdInsertTableItem(tab, 272);
446 mdInsertTableItem(tab, 273);
447 mdInsertTableItem(tab, 274);
448 mdInsertTableItem(tab, 275);
449 mdInsertTableItem(tab, 276);
450 mdInsertTableItem(tab, 277);
451 mdInsertTableItem(tab, 278);
452 mdInsertTableItem(tab, 279);
453 mdInsertTableItem(tab, 280);
454
455
456 tab = mdNewTable(IMAP_DEFAULT);
457 mdInsertTableItem(tab, 136);
458 mdInsertTableItem(tab, 137);
459 mdInsertTableItem(tab, 138);
460 mdInsertTableItem(tab, 139);
461 mdInsertTableItem(tab, 140);
462 mdInsertTableItem(tab, 141);
463 mdInsertTableItem(tab, 142);
464
465 tab = mdNewTable(IRC_DEFAULT);
466 mdInsertTableItem(tab, 125);
467
468 tab = mdNewTable(SIP_DEFAULT);
469 mdInsertTableItem(tab, 155);
470 mdInsertTableItem(tab, 156);
471 mdInsertTableItem(tab, 157);
472 mdInsertTableItem(tab, 158);
473 mdInsertTableItem(tab, 159);
474 mdInsertTableItem(tab, 160);
475 mdInsertTableItem(tab, 161);
476
477 tab = mdNewTable(MYSQL_DEFAULT);
478 mdInsertTableItem(tab, 223);
479 mdInsertTableItem(tab, 225);
480
481 tab = mdNewTable(SLP_DEFAULT);
482 mdInsertTableItem(tab, 128);
483 mdInsertTableItem(tab, 129);
484 mdInsertTableItem(tab, 130);
485
486 tab = mdNewTable(POP3_DEFAULT);
487 mdInsertTableItem(tab, 124);
488
489 tab = mdNewTable(RTSP_DEFAULT);
490 mdInsertTableItem(tab, 143);
491 mdInsertTableItem(tab, 144);
492 mdInsertTableItem(tab, 145);
493 mdInsertTableItem(tab, 146);
494 mdInsertTableItem(tab, 147);
495 mdInsertTableItem(tab, 148);
496 mdInsertTableItem(tab, 149);
497 mdInsertTableItem(tab, 150);
498 mdInsertTableItem(tab, 151);
499 mdInsertTableItem(tab, 152);
500 mdInsertTableItem(tab, 153);
501 mdInsertTableItem(tab, 154);
502
503 tab = mdNewTable(NNTP_DEFAULT);
504 mdInsertTableItem(tab, 172);
505 mdInsertTableItem(tab, 173);
506
507 tab = mdNewTable(SSL_DEFAULT);
508 mdInsertTableItem(tab, 186);
509 mdInsertTableItem(tab, 187);
510 mdInsertTableItem(tab, 188);
511 mdInsertTableItem(tab, 189);
512 mdInsertTableItem(tab, 190);
513 mdInsertTableItem(tab, 191);
514 mdInsertTableItem(tab, 192);
515 mdInsertTableItem(tab, 193);
516 mdInsertTableItem(tab, 194);
517 mdInsertTableItem(tab, 195);
518 mdInsertTableItem(tab, 196);
519 mdInsertTableItem(tab, 197);
520 mdInsertTableItem(tab, 198);
521 mdInsertTableItem(tab, 199);
522 mdInsertTableItem(tab, 200);
523 mdInsertTableItem(tab, 201);
524 mdInsertTableItem(tab, 202);
525 mdInsertTableItem(tab, 203);
526 mdInsertTableItem(tab, 204);
527 mdInsertTableItem(tab, 205);
528 mdInsertTableItem(tab, 206);
529 mdInsertTableItem(tab, 207);
530 mdInsertTableItem(tab, 244);
531 mdInsertTableItem(tab, 245);
532 mdInsertTableItem(tab, 246);
533 mdInsertTableItem(tab, 247);
534 mdInsertTableItem(tab, 248);
535 mdInsertTableItem(tab, 249);
536 mdInsertTableItem(tab, 250);
537 mdInsertTableItem(tab, 288);
538 mdInsertTableItem(tab, 443);
539 mdInsertTableItem(tab, 294);
540 mdInsertTableItem(tab, 295);
541 mdInsertTableItem(tab, 296);
542 mdInsertTableItem(tab, 299);
543 mdInsertTableItem(tab, 298);
544
545 tab = mdNewTable(INDEX_DEFAULT);
546 mdInsertTableItem(tab, 0);
547
548 tab = mdNewTable(DHCP_DEFAULT);
549 mdInsertTableItem(tab, 242);
550 mdInsertTableItem(tab, 243);
551 mdInsertTableItem(tab, 297);
552
553 tab = mdNewTable(P0F_DEFAULT);
554 mdInsertTableItem(tab, 36);
555 mdInsertTableItem(tab, 37);
556 mdInsertTableItem(tab, 107);
557 mdInsertTableItem(tab, 36|FB_IE_VENDOR_BIT_REVERSE);
558 mdInsertTableItem(tab, 37|FB_IE_VENDOR_BIT_REVERSE);
559 mdInsertTableItem(tab, 107|FB_IE_VENDOR_BIT_REVERSE);
560
561 tab = mdNewTable(RTP_DEFAULT);
562 mdInsertTableItem(tab, 287);
563
564 tab = mdNewTable(DNP_DEFAULT);
565 mdInsertTableItem(tab, 284);
566
567 tab = mdNewTable(MODBUS_DEFAULT);
568 mdInsertTableItem(tab, 285);
569
570 tab = mdNewTable(ENIP_DEFAULT);
571 mdInsertTableItem(tab, 286);
572
573 }
574
mdExporterExpandBuf(mdFlowExporter_t * exporter)575 static gboolean mdExporterExpandBuf(
576 mdFlowExporter_t *exporter)
577 {
578 g_debug("Expanding output buffer for exporter %s", exporter->name);
579
580 /* free the old buffer */
581 g_slice_free1(exporter->buf->buflen, exporter->buf->buf);
582 /* double the size */
583 exporter->buf->buflen = (exporter->buf->buflen * 2);
584 exporter->buf->buf = g_slice_alloc(exporter->buf->buflen);
585 if (exporter->buf->buf == NULL) {
586 return FALSE;
587 }
588 exporter->buf->cp = exporter->buf->buf;
589 return TRUE;
590 }
591
592
593 /**
594 * mdGetTableItem
595 *
596 * retrieve the name of the table or file associated with this info element
597 * id as given by the user, or by default.
598 *
599 */
mdGetTableItem(uint16_t id)600 static char * mdGetTableItem(
601 uint16_t id)
602 {
603
604 mdTableInfo_t *ret = NULL;
605
606 ret = g_hash_table_lookup(table_hash, GUINT_TO_POINTER((unsigned int)id));
607 if (ret) {
608 return ret->table_name;
609 }
610
611 return NULL;
612 }
613
614 /**
615 * mdNewFieldList
616 *
617 */
mdNewFieldList()618 mdFieldList_t *mdNewFieldList(
619 )
620 {
621 return g_slice_new0(mdFieldList_t);
622 }
623
624 /**
625 * mdNewFlowExporter
626 *
627 *
628 */
mdNewFlowExporter(mdTransportType_t type)629 mdFlowExporter_t *mdNewFlowExporter(
630 mdTransportType_t type)
631 {
632
633 mdFlowExporter_t *exporter = g_slice_new0(mdFlowExporter_t);
634
635 exporter->type = type;
636 exporter->mysql = NULL;
637 exporter->spec.host = NULL;
638 exporter->spec.svc = NULL;
639 exporter->spec.ssl_ca_file = NULL;
640 exporter->spec.ssl_cert_file = NULL;
641 exporter->spec.ssl_key_file = NULL;
642 exporter->spec.ssl_key_pass = NULL;
643 exporter->spec.vai = NULL;
644 exporter->spec.vssl_ctx = NULL;
645 exporter->delimiter = '|';
646 exporter->dpi_delimiter = 0;
647 exporter->dnsdedup = FALSE;
648 exporter->dnsdeduponly = FALSE;
649 exporter->no_flow_stats = FALSE;
650 exporter->timestamp_files = FALSE;
651 /* set default session initializer */
652 exporter->sess_init = mdInitExporterSession;
653
654 if (type == UDP) {
655 exporter->spec.transport = FB_UDP;
656 } else {
657 exporter->spec.transport = FB_TCP;
658 }
659
660 if (type == TEXT) {
661 exporter->buf = g_slice_new0(mdBuf_t);
662 exporter->buf->buf = g_slice_alloc(MD_MSGLEN_STD + 1);
663 exporter->buf->buflen = MD_MSGLEN_STD + 1;
664 exporter->buf->cp = exporter->buf->buf;
665 }
666
667 exporter->dpi_field_table = NULL;
668
669 exporter->metadata_export = FALSE;
670
671 return exporter;
672 }
673
674 /**
675 * mdInsertDPIFieldItem
676 *
677 *
678 */
mdInsertDPIFieldItem(mdFlowExporter_t * exporter,int ie)679 void mdInsertDPIFieldItem(
680 mdFlowExporter_t *exporter,
681 int ie)
682 {
683
684 int on = 1;
685
686 if (exporter->dpi_field_table == NULL) {
687 exporter->dpi_field_table = g_hash_table_new((GHashFunc)g_direct_hash,
688 (GEqualFunc)g_direct_equal);
689 if (exporter->dpi_field_table == NULL) {
690 g_warning("Can not create DPI Field List Hash Table.");
691 return;
692 }
693 }
694
695 g_hash_table_insert(exporter->dpi_field_table, GUINT_TO_POINTER(ie),
696 GUINT_TO_POINTER(on));
697 }
698
699
700 /**
701 * mdGetDPIItem
702 *
703 *
704 */
mdGetDPIItem(GHashTable * table,uint16_t id)705 static gboolean mdGetDPIItem(
706 GHashTable *table,
707 uint16_t id)
708 {
709 gboolean rc;
710 void *key = NULL;
711 gpointer value = NULL;
712
713 rc = g_hash_table_lookup_extended(table,
714 GUINT_TO_POINTER((unsigned int)id),
715 key, &value);
716
717 return rc;
718 }
719
720 /**
721 * mdExporterSetName
722 *
723 *
724 */
mdExporterSetName(mdFlowExporter_t * exporter,char * name)725 void mdExporterSetName(
726 mdFlowExporter_t *exporter,
727 char *name)
728 {
729 exporter->name = g_strdup(name);
730 }
731
732 /**
733 * mdExporterSetPort
734 *
735 *
736 */
mdExporterSetPort(mdFlowExporter_t * exporter,char * port)737 void mdExporterSetPort(
738 mdFlowExporter_t *exporter,
739 char *port)
740 {
741 exporter->spec.svc = g_strdup(port);
742 }
743
744 /**
745 * mdExporterSetHost
746 *
747 *
748 */
mdExporterSetHost(mdFlowExporter_t * exporter,char * host)749 void mdExporterSetHost(
750 mdFlowExporter_t *exporter,
751 char *host)
752 {
753 exporter->spec.host = g_strdup(host);
754 }
755
756 /**
757 * mdExporterSetRotate
758 *
759 *
760 */
mdExporterSetRotate(mdFlowExporter_t * exporter,uint32_t rotate)761 void mdExporterSetRotate(
762 mdFlowExporter_t *exporter,
763 uint32_t rotate)
764 {
765 exporter->rotate = rotate * 1000;
766 }
767
768 /**
769 * mdExporterSetFileSpec
770 *
771 *
772 */
mdExporterSetFileSpec(mdFlowExporter_t * exporter,char * spec)773 void mdExporterSetFileSpec(
774 mdFlowExporter_t *exporter,
775 char *spec)
776 {
777 exporter->outspec = g_strdup(spec);
778 }
779
780 /**
781 * mdExporterSetDelim
782 *
783 */
mdExporterSetDelim(mdFlowExporter_t * exporter,char * delim)784 void mdExporterSetDelim(
785 mdFlowExporter_t *exporter,
786 char *delim)
787 {
788 exporter->delimiter = *delim;
789 }
790
791 /**
792 * mdExporterSetDPIDelim
793 *
794 */
mdExporterSetDPIDelim(mdFlowExporter_t * exporter,char * delim)795 void mdExporterSetDPIDelim(
796 mdFlowExporter_t *exporter,
797 char *delim)
798 {
799 exporter->dpi_delimiter = *delim;
800 }
801
802 /**
803 * mdExporterSetMovePath
804 *
805 */
mdExporterSetMovePath(mdFlowExporter_t * exporter,char * path)806 void mdExporterSetMovePath(
807 mdFlowExporter_t *exporter,
808 char *path)
809 {
810 exporter->mv_path = g_strdup(path);
811 }
812
813 /**
814 * mdExporterSetNoFlow
815 *
816 *
817 */
mdExporterSetNoFlow(mdFlowExporter_t * exporter)818 void mdExporterSetNoFlow(
819 mdFlowExporter_t *exporter)
820 {
821 exporter->no_flow = TRUE;
822 }
823
824 /**
825 * mdExporterSetFlowExportLock
826 *
827 *
828 */
mdExporterSetLock(mdFlowExporter_t * exporter)829 void mdExporterSetLock(
830 mdFlowExporter_t *exporter)
831 {
832 exporter->lock = TRUE;
833 }
834
835 /**
836 * mdExporterGZIPFiles
837 *
838 */
mdExporterGZIPFiles(mdFlowExporter_t * exporter)839 void mdExporterGZIPFiles(
840 mdFlowExporter_t *exporter)
841 {
842 exporter->gzip = TRUE;
843 }
844
mdExporterDedupPerFlow(mdFlowExporter_t * exporter)845 void mdExporterDedupPerFlow(
846 mdFlowExporter_t *exporter)
847 {
848 exporter->dedup_per_flow = TRUE;
849 }
850
851 /**
852 * mdExporterSetFlowOnly
853 *
854 *
855 */
mdExporterSetFlowOnly(mdFlowExporter_t * exporter)856 gboolean mdExporterSetFlowOnly(
857 mdFlowExporter_t *exporter)
858 {
859 if (exporter->flowonly || exporter->dnsdedup ||
860 exporter->ssldedup || exporter->dns_rr_only ||
861 exporter->dedupconfig)
862 {
863 return FALSE;
864 }
865
866 exporter->flowonly = TRUE;
867 exporter->no_stats = 1;
868 exporter->sess_init = mdInitExporterSessionFlowOnly;
869
870 return TRUE;
871 }
872
873 /**
874 * mdExporterSetDPIOnly
875 *
876 *
877 */
mdExporterSetDPIOnly(mdFlowExporter_t * exporter)878 gboolean mdExporterSetDPIOnly(
879 mdFlowExporter_t *exporter)
880 {
881 if (exporter->flowonly)
882 {
883 return FALSE;
884 }
885
886 exporter->dpionly = TRUE;
887 exporter->no_stats = 1;
888 return TRUE;
889 }
890
891 /**
892 * mdExporterSetStats
893 *
894 *
895 */
mdExporterSetStats(mdFlowExporter_t * exporter,uint8_t mode)896 void mdExporterSetStats(
897 mdFlowExporter_t *exporter,
898 uint8_t mode)
899 {
900 if (exporter->flowonly || exporter->dpionly || exporter->dnsdeduponly ||
901 exporter->dns_rr_only || exporter->ssldeduponly)
902 {
903 exporter->no_stats = 0;
904 } else {
905 /* no_stats = 2 means JUST STATS!!! */
906 exporter->no_stats = mode;
907 }
908 }
909
910 /**
911 * mdExporterSetDeDup
912 *
913 *
914 */
mdExporterSetDNSDeDup(mdFlowExporter_t * exporter)915 void mdExporterSetDNSDeDup(
916 mdFlowExporter_t *exporter)
917 {
918 exporter->dnsdedup = TRUE;
919 }
920
921 /**
922 * mdExporterSetDeDupConfig
923 *
924 *
925 */
mdExporterSetDeDupConfig(mdFlowExporter_t * exporter)926 void mdExporterSetDeDupConfig(
927 mdFlowExporter_t *exporter)
928 {
929 exporter->dedupconfig = TRUE;
930 exporter->sess_init = mdInitExporterSessionDedupOnly;
931 }
932
933 /**
934 * mdExporterSetSSLDeDupConfig
935 *
936 *
937 */
mdExporterSetSSLDeDupConfig(mdFlowExporter_t * exporter)938 void mdExporterSetSSLDeDupConfig(
939 mdFlowExporter_t *exporter)
940 {
941 exporter->ssldedup = TRUE;
942 exporter->no_stats = 1;
943 }
944
945 /**
946 * mdExporterSetSSLDedupOnly
947 *
948 *
949 */
mdExporterSetSSLDeDupOnly(mdFlowExporter_t * exporter,gboolean dedup_only)950 gboolean mdExporterSetSSLDeDupOnly(
951 mdFlowExporter_t *exporter,
952 gboolean dedup_only)
953 {
954 if (exporter->flowonly || exporter->dnsdeduponly ||
955 exporter->dns_rr_only || exporter->deduponly)
956 {
957 return FALSE;
958 }
959
960 if (dedup_only) {
961 if (exporter->dnsdedup || exporter->ssldedup)
962 {
963 return FALSE;
964 }
965 exporter->no_flow = TRUE;
966 }
967
968 exporter->ssldedup = TRUE;
969 exporter->ssldeduponly = dedup_only;
970 exporter->no_stats = 1;
971 exporter->no_index = TRUE;
972 exporter->sess_init = mdInitExporterSessionSSLDedupOnly;
973 return TRUE;
974 }
975
976 /**
977 * mdExporterDedupOnly
978 *
979 */
mdExporterDedupOnly(mdFlowExporter_t * exporter)980 gboolean mdExporterDedupOnly(
981 mdFlowExporter_t *exporter)
982 {
983
984 if (exporter->flowonly || exporter->dnsdedup ||
985 exporter->ssldedup || exporter->dns_rr_only )
986 {
987 return FALSE;
988 }
989
990 exporter->deduponly = TRUE;
991 exporter->no_stats = 1;
992 exporter->no_index = TRUE;
993 exporter->no_flow = TRUE;
994 exporter->sess_init = mdInitExporterSessionDedupOnly;
995 return TRUE;
996 }
997
998
999 /**
1000 * mdExporterSetPrintHeader
1001 *
1002 *
1003 */
mdExporterSetPrintHeader(mdFlowExporter_t * exporter)1004 void mdExporterSetPrintHeader(
1005 mdFlowExporter_t *exporter)
1006 {
1007 exporter->print_header = TRUE;
1008 }
1009
1010 /**
1011 * mdExporterSetEscapeChars
1012 *
1013 *
1014 */
mdExporterSetEscapeChars(mdFlowExporter_t * exporter)1015 void mdExporterSetEscapeChars(
1016 mdFlowExporter_t *exporter)
1017 {
1018 exporter->escape_chars = TRUE;
1019 }
1020
1021 /**
1022 * mdExporterCompareNames
1023 *
1024 */
mdExporterCompareNames(mdFlowExporter_t * exporter,char * name)1025 gboolean mdExporterCompareNames(
1026 mdFlowExporter_t *exporter,
1027 char *name)
1028 {
1029
1030 if (!g_strcmp0(exporter->name, name)) {
1031 return TRUE;
1032 }
1033
1034 return FALSE;
1035 }
1036
mdExporterSetSSLConfig(mdFlowExporter_t * exporter,int * list,int type)1037 void mdExporterSetSSLConfig(
1038 mdFlowExporter_t *exporter,
1039 int *list,
1040 int type)
1041 {
1042 if (!exporter->ssl_config) {
1043 exporter->ssl_config = g_slice_new0(mdSSLConfig_t);
1044 }
1045
1046 if (type == 1) {
1047 exporter->ssl_config->issuer = list;
1048 } else if (type == 2) {
1049 exporter->ssl_config->subject = list;
1050 } else if (type == 3) {
1051 exporter->ssl_config->other = list;
1052 /* if there's a DPI field list - add any of these items in
1053 the DPI field list as well */
1054 if (exporter->dpi_field_table) {
1055 int i;
1056 for (i = 0; i < 300; i++) {
1057 if (list[i] == 1) {
1058 mdInsertDPIFieldItem(exporter, i);
1059 }
1060 }
1061 }
1062 } else if (type == 4) {
1063 exporter->ssl_config->extensions = list;
1064 }
1065 }
1066
1067
1068 /**
1069 * mdExporterSetDeDupOnly
1070 *
1071 *
1072 */
mdExporterSetDNSDeDupOnly(mdFlowExporter_t * exporter)1073 gboolean mdExporterSetDNSDeDupOnly(
1074 mdFlowExporter_t *exporter)
1075 {
1076 if (exporter->flowonly || exporter->ssldedup || exporter->dedupconfig ||
1077 exporter->dns_rr_only)
1078 {
1079 return FALSE;
1080 }
1081
1082 exporter->dnsdeduponly = TRUE;
1083 exporter->dnsdedup = TRUE;
1084 exporter->no_stats = 1;
1085 exporter->sess_init = mdInitExporterSessionDNSDedupOnly;
1086 exporter->no_flow = TRUE;
1087 return TRUE;
1088 }
1089
mdExporterGetDNSDedupStatus(mdFlowExporter_t * exporter)1090 gboolean mdExporterGetDNSDedupStatus(
1091 mdFlowExporter_t *exporter)
1092 {
1093 if (exporter->dnsdedup) {
1094 return TRUE;
1095 }
1096
1097 return FALSE;
1098 }
1099
mdExporterGetName(mdFlowExporter_t * exporter)1100 char *mdExporterGetName(
1101 mdFlowExporter_t *exporter)
1102 {
1103 return exporter->name;
1104 }
1105
mdExporterGetJson(mdFlowExporter_t * exporter)1106 gboolean mdExporterGetJson(
1107 mdFlowExporter_t *exporter)
1108 {
1109 return exporter->json;
1110 }
1111
mdExporterSetRemoveEmpty(mdFlowExporter_t * exporter)1112 void mdExporterSetRemoveEmpty(
1113 mdFlowExporter_t *exporter)
1114 {
1115 exporter->remove_empty = TRUE;
1116
1117 }
1118
mdExporterSetNoIndex(mdFlowExporter_t * exporter,gboolean val)1119 void mdExporterSetNoIndex(
1120 mdFlowExporter_t *exporter,
1121 gboolean val)
1122 {
1123 exporter->no_index = val;
1124 }
1125
mdExporterSetTimestampFiles(mdFlowExporter_t * exporter)1126 void mdExporterSetTimestampFiles(
1127 mdFlowExporter_t *exporter)
1128 {
1129 exporter->timestamp_files = TRUE;
1130 }
1131
mdExporterSetRemoveUploaded(mdFlowExporter_t * exporter)1132 void mdExporterSetRemoveUploaded(
1133 mdFlowExporter_t *exporter)
1134 {
1135 exporter->remove_uploaded = TRUE;
1136 }
1137
mdExporterSetId(mdFlowExporter_t * exporter,uint8_t id)1138 void mdExporterSetId(
1139 mdFlowExporter_t *exporter,
1140 uint8_t id)
1141 {
1142 exporter->id = id;
1143 }
1144
1145 /**
1146 * mdExporterSetNoFlowStats
1147 *
1148 */
mdExporterSetNoFlowStats(mdFlowExporter_t * exporter)1149 void mdExporterSetNoFlowStats(
1150 mdFlowExporter_t *exporter)
1151 {
1152 exporter->no_flow_stats = TRUE;
1153 }
1154
1155 /**
1156 * mdExporterSetJson
1157 *
1158 */
mdExporterSetJson(mdFlowExporter_t * exporter)1159 void mdExporterSetJson(
1160 mdFlowExporter_t *exporter)
1161 {
1162 exporter->json = TRUE;
1163 exporter->escape_chars = TRUE;
1164 }
1165
1166 /**
1167 * mdExportCustomList
1168 *
1169 *
1170 */
mdExportCustomList(mdFlowExporter_t * exporter,mdFieldList_t * list)1171 void mdExportCustomList(
1172 mdFlowExporter_t *exporter,
1173 mdFieldList_t *list)
1174 {
1175 exporter->custom_list = list;
1176 exporter->no_stats = 1;
1177 }
1178
mdExporterCustomListDPI(mdFlowExporter_t * exporter)1179 void mdExporterCustomListDPI(
1180 mdFlowExporter_t *exporter)
1181 {
1182 exporter->custom_list_dpi = TRUE;
1183 exporter->no_index = TRUE;
1184 }
1185
mdExporterGetType(mdFlowExporter_t * exporter)1186 int mdExporterGetType(
1187 mdFlowExporter_t *exporter)
1188 {
1189 return exporter->type;
1190 }
1191
1192 /**
1193 * mdExporterSetDNSRROnly
1194 *
1195 */
mdExporterSetDNSRROnly(mdFlowExporter_t * exporter,int mode)1196 gboolean mdExporterSetDNSRROnly(
1197 mdFlowExporter_t *exporter,
1198 int mode)
1199 {
1200 if (mode == 1 || mode == 2) {
1201 if (exporter->flowonly || exporter->ssldedup ||
1202 exporter->dedupconfig || exporter->dnsdedup)
1203 {
1204 return FALSE;
1205 }
1206 exporter->no_flow = TRUE;
1207 }
1208
1209 /* 1 is rr only */
1210 /* 2 is rr only full */
1211 /* 3 is rr */
1212 /* 4 is rr full */
1213
1214 exporter->dns_rr_only = mode;
1215 exporter->sess_init = mdInitExporterSessionDNSRROnly;
1216 exporter->no_stats = 1;
1217
1218 return TRUE;
1219
1220 }
1221
mdExporterSetSSLMD5Hash(mdFlowExporter_t * exporter)1222 gboolean mdExporterSetSSLMD5Hash(
1223 mdFlowExporter_t *exporter)
1224 {
1225 if (exporter->flowonly || exporter->dns_rr_only ||
1226 exporter->dnsdeduponly || exporter->dns_resp_only ||
1227 (exporter->no_stats == 2))
1228 {
1229 return FALSE;
1230 }
1231
1232 exporter->md5_hash = TRUE;
1233
1234 return TRUE;
1235 }
1236
1237
mdExporterSetSSLSHA1Hash(mdFlowExporter_t * exporter)1238 gboolean mdExporterSetSSLSHA1Hash(
1239 mdFlowExporter_t *exporter)
1240 {
1241 if (exporter->flowonly || exporter->dns_rr_only ||
1242 exporter->dnsdeduponly || exporter->dns_resp_only ||
1243 (exporter->no_stats == 2))
1244 {
1245 return FALSE;
1246 }
1247
1248 exporter->sha1_hash = TRUE;
1249
1250 return TRUE;
1251 }
1252
1253 /**
1254 * mdExporterSetDNSRespOnly
1255 *
1256 */
mdExporterSetDNSRespOnly(mdFlowExporter_t * exporter)1257 void mdExporterSetDNSRespOnly(
1258 mdFlowExporter_t *exporter)
1259 {
1260 exporter->dns_resp_only = TRUE;
1261 }
1262
1263 /**
1264 * mdExporterAddMySQLInfo
1265 *
1266 *
1267 */
mdExporterAddMySQLInfo(mdFlowExporter_t * exporter,char * user,char * password,char * db_name,char * db_host,char * table)1268 gboolean mdExporterAddMySQLInfo(
1269 mdFlowExporter_t *exporter,
1270 char *user,
1271 char *password,
1272 char *db_name,
1273 char *db_host,
1274 char *table)
1275 {
1276
1277 if (exporter->mysql == NULL) {
1278 exporter->mysql = g_slice_new0(mdMySQLInfo_t);
1279 }
1280 if (user) {
1281 exporter->mysql->user = g_strdup(user);
1282 }
1283 if (password) {
1284 exporter->mysql->password = g_strdup(password);
1285 }
1286 if (db_name) {
1287 exporter->mysql->db_name = g_strdup(db_name);
1288 }
1289 if (db_host) {
1290 exporter->mysql->db_host = g_strdup(db_host);
1291 }
1292 if (table) {
1293 exporter->mysql->table = g_strdup(table);
1294 }
1295
1296 #if HAVE_MYSQL
1297 if (exporter->mysql->user && exporter->mysql->password &&
1298 exporter->mysql->db_name)
1299 {
1300 exporter->mysql->conn = mysql_init(NULL);
1301 /* #if MYSQL_VERSION_ID >= 50013 */
1302 my_bool reconnect = 1;
1303 mysql_options(exporter->mysql->conn, MYSQL_OPT_RECONNECT, &reconnect);
1304 /* #endif */
1305 if (exporter->mysql->conn == NULL) {
1306 g_warning("Error Initializing Connection %u: %s\n",
1307 mysql_errno(exporter->mysql->conn),
1308 mysql_error(exporter->mysql->conn));
1309 return FALSE;
1310 }
1311 if (mysql_real_connect(exporter->mysql->conn, exporter->mysql->db_host,
1312 exporter->mysql->user,exporter->mysql->password,
1313 exporter->mysql->db_name, 0, NULL, 0) == NULL)
1314 {
1315 g_warning("Error Connection %u: %s",
1316 mysql_errno(exporter->mysql->conn),
1317 mysql_error(exporter->mysql->conn));
1318 return FALSE;
1319 }
1320 }
1321 #else
1322 g_warning("Invalid Keyword: super_mediator not configured for MySQL.");
1323 #endif
1324 return TRUE;
1325 }
1326
1327 /**
1328 * mdExporterSetMetadataExport
1329 *
1330 *
1331 */
mdExporterSetMetadataExport(mdFlowExporter_t * exporter)1332 void mdExporterSetMetadataExport(
1333 mdFlowExporter_t *exporter)
1334 {
1335 exporter->metadata_export = TRUE;
1336 }
1337
1338 /**
1339 * mdLockFile
1340 *
1341 * "Locks" a file. Really just prepends "." to the filename
1342 *
1343 */
mdLockFile(GString * path)1344 static void mdLockFile(
1345 GString *path)
1346 {
1347 char *find = NULL;
1348 gssize pos;
1349
1350
1351 find = g_strrstr(path->str, "/");
1352 if (find) {
1353 pos = find - path->str + 1;
1354 g_string_insert_c(path, pos, '.');
1355 } else {
1356 g_string_prepend_c(path, '.');
1357 }
1358
1359 }
1360
1361 /**
1362 * mdUnlockFile
1363 *
1364 * "Unlocks" a file. Really just renames the file.
1365 *
1366 */
mdUnlockFile(char * path)1367 static void mdUnlockFile(
1368 char *path)
1369 {
1370 GString *lock_name = NULL;
1371 char *find = NULL;
1372 gssize pos;
1373
1374
1375 lock_name = g_string_new("");
1376 g_string_assign(lock_name, path);
1377 find = g_strrstr(lock_name->str, "/");
1378 if (find) {
1379 pos = find - lock_name->str + 1;
1380 g_string_insert_c(lock_name, pos, '.');
1381 } else {
1382 g_string_prepend_c(lock_name, '.');
1383 }
1384 g_debug("Unlocking File %s", path);
1385
1386 if (g_rename(lock_name->str, path) != 0) {
1387 g_warning("Error renaming file from %s to %s",
1388 lock_name->str, path);
1389 }
1390 g_string_free(lock_name, TRUE);
1391 }
1392
1393
1394
1395
1396 /**
1397 * mdExportMultiFiles
1398 *
1399 *
1400 */
mdExportMultiFiles(mdFlowExporter_t * exporter)1401 gboolean mdExportMultiFiles(
1402 mdFlowExporter_t *exporter)
1403 {
1404 static gboolean on = FALSE;
1405 int offset;
1406 char *hold_spec;
1407
1408 if (!on) {
1409 exporter->multi_files = TRUE;
1410 on = TRUE;
1411 if (!g_file_test(exporter->outspec, G_FILE_TEST_IS_DIR)) {
1412 fprintf(stderr, "Error: MULTI_FILES requires PATH to be a File "
1413 "Directory\n");
1414 return FALSE;
1415 }
1416 offset = strlen(exporter->outspec);
1417 if (exporter->outspec[offset-1] != '/') {
1418 hold_spec = g_strconcat(exporter->outspec, "/", NULL);
1419 g_free(exporter->outspec);
1420 exporter->outspec = hold_spec;
1421 }
1422
1423 return TRUE;
1424 }
1425
1426 fprintf(stderr, "MULTI_FILES feature only valid for 1 Exporter\n");
1427 /* only 1 exporter can turn this feature on */
1428 return FALSE;
1429 }
1430
1431 /**
1432 * mdExporterFree
1433 *
1434 *
1435 */
mdExporterFree(mdFlowExporter_t * exporter)1436 void mdExporterFree(
1437 mdFlowExporter_t *exporter)
1438 {
1439 g_slice_free(mdFlowExporter_t, exporter);
1440 }
1441
1442 /**
1443 * mdLoadFile
1444 *
1445 * load a dpi file into the database.
1446 *
1447 */
mdLoadFile(mdFlowExporter_t * exporter,char * table,char * filename)1448 static void mdLoadFile(
1449 mdFlowExporter_t *exporter,
1450 char *table,
1451 char *filename)
1452 {
1453 #if HAVE_MYSQL
1454 char query[500];
1455 int err;
1456 unsigned long bid = 0;
1457 unsigned long aid = 0;
1458 mdMySQLInfo_t *mysql = exporter->mysql;
1459
1460 if (mysql->conn) {
1461 sprintf(query, "LOAD DATA LOCAL INFILE '%s' INTO TABLE %s.%s"
1462 " FIELDS TERMINATED BY '%c'", filename, mysql->db_name,
1463 table, exporter->delimiter);
1464 err = mysql_query(mysql->conn, query);
1465
1466 #if MYSQL_VERSION_ID >=50013
1467 bid = mysql_thread_id(mysql->conn);
1468 mysql_ping(mysql->conn);
1469 aid = mysql_thread_id(mysql->conn);
1470 #endif
1471 /* try again for specific errors */
1472 if (err) {
1473 if ((mysql_errno(mysql->conn) == 0) ||
1474 (mysql_errno(mysql->conn) == 1143))
1475 {
1476 g_debug("%s: Error importing local file %u: %s. "
1477 "Trying query again without LOCAL keyword.",
1478 exporter->name, mysql_errno(mysql->conn),
1479 mysql_error(mysql->conn));
1480 sprintf(query, "LOAD DATA INFILE '%s' INTO TABLE %s"
1481 " FIELDS TERMINATED BY '%c'", filename,
1482 table, exporter->delimiter);
1483 err = mysql_query(mysql->conn, query);
1484 } else if ( bid != aid ) {
1485 g_message("%s: Reconnected to MySQL Database.",exporter->name);
1486 sprintf(query, "LOAD DATA LOCAL INFILE '%s' INTO TABLE %s"
1487 " FIELDS TERMINATED BY '%c'", filename,
1488 table, exporter->delimiter);
1489 err = mysql_query(mysql->conn, query);
1490
1491 }
1492 }
1493
1494 if (err) {
1495 g_warning("%s: Error loading data %u:%s", exporter->name,
1496 mysql_errno(mysql->conn), mysql_error(mysql->conn));
1497 } else {
1498 g_debug("%s: Successfully imported file %s to table '%s'",
1499 exporter->name, filename, table);
1500 if (exporter->remove_uploaded) {
1501 if (!g_remove(filename)) {
1502 g_debug("%s: Removed Imported File '%s'", exporter->name,
1503 filename);
1504 } else {
1505 g_warning("%s: Error removing file: %d", exporter->name,
1506 g_file_error_from_errno(errno));
1507 }
1508 }
1509 }
1510 }
1511 #endif
1512
1513 }
1514
1515
1516
1517 /**
1518 * mdGetTableFile
1519 *
1520 * returns the file pointer for this element id.
1521 */
mdGetTableFile(mdFlowExporter_t * exporter,uint16_t id)1522 static FILE * mdGetTableFile(
1523 mdFlowExporter_t *exporter,
1524 uint16_t id)
1525 {
1526
1527 mdTableInfo_t *ret = NULL;
1528 GString *file_name;
1529 uint64_t start_secs;
1530
1531 if (!table_hash) {
1532 mdBuildDefaultTableHash();
1533 }
1534 ret = g_hash_table_lookup(table_hash, GUINT_TO_POINTER((unsigned int)id));
1535 if (ret) {
1536 if (!ret->table_file ||
1537 (ret->last_rotate_ms &&
1538 (exporter->last_rotate_ms != ret->last_rotate_ms)))
1539 {
1540 file_name = g_string_new("");
1541 g_string_assign(file_name, exporter->outspec);
1542 if (ret->table_file) {
1543 mdCloseAndUnlock(exporter, ret->table_file, ret->file_name,
1544 ret->table_name);
1545 /*if (exporter->lock) {
1546 mdUnlockFile(ret->file_name);
1547 }
1548 fclose(ret->table_file);
1549 if (exporter->mysql) {
1550 mdLoadFile(exporter, ret->table_name, ret->file_name);
1551 }
1552 g_free(ret->file_name);*/
1553 }
1554 start_secs = exporter->last_rotate_ms / 1000;
1555 g_string_append_printf(file_name, "%s.txt", ret->table_name);
1556 if (exporter->timestamp_files) {
1557 md_util_time_g_string_append(file_name, start_secs, TIME_FMT);
1558 } else {
1559 g_string_append_printf(file_name, "%d", ret->serial);
1560 }
1561 ret->serial++;
1562 ret->file_name = g_strdup(file_name->str);
1563 if (exporter->lock) {
1564 mdLockFile(file_name);
1565 }
1566 ret->table_file = fopen(file_name->str, "w");
1567 ret->last_rotate_ms = exporter->last_rotate_ms;
1568 if (ret->table_file == NULL) {
1569 g_warning("%s: Error Opening File %s", exporter->name,
1570 file_name->str);
1571 }
1572 g_debug("%s: Opening Text File %s", exporter->name,
1573 file_name->str);
1574 g_string_free(file_name, TRUE);
1575 }
1576 return ret->table_file;
1577 }
1578
1579 return NULL;
1580 }
1581
1582
1583 /**
1584 * mdExporterVerifySetup
1585 *
1586 * verifies that the exporters are appropriately setup
1587 * and that all configuration parameters were used
1588 * correctly
1589 *
1590 * @param exp to be verified
1591 * @param err not really used
1592 * @return true if correct
1593 */
mdExporterVerifySetup(mdFlowExporter_t * exporter)1594 gboolean mdExporterVerifySetup(
1595 mdFlowExporter_t *exporter)
1596 {
1597 switch (exporter->type) {
1598 case SPREAD:
1599 #if HAVE_SPREAD
1600 if (md_out_groups == NULL) {
1601 fprintf(stderr, "Error EXPORTER %s: SPREAD Exporter "
1602 "Requires AT LEAST ONE group.\n", exporter->name);
1603 return FALSE;
1604 } else if (exporter->outspec == NULL) {
1605 fprintf(stderr, "Error EXPORTER %s: SPREAD Exporter Requires DAEMON Name.\n",
1606 exporter->name);
1607 return FALSE;
1608 }
1609 if (exporter->ssl_config) {
1610 fprintf(stderr, "Error EXPORTER %s: SSL_CONFIG does not apply to SPREAD"
1611 " Exporters.\n", exporter->name);
1612 return FALSE;
1613 }
1614 #else
1615 fprintf(stderr, "Error: SPREAD Not enabled.\n");
1616 return FALSE;
1617 #endif
1618 break;
1619 case FILEHANDLER:
1620 if (exporter->outspec == NULL) {
1621 fprintf(stderr, "Error: FILE Exporter %s Requires a FILE\n", exporter->name);
1622 return FALSE;
1623 }
1624 if (exporter->lock && exporter->rotate == 0) {
1625 fprintf(stderr, "Error EXPORTER %s: LOCK Only valid with ROTATE\n",
1626 exporter->name);
1627 return FALSE;
1628 }
1629 if (exporter->timestamp_files) {
1630 g_debug("Keyword TIMESTAMP_FILES is ignored for FILEHANDLER "
1631 "Exporters %s\n", exporter->name);
1632 exporter->timestamp_files = FALSE;
1633 }
1634 exporter->remove_empty = TRUE;
1635 case TCP:
1636 case UDP:
1637 if (exporter->type != FILEHANDLER) {
1638 if (exporter->spec.host == NULL) {
1639 exporter->spec.host = g_strdup("localhost");
1640 }
1641 if (exporter->spec.svc == NULL) {
1642 fprintf(stderr, "Error: TCP/UDP Exporter %s Requires PORT\n", exporter->name);
1643 return FALSE;
1644 }
1645 }
1646 if (exporter->ssl_config) {
1647 fprintf(stderr, "Error: SSL_CONFIG does not apply to IPFIX"
1648 " Exporters.\n Remove ISSUER, SUBJECT, EXTENSIONS, OTHER keywords\n");
1649 return FALSE;
1650 }
1651 if (exporter->no_flow == 0) {
1652 exporter->sess_init = mdInitExporterSession;
1653 }
1654 if (exporter->dnsdeduponly) {
1655 if (exporter->ssldedup || exporter->dedupconfig || exporter->flowonly
1656 || exporter->dns_rr_only)
1657 {
1658 fprintf(stderr, "Error exporter %s: DNS_DEDUP_ONLY not permitted with SSL_DEDUP,"
1659 " DEDUP_CONFIG, FLOW_ONLY, or DNS_RR\n", exporter->name);
1660 return FALSE;
1661 }
1662 }
1663 if (exporter->ssldeduponly) {
1664 if (exporter->dnsdedup || exporter->dedupconfig || exporter->flowonly
1665 || exporter->dns_rr_only)
1666 {
1667 fprintf(stderr,"Error exporter %s: SSL_DEDUP_ONLY not permitted with DNS_DEDUP,"
1668 " DEDUP_CONFIG, FLOW_ONLY, or DNS_RR\n", exporter->name);
1669 return FALSE;
1670 }
1671 }
1672 if (exporter->deduponly) {
1673 if (exporter->dnsdedup || exporter->ssldedup || exporter->flowonly
1674 || exporter->dns_rr_only)
1675 {
1676 fprintf(stderr,"Error exporter %s: DEDUP_ONLY not permitted with DNS_DEDUP,"
1677 " DEDUP_CONFIG, FLOW_ONLY, or DNS_RR\n", exporter->name);
1678 return FALSE;
1679 }
1680 }
1681 if ((exporter->dns_rr_only == 1) || (exporter->dns_rr_only == 2)) {
1682 if (exporter->dnsdedup || exporter->ssldedup || exporter->flowonly
1683 || exporter->dedupconfig)
1684 {
1685 fprintf(stderr,"Error exporter %s: DNS_RR_ONLY not permitted with DNS_DEDUP,"
1686 " DEDUP_CONFIG, FLOW_ONLY, or SSL_DEDUP\n", exporter->name);
1687 return FALSE;
1688 }
1689 }
1690 break;
1691 case TEXT:
1692 exporter->BLprint_fn = mdExportBL;
1693 exporter->VLprint_fn = mdAppendDPIStr;
1694 exporter->remove_empty = TRUE;
1695 if (exporter->custom_list && !exporter->json) {
1696 mdSetFieldListDecoratorCustom(exporter->custom_list,
1697 exporter->delimiter);
1698 }
1699
1700 if (exporter->outspec == NULL) {
1701 fprintf(stderr, "Error: TEXT Exporter %s Requires "
1702 "a FILE or DIRECTORY Path.\n", exporter->name);
1703 return FALSE;
1704 }
1705 if (exporter->lock && exporter->rotate ==0) {
1706 fprintf(stderr, "Error EXPORTER %s: LOCK Only valid with ROTATE\n",
1707 exporter->name);
1708 return FALSE;
1709 }
1710 if (exporter->multi_files && !exporter->dpionly) {
1711 fprintf(stderr, "Error EXPORTER %s: MULTI_FILES Only Valid "
1712 "with DPI_ONLY\n", exporter->name);
1713 return FALSE;
1714 }
1715 if (exporter->multi_files && exporter->dnsdedup) {
1716 fprintf(stderr, "Error EXPORTER %s: MULTI_FILES not valid with DEDUP\n",
1717 exporter->name);
1718 return FALSE;
1719 }
1720 if (exporter->timestamp_files && !exporter->rotate) {
1721 fprintf(stderr, "Error EXPORTER %s: TIMESTAMP_FILES only valid with ROTATE\n",
1722 exporter->name);
1723 return FALSE;
1724 }
1725
1726 if (exporter->multi_files && exporter->dpi_field_table) {
1727 fprintf(stderr, "Error EXPORTER %s: Invalid DPI_FIELD_LIST with MULTI_FILES. "
1728 "Use DPI_CONFIG block to configure MULTI_FILES.\n",
1729 exporter->name);
1730 return FALSE;
1731 }
1732
1733 if (exporter->multi_files && table_hash &&
1734 (exporter->md5_hash || exporter->sha1_hash)) {
1735 fprintf(stderr, "Error EXPORTER %s: For MULTI_FILES USE 299 for MD5_HASH or "
1736 " 298 for SHA1_HASH in the DPI_CONFIG block.\n",
1737 exporter->name);
1738 return FALSE;
1739 }
1740
1741 if (exporter->multi_files && table_hash) {
1742 if (mdGetTableItem(299)) {
1743 exporter->md5_hash = TRUE;
1744 }
1745 if (mdGetTableItem(298)) {
1746 exporter->sha1_hash = TRUE;
1747 }
1748 }
1749
1750 if (exporter->custom_list_dpi) {
1751 exporter->BLprint_fn = mdExportBLCustomList;
1752 }
1753
1754 if (!exporter->custom_list && exporter->custom_list_dpi) {
1755 mdFieldList_t *item = NULL;
1756 item = mdCreateFieldList(NONE_FIELD);
1757 mdExporterSetDPIOnly(exporter);
1758 mdExportCustomList(exporter, item);
1759 }
1760 if (exporter->dedupconfig && !exporter->json) {
1761 exporter->deduponly = TRUE;
1762 exporter->no_stats = 1;
1763 }
1764
1765 /* create a basic flow printing custom list */
1766 if (!exporter->custom_list) {
1767 if (!exporter->multi_files && !exporter->dnsdeduponly
1768 && !exporter->dpionly && !exporter->ssldeduponly &&
1769 !exporter->deduponly && !exporter->dns_rr_only)
1770 {
1771 /* if JSON - don't print payload */
1772 gboolean payload_on = exporter->json ? FALSE : TRUE;
1773 exporter->custom_list = mdCreateBasicFlowList(payload_on);
1774 exporter->no_stats = 0;
1775 if (!exporter->json) {
1776 mdSetFieldListDecoratorBasic(exporter->custom_list,
1777 exporter->delimiter);
1778 }
1779 if (!exporter->flowonly) {
1780 /* turn on DPI... */
1781 exporter->basic_list_dpi = TRUE;
1782 }
1783 }
1784 if (exporter->json && exporter->dpionly) {
1785 exporter->custom_list = mdCreateIndexFlowList();
1786 exporter->custom_list_dpi = TRUE;
1787 }
1788 }
1789
1790 if (exporter->dpionly && exporter->custom_list &&
1791 (!exporter->custom_list_dpi && !exporter->basic_list_dpi))
1792 {
1793 fprintf(stderr, "Error Exporter %s: Specified 'DPI_ONLY' "
1794 "but DPI not listed in custom FIELD list.\n",
1795 exporter->name);
1796 return FALSE;
1797 }
1798 if (exporter->dpi_delimiter == 0) {
1799 /* not set by user */
1800 exporter->dpi_delimiter = exporter->delimiter;
1801 }
1802 if (exporter->flowonly && exporter->custom_list &&
1803 exporter->custom_list_dpi)
1804 {
1805 g_warning("FLOW_ONLY keyword for EXPORTER %s "
1806 "is ignored due to DPI in "
1807 "custom FIELD list.", exporter->name);
1808 }
1809 if (exporter->flowonly && exporter->dpi_field_table) {
1810 g_warning("FLOW_ONLY keyword is present with DPI_FIELD_LIST. "
1811 "Ignoring DPI_FIELD_LIST for EXPORTER %s",
1812 exporter->name);
1813 }
1814
1815 if (!exporter->multi_files && exporter->rotate) {
1816 exporter->timestamp_files = TRUE;
1817 }
1818 if (exporter->dnsdeduponly) {
1819 if (exporter->custom_list) {
1820 g_warning("Warning: FIELD list is ignored due to"
1821 " presence of DNS_DEDUP_ONLY keyword in"
1822 " EXPORTER %s.", exporter->name);
1823 }
1824 }
1825
1826 if (exporter->ssldedup) {
1827 if (exporter->multi_files) {
1828 fprintf(stderr, "Error: MULTI_FILES not compatible"
1829 " with SSL_DEDUP_ONLY or SSL_DEDUP for EXPORTER %s\n",
1830 exporter->name);
1831 return FALSE;
1832 }
1833 }
1834
1835 if (exporter->ssldeduponly) {
1836 if (exporter->custom_list) {
1837 g_warning("Warning: FIELD list is ignored due to"
1838 " presence of SSL_DEDUP_ONLY keyword for "
1839 "EXPORTER %s", exporter->name);
1840 }
1841 }
1842
1843 if (exporter->mysql) {
1844 if (exporter->flowonly) {
1845 if (exporter->mysql->table == NULL) {
1846 exporter->mysql->table = g_strdup(INDEX_DEFAULT);
1847 }
1848 }
1849 if (exporter->dnsdeduponly) {
1850 if (exporter->mysql->table == NULL) {
1851 exporter->mysql->table = g_strdup(DNS_DEDUP_DEFAULT);
1852 }
1853 }
1854 if (exporter->no_stats == 2) {
1855 if (exporter->mysql->table == NULL) {
1856 exporter->mysql->table = g_strdup(YAF_STATS_DEFAULT);
1857 }
1858 }
1859 if (exporter->custom_list) {
1860 if (exporter->mysql->table == NULL) {
1861 fprintf(stderr, "Error: Custom FIELD List with MySQL import "
1862 "requires MYSQL_TABLE name for EXPORTER %s.\n",
1863 exporter->name);
1864 return FALSE;
1865 }
1866 }
1867 }
1868
1869 if (exporter->dedupconfig && !exporter->json) {
1870 int offset;
1871 char *hold_spec;
1872
1873 if (!g_file_test(exporter->outspec, G_FILE_TEST_IS_DIR)) {
1874 fprintf(stderr, "Error EXPORTER %s: DEDUP_CONFIG requires "
1875 "PATH to be a File Directory\n", exporter->name);
1876 return FALSE;
1877 }
1878 offset = strlen(exporter->outspec);
1879 if (exporter->outspec[offset-1] != '/') {
1880 hold_spec = g_strconcat(exporter->outspec, "/", NULL);
1881 g_free(exporter->outspec);
1882 exporter->outspec = hold_spec;
1883 }
1884 }
1885
1886 if (exporter->json) {
1887
1888 if (exporter->multi_files) {
1889 fprintf(stderr, "Error EXPORTER %s: MULTI_FILES not valid with JSON\n",
1890 exporter->name);
1891 return FALSE;
1892 }
1893
1894 if (exporter->print_header) {
1895 g_warning("PRINT_HEADER is ignored with JSON.");
1896 exporter->print_header = FALSE;
1897 }
1898
1899 if (exporter->custom_list) {
1900 mdSetFieldListDecoratorJSON(exporter->custom_list);
1901 }
1902 exporter->escape_chars = TRUE;
1903
1904 }
1905
1906 if (exporter->multi_files) {
1907 exporter->BLprint_fn = mdExportBLMultiFiles;
1908 exporter->VLprint_fn = mdAppendDPIStrMultiFiles;
1909 }
1910 if (exporter->json) {
1911 exporter->BLprint_fn = mdJsonizeBLElement;
1912 exporter->VLprint_fn = mdJsonizeVLElement;
1913 }
1914
1915 break;
1916 default:
1917 /* this really should never happen */
1918 fprintf(stderr, "Error: Invalid Transport for Exporter %s\n",
1919 exporter->name);
1920 return FALSE;
1921
1922 }
1923
1924 if (exporter->deduponly && !exporter->dedupconfig) {
1925 fprintf(stderr, "Error: DEDUP_ONLY was set for Exporter %s "
1926 "but no corresponding DEDUP_CONFIG block was found.\n", exporter->name);
1927 return FALSE;
1928 }
1929
1930 num_exporters++;
1931
1932 mdExporterSetId(exporter, num_exporters);
1933
1934 if (!exporter->name) {
1935 exporter->name = g_strdup_printf("E%d", exporter->id);
1936 }
1937
1938 return TRUE;
1939 }
1940
1941 /**
1942 * mdExporterMoveFile
1943 *
1944 *
1945 */
mdExporterMoveFile(char * file,char * new_dir)1946 static GString * mdExporterMoveFile(
1947 char *file,
1948 char *new_dir)
1949 {
1950
1951 GString *new_file = NULL;
1952 char *filename;
1953
1954 filename = g_strrstr(file, "/");
1955
1956 new_file = g_string_new("");
1957
1958 g_string_append_printf(new_file, "%s", new_dir);
1959 g_string_append_printf(new_file, "%s", filename);
1960
1961 if (g_rename(file, new_file->str) != 0) {
1962 g_string_free(new_file, TRUE);
1963 return NULL;
1964 }
1965
1966 return new_file;
1967 }
1968
1969
1970
1971 /**
1972 * mdCloseAndUnlock
1973 *
1974 * close a file, unlock it, possibly remove it.
1975 *
1976 */
mdCloseAndUnlock(mdFlowExporter_t * exporter,FILE * fp,char * filename,char * table)1977 static void mdCloseAndUnlock(
1978 mdFlowExporter_t *exporter,
1979 FILE *fp,
1980 char *filename,
1981 char *table)
1982 {
1983 gboolean rm = FALSE;
1984 GString *mv_name = NULL;
1985 char *table_name = NULL;
1986
1987 if (fp == NULL || filename == NULL) {
1988 return;
1989 }
1990
1991 if (filename[0] != '-' && (strlen(filename) != 1)) {
1992 g_debug("%s: Closing File %s", exporter->name, filename);
1993 }
1994
1995 if (exporter->remove_empty) {
1996 fseek(fp, 0L, SEEK_END);
1997 if (!ftell(fp)) {
1998 rm = TRUE;
1999 }
2000 }
2001
2002 fclose(fp);
2003
2004 if (exporter->lock) {
2005 mdUnlockFile(filename);
2006 }
2007
2008 if (rm) {
2009 g_debug("%s: Removing Empty File %s", exporter->name, filename);
2010 g_remove(filename);
2011 }
2012
2013 fp = NULL;
2014
2015 if (exporter->mysql && !rm) {
2016 if (exporter->flowonly || exporter->dnsdeduponly ||
2017 exporter->multi_files || exporter->no_stats == 2 ||
2018 exporter->custom_list)
2019 {
2020 table_name = table ? table : exporter->mysql->table;
2021 mdLoadFile(exporter, table_name, filename);
2022 /* don't compress if already removed */
2023 if (exporter->remove_uploaded) {
2024 if (filename) {
2025 g_free(filename);
2026 }
2027 return;
2028 }
2029 }
2030 }
2031
2032 if (exporter->mv_path && !rm) {
2033 mv_name = mdExporterMoveFile(filename, exporter->mv_path);
2034 if (!mv_name) {
2035 g_warning("Unable to move file to %s", exporter->mv_path);
2036 }
2037 }
2038
2039 if (exporter->gzip && !rm) {
2040 if (mv_name) {
2041 md_util_compress_file(mv_name->str);
2042 } else {
2043 md_util_compress_file(filename);
2044 }
2045 }
2046
2047 if (mv_name) {
2048 g_string_free(mv_name, TRUE);
2049 }
2050
2051 if (filename) {
2052 g_free(filename);
2053 }
2054 }
2055
2056
2057
2058
2059 /**
2060 * mdOpenFileExport
2061 *
2062 * open an IPFIX file, close the current IPFIX file
2063 *
2064 */
mdOpenFileExport(mdFlowExporter_t * exporter,const char * path,GError ** err)2065 static fBuf_t *mdOpenFileExport(
2066 mdFlowExporter_t *exporter,
2067 const char *path,
2068 GError **err)
2069 {
2070
2071 fBuf_t *fbuf = NULL;
2072 fbSession_t *session;
2073
2074 if (exporter == NULL) {
2075 exporter = g_new0(mdFlowExporter_t, 1);
2076 }
2077
2078
2079 if (strlen(path) == 1 && path[0] == '-') {
2080 g_debug("%s: Writing to stdout", exporter->name);
2081 exporter->lfp = stdout;
2082 } else {
2083 exporter->lfp = fopen(path, "w");
2084
2085 g_debug("%s: Opening File %s", exporter->name, path);
2086 }
2087
2088 if ( exporter->lfp == NULL ) {
2089 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2090 "%s: Can not open file %s for writing", exporter->name,
2091 path);
2092 return NULL;
2093 }
2094
2095 exporter->exporter = fbExporterAllocFP(exporter->lfp);
2096
2097 if ( exporter->exporter == NULL ) {
2098 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2099 "%s: Error creating the exporter", exporter->name);
2100 return NULL;
2101 }
2102
2103 if (!(session = exporter->sess_init(NULL, err, exporter->no_stats, exporter->metadata_export))) {
2104 return NULL;
2105 }
2106
2107 fbuf = fBufAllocForExport(session, exporter->exporter);
2108
2109 if (!fbSessionExportTemplates(session, err)) {
2110 if (fbuf) fBufFree(fbuf);
2111 return NULL;
2112 }
2113
2114 return fbuf;
2115
2116 }
2117
2118 /**
2119 * mdOpenTextFileExport
2120 *
2121 * open a new text file, close the current one
2122 *
2123 *
2124 */
mdOpenTextFileExport(mdFlowExporter_t * exporter,const char * path,GError ** err)2125 static gboolean mdOpenTextFileExport(
2126 mdFlowExporter_t *exporter,
2127 const char *path,
2128 GError **err)
2129 {
2130 GString *str;
2131 size_t rc;
2132
2133 if (exporter == NULL) {
2134 exporter = g_new0(mdFlowExporter_t, 1);
2135 }
2136
2137 if (strlen(path) == 1 && path[0] == '-') {
2138 g_debug("%s: Writing Text to stdout", exporter->name);
2139 exporter->lfp = stdout;
2140 } else {
2141 g_debug("%s: Opening Text File: %s", exporter->name, path);
2142 exporter->lfp = fopen(path, "w+");
2143 }
2144 if (exporter->lfp == NULL) {
2145 if (errno == 2) {
2146 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2147 "%s: Error opening file %s: No such file or directory",
2148 exporter->name, path);
2149 } else {
2150 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2151 "%s: Can not open file %s for writing", exporter->name,
2152 path);
2153 }
2154 return FALSE;
2155 }
2156
2157 if (exporter->print_header) {
2158 str = g_string_new("");
2159 mdPrintBasicHeader(str, exporter->delimiter);
2160 rc = fwrite(str->str, 1, str->len, exporter->lfp);
2161
2162 if (rc != str->len) {
2163 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2164 "%s: Error writing to file: %s\n", exporter->name,
2165 strerror(errno));
2166 return FALSE;
2167 }
2168 g_string_free(str, TRUE);
2169 }
2170
2171 return TRUE;
2172 }
2173
2174
2175 /**
2176 * mdOutputClose
2177 *
2178 * emit the fbuf, and free it.
2179 *
2180 */
mdOutputClose(fBuf_t * fbuf,gboolean flush,GError ** err)2181 static gboolean mdOutputClose(
2182 fBuf_t *fbuf,
2183 gboolean flush,
2184 GError **err)
2185 {
2186
2187 gboolean ok = TRUE;
2188
2189 if (fbuf == NULL) {
2190 return ok;
2191 }
2192
2193 if (flush) {
2194 ok = fBufEmit(fbuf, err);
2195 }
2196
2197 fBufFree(fbuf);
2198 fbuf = NULL;
2199
2200 return ok;
2201
2202 }
2203
2204 /**
2205 * mdFileOpenRotater
2206 *
2207 * get a new filename for file rotaters in the format of
2208 * outspec-TIME-serial_no
2209 *
2210 */
mdFileOpenRotater(mdFlowExporter_t * exporter)2211 static GString *mdFileOpenRotater(
2212 mdFlowExporter_t *exporter)
2213 {
2214 GString *namebuf = NULL;
2215 static uint32_t serial = 0;
2216 time_t cur_time= time(NULL);
2217
2218 namebuf = g_string_new("");
2219
2220 if (exporter->type == TEXT) {
2221 g_string_append_printf(namebuf, "%s.", exporter->outspec);
2222 } else {
2223 g_string_append_printf(namebuf, "%s-", exporter->outspec);
2224 }
2225
2226 if (exporter->timestamp_files) {
2227 uint64_t flow_secs = exporter->last_rotate_ms /1000;
2228 md_util_time_g_string_append(namebuf, flow_secs, TIME_FMT);
2229 } else {
2230 md_util_time_g_string_append(namebuf, cur_time, TIME_FMT);
2231 }
2232
2233 if (!exporter->timestamp_files) {
2234 g_string_append_printf(namebuf, "-%05u", serial++);
2235 }
2236
2237 return namebuf;
2238 }
2239
2240 /**
2241 * mdOpenTextOutput
2242 *
2243 * open a new text exporter
2244 *
2245 */
mdOpenTextOutput(mdFlowExporter_t * exporter,GError ** err)2246 static gboolean mdOpenTextOutput(
2247 mdFlowExporter_t *exporter,
2248 GError **err)
2249 {
2250 GString *namebuf = NULL;
2251 gboolean rc;
2252
2253 if (exporter->multi_files || (exporter->dedupconfig && !exporter->json)) {
2254 return TRUE;
2255 }
2256
2257 if (exporter->rotate) {
2258 namebuf = mdFileOpenRotater(exporter);
2259 if (exporter->json) {
2260 g_string_append_printf(namebuf, ".json");
2261 } else {
2262 g_string_append_printf(namebuf, ".txt");
2263 }
2264 exporter->current_fname = g_strdup(namebuf->str);
2265 if (exporter->lock) {
2266 mdLockFile(namebuf);
2267 }
2268 rc = mdOpenTextFileExport(exporter, namebuf->str, err);
2269 g_string_free(namebuf, TRUE);
2270 return rc;
2271 }
2272
2273 return mdOpenTextFileExport(exporter, exporter->outspec, err);
2274 }
2275
2276 /**
2277 * mdTextFileRotate
2278 *
2279 * close the current text file, and get a new filename
2280 * for the new one
2281 *
2282 */
mdTextFileRotate(mdFlowExporter_t * exporter,uint64_t cur_time,GError ** err)2283 static gboolean mdTextFileRotate(
2284 mdFlowExporter_t *exporter,
2285 uint64_t cur_time,
2286 GError **err)
2287 {
2288 GString *namebuf;
2289 gboolean rc = FALSE;
2290
2291 if (exporter->multi_files) {
2292 exporter->last_rotate_ms = cur_time;
2293 return TRUE;
2294 }
2295
2296 if (exporter->dedupconfig && !exporter->json) {
2297 return TRUE;
2298 }
2299
2300 if (exporter->last_rotate_ms == 0) {
2301 exporter->last_rotate_ms = cur_time;
2302 return mdOpenTextOutput(exporter, err);
2303 }
2304
2305 exporter->last_rotate_ms = cur_time;
2306
2307 if (exporter->lfp) {
2308 mdCloseAndUnlock(exporter, exporter->lfp, exporter->current_fname,
2309 NULL);
2310 }
2311
2312 namebuf = mdFileOpenRotater(exporter);
2313
2314 if (exporter->json) {
2315 g_string_append_printf(namebuf, ".json");
2316 } else {
2317 g_string_append_printf(namebuf, ".txt");
2318 }
2319
2320 exporter->current_fname = g_strdup(namebuf->str);
2321
2322 if (exporter->lock) {
2323 mdLockFile(namebuf);
2324 }
2325
2326 rc = mdOpenTextFileExport(exporter, namebuf->str, err);
2327
2328 g_string_free(namebuf, TRUE);
2329
2330 return rc;
2331 }
2332
mdVerifyRotatePath(mdFlowExporter_t * exporter,GError ** err)2333 static gboolean mdVerifyRotatePath(
2334 mdFlowExporter_t *exporter,
2335 GError **err)
2336 {
2337 FILE *tmp = NULL;
2338 GString *tmpname = NULL;
2339 /* test that path exists and file can be created */
2340
2341 tmpname = mdFileOpenRotater(exporter);
2342
2343 tmp = fopen(tmpname->str, "w+");
2344
2345 if (tmp == NULL) {
2346 if (errno == 2) {
2347 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2348 "%s: Error opening file %s: No such file or directory",
2349 exporter->name, exporter->outspec);
2350 } else {
2351 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2352 "%s: Can not open file %s for writing",
2353 exporter->name, exporter->outspec);
2354 }
2355 return FALSE;
2356 } else {
2357 /* close and remove empty temp file */
2358 fclose(tmp);
2359 g_remove(tmpname->str);
2360 g_string_free(tmpname, TRUE);
2361 }
2362
2363 return TRUE;
2364 }
2365
2366 /**
2367 * mdFileRotate
2368 *
2369 * rotate IPFIX files, will have ".med" suffix
2370 *
2371 */
mdFileRotate(mdFlowExporter_t * exporter,uint64_t cur_time,GError ** err)2372 static fBuf_t *mdFileRotate(
2373 mdFlowExporter_t *exporter,
2374 uint64_t cur_time,
2375 GError **err)
2376 {
2377 GString *namebuf;
2378 fBuf_t *buf = NULL;
2379
2380 exporter->last_rotate_ms = cur_time;
2381
2382 mdOutputClose(exporter->fbuf, TRUE, err);
2383
2384 if (exporter->lfp) {
2385 mdCloseAndUnlock(exporter, exporter->lfp, exporter->current_fname,
2386 NULL);
2387 }
2388
2389 namebuf = mdFileOpenRotater(exporter);
2390
2391 g_string_append_printf(namebuf, ".med");
2392
2393 exporter->current_fname = g_strdup(namebuf->str);
2394
2395 if (exporter->lock) {
2396 mdLockFile(namebuf);
2397 }
2398
2399 buf = mdOpenFileExport(exporter, namebuf->str, err);
2400
2401 g_string_free(namebuf, TRUE);
2402
2403 return buf;
2404
2405 }
2406
2407 #if HAVE_SPREAD
2408 /**
2409 * mdSpreadExport
2410 *
2411 * open a new Spread Exporter
2412 *
2413 */
mdSpreadExport(mdConfig_t * cfg,mdFlowExporter_t * exporter,GError ** err)2414 static fBuf_t *mdSpreadExport(
2415 mdConfig_t *cfg,
2416 mdFlowExporter_t *exporter,
2417 GError **err)
2418 {
2419
2420 fbSession_t *session;
2421 fBuf_t *fbuf = NULL;
2422
2423 session = fbSessionAlloc(mdInfoModel());
2424 cfg->out_spread.session = session;
2425
2426 cfg->out_spread.daemon = exporter->outspec;
2427
2428 exporter->exporter = fbExporterAllocSpread(&(cfg->out_spread));
2429
2430 if (exporter->exporter == NULL) {
2431 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2432 "Unabled to create Spread Exporter\n");
2433 return NULL;
2434 }
2435
2436 fbuf = fBufAllocForExport(session, exporter->exporter);
2437
2438 if (!(session = mdInitSpreadExporterSession(session,
2439 exporter->dnsdedup, err)))
2440 {
2441 if (fbuf) fBufFree(fbuf);
2442 return NULL;
2443 }
2444
2445 if (!fbSessionExportTemplates(session, err)) {
2446 if (fbuf) fBufFree(fbuf);
2447 return NULL;
2448 }
2449
2450 /* set internal template ? */
2451 if (!fBufSetInternalTemplate(fbuf, YAF_SILK_FLOW_TID, err)){
2452 return NULL;
2453 }
2454
2455 return fbuf;
2456 }
2457 #endif
2458
2459 /**
2460 * mdOpenIntExport
2461 *
2462 * open a TCP/UDP Exporter
2463 *
2464 */
mdOpenIntExport(mdFlowExporter_t * exporter,GError ** err)2465 static fBuf_t *mdOpenIntExport(
2466 mdFlowExporter_t *exporter,
2467 GError **err)
2468 {
2469
2470 fbSession_t *session;
2471 fBuf_t *fbuf = NULL;
2472
2473 if (!(session = exporter->sess_init(NULL, err, exporter->no_stats, exporter->metadata_export))) {
2474 return NULL;
2475 }
2476
2477 exporter->exporter = fbExporterAllocNet(&(exporter->spec));
2478
2479 if (exporter->exporter == NULL) {
2480 return NULL;
2481 }
2482
2483 fbuf = fBufAllocForExport(session, exporter->exporter);
2484
2485 if (!fbSessionExportTemplates(session, err)) {
2486 if (fbuf) fBufFree(fbuf);
2487 return NULL;
2488 }
2489
2490 /* set internal template ? */
2491 /*if (!fBufSetInternalTemplate(fbuf, YAF_SILK_FLOW_TID, err)){
2492 return NULL;
2493 }*/
2494
2495
2496 return fbuf;
2497 }
2498
2499
2500 /**
2501 * mdOutputOpen
2502 *
2503 * configure the new exporter
2504 *
2505 */
mdOutputOpen(mdConfig_t * cfg,mdFlowExporter_t * exporter,GError ** err)2506 static fBuf_t *mdOutputOpen(
2507 mdConfig_t *cfg,
2508 mdFlowExporter_t *exporter,
2509 GError **err)
2510 {
2511
2512 #if HAVE_SPREAD
2513 if (exporter->type == SPREAD) {
2514 return mdSpreadExport(cfg, exporter, err);
2515 }
2516 #endif
2517 if (exporter->type == TCP || exporter->type == UDP) {
2518 return mdOpenIntExport(exporter, err);
2519 }
2520
2521 if (exporter->rotate) {
2522 return mdFileRotate(exporter, cfg->ctime, err);
2523 }
2524
2525 return mdOpenFileExport(exporter, exporter->outspec, err);
2526
2527 }
2528
2529 /**
2530 * mdExporterWriteOptions
2531 *
2532 * write an IPFIX Options Record
2533 *
2534 * @param cfg - mediator configuration options
2535 * @param exporter - exporter to write to
2536 * @param tid - template id
2537 * @param rec - the options record to write
2538 * @param rec_length - length of record to write
2539 * @param err
2540 * @return TRUE if no errors
2541 */
mdExporterWriteOptions(mdConfig_t * cfg,mdFlowExporter_t * exporter,uint8_t * rec,size_t rec_length,uint16_t tid,GError ** err)2542 gboolean mdExporterWriteOptions(
2543 mdConfig_t *cfg,
2544 mdFlowExporter_t *exporter,
2545 uint8_t *rec,
2546 size_t rec_length,
2547 uint16_t tid,
2548 GError **err)
2549 {
2550 size_t bytes;
2551
2552 if (exporter->no_stats == 1) {
2553 return TRUE;
2554 }
2555
2556 if (!exporter->active) {
2557 if (cfg->ctime - exporter->last_restart_ms > MD_RESTART_MS) {
2558 if (!mdExporterRestart(cfg, exporter, err)) {
2559 g_message("Error restarting exporter %s: %s",
2560 exporter->name, (*err)->message);
2561 g_clear_error(err);
2562 return TRUE;
2563 }
2564 } else {
2565 return TRUE;
2566 }
2567 }
2568
2569 if (exporter->fbuf) {
2570 if (!fBufSetInternalTemplate(exporter->fbuf, tid, err))
2571 {
2572 return FALSE;
2573 }
2574
2575 if (exporter->type == SPREAD) {
2576 #if HAVE_SPREAD
2577 fBufSetSpreadExportGroup(exporter->fbuf, cfg->out_spread.groups,
2578 num_out_groups, err);
2579 if (!mdSetSpreadExportTemplate(exporter->fbuf, &(cfg->out_spread),
2580 tid, cfg->out_spread.groups,
2581 num_out_groups, err))
2582 {
2583 goto err;
2584 }
2585 #endif
2586 } else {
2587 if (!mdSetExportTemplate(exporter->fbuf, tid, err))
2588 {
2589 goto err;
2590 }
2591 }
2592
2593 if (!(fBufAppend(exporter->fbuf, (uint8_t *)rec, rec_length, err)))
2594 {
2595 fBufFree(exporter->fbuf);
2596 exporter->fbuf = NULL;
2597 goto err;
2598 }
2599
2600 } else {
2601
2602 if (exporter->rotate) {
2603 if ((cfg->ctime - exporter->last_rotate_ms) > exporter->rotate) {
2604 if (!mdTextFileRotate(exporter, cfg->ctime, err)) {
2605 exporter->last_rotate_ms = 0;
2606 goto err;
2607 }
2608 }
2609 }
2610
2611 if(tid == YAF_STAT_OPTN_FLOW_TID){ /* TODO: add support for Tombstone */
2612 if (exporter->json) {
2613 bytes = mdPrintJsonStats((yaf_stats_option_t *)rec,
2614 cfg->collector_name, exporter->lfp, err);
2615 } else {
2616 bytes = mdPrintStats((yaf_stats_option_t *)rec, cfg->collector_name,
2617 exporter->lfp, exporter->delimiter, exporter->no_stats,
2618 err);
2619 }
2620 if (!bytes) {
2621 goto err;
2622 }
2623
2624 exporter->exp_bytes += bytes;
2625 }
2626 }
2627
2628 /* update exporter stats */
2629 ++(exporter->exp_stats);
2630 exporter->exp_bytes += rec_length;
2631
2632 return TRUE;
2633
2634 err:
2635 g_warning("Error writing option reord: %s", (*err)->message);
2636 g_clear_error(err);
2637 g_warning("Deactivating Exporter %s.", exporter->name);
2638 exporter->active = FALSE;
2639 if (!mdExporterRestart(cfg, exporter, err)) {
2640 g_warning("Error restarting exporter %s: %s",
2641 exporter->name, (*err)->message);
2642 g_clear_error(err);
2643 }
2644 return TRUE;
2645 }
2646
2647
mdExporterfBufSetup(mdConfig_t * cfg,mdFlowExporter_t * exporter,mdFullFlow_t * flow,GError ** err,md_sess_init_fn sess_init,uint16_t int_tid,uint16_t ext_tid)2648 static gboolean mdExporterfBufSetup(
2649 mdConfig_t *cfg,
2650 mdFlowExporter_t *exporter,
2651 mdFullFlow_t *flow,
2652 GError **err,
2653 md_sess_init_fn sess_init,
2654 uint16_t int_tid,
2655 uint16_t ext_tid)
2656 {
2657
2658 #if HAVE_SPREAD
2659 char *groups[10];
2660 int num_groups;
2661 #endif
2662
2663 if (exporter->rotate) {
2664 if (exporter->last_rotate_ms &&
2665 (cfg->ctime - exporter->last_rotate_ms) > exporter->rotate)
2666 {
2667 exporter->fbuf = (fBuf_t *)mdFileRotate(exporter, cfg->ctime,
2668 err);
2669 } else if (exporter->last_rotate_ms == 0) {
2670 exporter->last_rotate_ms = cfg->ctime;
2671 }
2672 }
2673
2674 if (exporter->type == SPREAD) {
2675 #if HAVE_SPREAD
2676 if (cfg->mdspread) {
2677 if ((num_groups = mdSpreadExporterFilter(cfg->mdspread,
2678 flow, groups)))
2679 {
2680 fBufSetSpreadExportGroup(exporter->fbuf, groups, num_groups,
2681 err);
2682 } else {
2683 return FALSE;
2684 }
2685 }
2686 #endif
2687
2688 } else if (exporter->type == UDP) {
2689
2690 if ((cfg->ctime - exporter->lastUdpTempTime) >
2691 ((cfg->udp_template_timeout)/3))
2692 {
2693 if (!fbSessionExportTemplates(fBufGetSession(exporter->fbuf),
2694 err))
2695 {
2696 g_warning("Failed to renew UDP Templates: %s",
2697 (*err)->message);
2698 g_clear_error(err);
2699 }
2700 exporter->lastUdpTempTime = cfg->ctime;
2701 }
2702 }
2703
2704 /* set internal template */
2705 if (!fBufSetInternalTemplate(exporter->fbuf, int_tid, err)) {
2706
2707 /* if template doesn't exist, then use the sess_init function
2708 to create templates and add them to the session */
2709 if (!g_error_matches(*err, FB_ERROR_DOMAIN, FB_ERROR_TMPL)) {
2710 return FALSE;
2711 }
2712
2713 g_clear_error(err);
2714
2715 if (!sess_init(fBufGetSession(exporter->fbuf), err,exporter->no_stats, exporter->metadata_export))
2716 {
2717 return FALSE;
2718 }
2719
2720 if (!fBufSetInternalTemplate(exporter->fbuf, int_tid, err)) {
2721 return FALSE;
2722 }
2723 }
2724
2725 if (exporter->type == SPREAD) {
2726 #if HAVE_SPREAD
2727 if (!mdSetSpreadExportTemplate(exporter->fbuf, &(cfg->out_spread),
2728 ext_tid, groups, num_groups, err))
2729 {
2730 return FALSE;
2731 }
2732 #endif
2733 } else {
2734 if (!mdSetExportTemplate(exporter->fbuf, ext_tid, err)) {
2735 return FALSE;
2736 }
2737 }
2738
2739 return TRUE;
2740
2741 }
2742
2743 /**
2744 * mdExporterWriteFlow
2745 *
2746 * write a mediator flow record
2747 *
2748 * @param cfg - mediator configuration options
2749 * @param exporter - exporter to write to
2750 * @param flow - a full mediator flow
2751 * @param err
2752 * @return TRUE if no errors were encountered
2753 */
mdExporterWriteFlow(mdConfig_t * cfg,mdFlowExporter_t * exporter,mdFullFlow_t * flow,GError ** err)2754 int mdExporterWriteFlow(
2755 mdConfig_t *cfg,
2756 mdFlowExporter_t *exporter,
2757 mdFullFlow_t *flow,
2758 GError **err)
2759 {
2760
2761 gboolean rc;
2762 uint16_t tid;
2763 int ret;
2764 char *indexstr = NULL;
2765 size_t indexlen = 0;
2766
2767 if (exporter->fbuf && exporter->dns_rr_only) {
2768 /* dns rr only? */
2769 if ((flow->rec->silkAppLabel == 53) && flow->app) {
2770 if (!mdExportDNSRR(cfg, exporter, flow, flow->tid, err)) {
2771 return -1;
2772 }
2773 }
2774 }
2775
2776 if (exporter->dnsdeduponly || (exporter->no_stats == 2) ||
2777 exporter->ssldeduponly || exporter->deduponly || exporter->no_flow)
2778 {
2779 return 0;
2780 }
2781
2782 tid = mdConvertToSiLK(flow->rec, flow->tid);
2783
2784 if (!exporter->flowonly) {
2785 /* If FLOW_ONLY isn't present, include STML */
2786 tid |= YTF_LIST;
2787 /* check to see if TCP is in main record, if not-keep it in list */
2788 if ((flow->tid & 0x0020) == 0) {
2789 /* don't include tcp in reg template - use stml */
2790 tid &= 0xFFDF;
2791 }
2792
2793 /* Since we use Template PAIRS for SSL - we need to change the template
2794 ID from our internal template ID back to the external template ID
2795 that is used to define SSL Entries */
2796 if (flow->app_tid == SM_INTSSL_FLOW_TID) {
2797 yaf_newssl_t *sslflow = (yaf_newssl_t *)flow->app;
2798 fbSubTemplateList_t *stl = &(sslflow->sslCertList);
2799 stl->tmplID = YAF_NEW_SSL_CERT_TID;
2800 flow->cert->tmplID = YAF_NEW_SSL_FLOW_TID;
2801 }
2802
2803 } else {
2804 if (flow->rec->flowEndReason == UDP_FORCE) {
2805 /* ignore dns records */
2806 return 0;
2807 }
2808 }
2809
2810 if (!exporter->active) {
2811 if (cfg->ctime - exporter->last_restart_ms > MD_RESTART_MS) {
2812 if (!mdExporterRestart(cfg, exporter, err)) {
2813 g_message("Error restarting exporter %s: %s",
2814 exporter->name, (*err)->message);
2815 g_clear_error(err);
2816 return 0;
2817 }
2818 } else {
2819 return 0;
2820 }
2821 }
2822
2823 if (exporter->fbuf) {
2824
2825 if (exporter->dpionly) {
2826 if (flow->app_tid == 0) {
2827 return 0;
2828 }
2829 }
2830
2831 if (!mdExporterfBufSetup(cfg, exporter, flow, err,
2832 mdInitExporterSession, YAF_SILK_FLOW_TID, tid))
2833 {
2834 goto err;
2835 }
2836
2837 if (cfg->usec_sleep) {
2838 usleep(cfg->usec_sleep);
2839 }
2840
2841 exporter->exp_bytes += sizeof(md_main_template_t);
2842
2843 rc = fBufAppend(exporter->fbuf, (uint8_t *)flow->rec,
2844 sizeof(md_main_template_t), err);
2845
2846 if (!rc) {
2847 fBufFree(exporter->fbuf);
2848 goto err;
2849 }
2850
2851 } else if (exporter->type == TEXT) {
2852
2853 if (exporter->rotate) {
2854 if ((cfg->ctime - exporter->last_rotate_ms) > exporter->rotate) {
2855 if (!mdTextFileRotate(exporter, cfg->ctime, err)) {
2856 exporter->last_rotate_ms = 0;
2857 goto err;
2858 }
2859 }
2860 }
2861
2862 if (exporter->custom_list) {
2863 ret = mdCustomFlowPrint(exporter->custom_list, flow, exporter,
2864 err);
2865 if (ret == 1) {
2866 return 0;
2867 }
2868 } else {
2869 /* if it's not custom - it's multi-files OR DPI-only */
2870 if (mdExporterDPIGetIndexStr(exporter, flow)) {
2871 indexlen = MD_MSG_LEN(exporter->buf);
2872 indexstr = malloc(indexlen);
2873 memcpy(indexstr, exporter->buf->buf, indexlen);
2874 /* reset buffer */
2875 exporter->buf->cp = exporter->buf->buf;
2876 ret = mdExporterDPIFlowPrint(exporter, flow,
2877 indexstr, indexlen, err);
2878 if (ret > 0) free(indexstr);
2879 /* didn't actually add anything */
2880 if (ret == 1) return 0;
2881 } else {
2882 /* NO DPI DATA - continue */
2883 return 0;
2884 }
2885 }
2886
2887 if (ret < 0) {
2888 goto err;
2889 } else if (ret == 0) {
2890 /* realloc bigger buffer and try again */
2891 if (!mdExporterExpandBuf(exporter)) {
2892 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
2893 "Error allocating memory for exporter %s",
2894 exporter->name);
2895 return -1;
2896 }
2897 if (exporter->custom_list) {
2898 ret = mdCustomFlowPrint(exporter->custom_list, flow, exporter,
2899 err);
2900 } else if (indexstr) {
2901 ret = mdExporterDPIFlowPrint(exporter, flow,
2902 indexstr, indexlen, err);
2903 if (ret > 0) free(indexstr);
2904 }
2905 if (ret < 0) {
2906 goto err;
2907 } else if (ret == 0) {
2908 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2909 "Error writing to buffer for exporter %s",
2910 exporter->name);
2911 goto err;
2912 }
2913 }
2914
2915 } /* TEXT type exporter */
2916
2917 ++(exporter->exp_flows);
2918
2919 return 1;
2920
2921 err:
2922 g_warning("Error writing flow: %s", (*err)->message);
2923 g_clear_error(err);
2924 g_warning("Deactivating Exporter %s.", exporter->name);
2925 exporter->active = FALSE;
2926 if (!mdExporterRestart(cfg, exporter, err)) {
2927 g_warning("Error restarting exporter %s: %s",
2928 exporter->name, (*err)->message);
2929 g_clear_error(err);
2930 }
2931 return 1;
2932 }
2933
2934 /**
2935 * mdExporterWriteRecord
2936 *
2937 * write a DNS de-duplicated record to the given exporter
2938 *
2939 * @param cfg - mediator configuration options
2940 * @param exporter - exporter to write to
2941 * @param tid - template id
2942 * @param rec - the record to write
2943 * @param rec_length - length of record to write
2944 * @param err
2945 * @return TRUE if no errors
2946 */
mdExporterWriteRecord(mdConfig_t * cfg,mdFlowExporter_t * exporter,uint16_t tid,uint8_t * rec,size_t rec_length,GError ** err)2947 gboolean mdExporterWriteRecord(
2948 mdConfig_t *cfg,
2949 mdFlowExporter_t *exporter,
2950 uint16_t tid,
2951 uint8_t *rec,
2952 size_t rec_length,
2953 GError **err)
2954 {
2955 int ret;
2956 gboolean print_last_seen = FALSE;
2957
2958 if (exporter == NULL) {
2959 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
2960 "Exporter Node Exists, but No Type\n");
2961 return FALSE;
2962 }
2963
2964 if (!exporter->dnsdedup) {
2965 return TRUE;
2966 }
2967
2968 if (!exporter->active) {
2969 if (cfg->ctime - exporter->last_restart_ms > MD_RESTART_MS) {
2970 if (!mdExporterRestart(cfg, exporter, err)) {
2971 g_message("Error restarting exporter %s: %s",
2972 exporter->name, (*err)->message);
2973 g_clear_error(err);
2974 return TRUE;
2975 }
2976 } else {
2977 return TRUE;
2978 }
2979
2980 }
2981
2982 if (exporter->fbuf) {
2983 /* remove null char at the end of dnsrrname? */
2984
2985 if (!mdExporterfBufSetup(cfg, exporter, NULL, err,
2986 mdInitExporterSessionDNSDedupOnly,
2987 MD_DNS_FULL, tid))
2988 {
2989 return FALSE;
2990 }
2991
2992 if (!fBufAppend(exporter->fbuf, (uint8_t *)rec, rec_length, err)) {
2993 fBufFree(exporter->fbuf);
2994 goto err;
2995 }
2996 /* update stats */
2997 exporter->exp_bytes += rec_length;
2998
2999 }
3000
3001 if (exporter->type == TEXT) {
3002 if (tid & MD_LAST_SEEN) {
3003 print_last_seen = TRUE;
3004 }
3005
3006 if (exporter->rotate) {
3007 if ((cfg->ctime - exporter->last_rotate_ms) > exporter->rotate) {
3008 if (!mdTextFileRotate(exporter, cfg->ctime, err)) {
3009 exporter->last_rotate_ms = 0;
3010 goto err;
3011 }
3012 }
3013 }
3014
3015 if (exporter->json) {
3016 ret = mdJsonifyDNSDedupRecord(exporter->lfp, exporter->buf, rec,
3017 print_last_seen,
3018 cfg->dns_base64_encode, err);
3019 } else {
3020 ret = mdPrintDNSRecord(exporter->lfp, exporter->buf,
3021 exporter->delimiter, rec,
3022 cfg->dns_base64_encode, print_last_seen,
3023 exporter->escape_chars, err);
3024 }
3025
3026 if (ret < 0) {
3027 goto err;
3028 } else if (ret == 0) {
3029 /* realloc bigger buffer and try again */
3030 if (!mdExporterExpandBuf(exporter)) {
3031 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
3032 "Error allocating memory for exporter %s",
3033 exporter->name);
3034 return FALSE;
3035 }
3036 if (exporter->json) {
3037 ret = mdJsonifyDNSDedupRecord(exporter->lfp, exporter->buf,
3038 rec, print_last_seen,
3039 cfg->dns_base64_encode, err);
3040 } else {
3041 ret = mdPrintDNSRecord(exporter->lfp, exporter->buf,
3042 exporter->delimiter, rec,
3043 cfg->dns_base64_encode, print_last_seen,
3044 exporter->escape_chars, err);
3045 }
3046
3047 if (ret < 0) {
3048 goto err;
3049 } else if (ret == 0) {
3050 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
3051 "Error writing to buffer for exporter %s",
3052 exporter->name);
3053 goto err;
3054 }
3055 }
3056
3057 exporter->exp_bytes += ret;
3058 }
3059
3060 ++(exporter->exp_flows);
3061
3062 return TRUE;
3063
3064 err:
3065
3066 g_warning("Error writing DNS Record: %s", (*err)->message);
3067 g_clear_error(err);
3068 g_warning("Deactivating Exporter %s.", exporter->name);
3069 exporter->active = FALSE;
3070 if (!mdExporterRestart(cfg, exporter, err)) {
3071 g_warning("Error restarting exporter %s: %s",
3072 exporter->name, (*err)->message);
3073 g_clear_error(err);
3074 }
3075
3076 return TRUE;
3077 }
3078
3079 /**
3080 * mdExportersInit
3081 *
3082 * cycle through exporters and open their output methods
3083 *
3084 */
mdExportersInit(mdConfig_t * cfg,md_export_node_t * node,GError ** err)3085 gboolean mdExportersInit(
3086 mdConfig_t *cfg,
3087 md_export_node_t *node,
3088 GError **err)
3089 {
3090 md_export_node_t *cnode = NULL;
3091
3092 for (cnode = node; cnode; cnode = cnode->next) {
3093
3094 if (cnode->exp == NULL) {
3095 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_SETUP,
3096 "Error: No Exporter Defined.\n");
3097 return FALSE;
3098 }
3099
3100 if (cnode->exp->type == TEXT) {
3101 if (!cnode->exp->rotate) {
3102 if (!mdOpenTextOutput(cnode->exp, err)) {
3103 return FALSE;
3104 }
3105 } else {
3106 if (!mdVerifyRotatePath(cnode->exp, err)) {
3107 return FALSE;
3108 }
3109 }
3110 } else {
3111 cnode->exp->fbuf = mdOutputOpen(cfg, cnode->exp, err);
3112 if (cnode->exp->fbuf == NULL) {
3113 g_warning("Error connecting to exporter: %s", (*err)->message);
3114 g_clear_error(err);
3115 cnode->exp->active = FALSE;
3116 continue;
3117 /*return FALSE;*/
3118 }
3119
3120 if (cnode->dedup) {
3121 if (!md_dedup_add_templates(cnode->dedup, cnode->exp->fbuf,
3122 err))
3123 {
3124 g_warning("Error adding dedup templates: %s",
3125 (*err)->message);
3126 cnode->exp->active = FALSE;
3127 continue;
3128 }
3129 }
3130
3131 /* just try to emit, there will be an error if not connected */
3132 if (!fBufEmit(cnode->exp->fbuf, err)) {
3133 if (cnode->exp->fbuf) fBufFree(cnode->exp->fbuf);
3134 g_warning("Error connecting to exporter: %s", (*err)->message);
3135 g_clear_error(err);
3136 cnode->exp->active = FALSE;
3137 continue;
3138 /*return FALSE;*/
3139 }
3140 }
3141
3142 g_message("%s: Exporter Active.", cnode->exp->name);
3143 cnode->exp->active = TRUE;
3144 }
3145
3146 return TRUE;
3147 }
3148
mdExporterRestart(mdConfig_t * cfg,mdFlowExporter_t * exporter,GError ** err)3149 gboolean mdExporterRestart(
3150 mdConfig_t *cfg,
3151 mdFlowExporter_t *exporter,
3152 GError **err)
3153 {
3154 exporter->last_restart_ms = cfg->ctime;
3155
3156 if (exporter == NULL) {
3157 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_SETUP,
3158 "Error: No Exporter Defined.\n");
3159 return FALSE;
3160 }
3161
3162 if (exporter->type == TEXT) {
3163 if (!exporter->rotate) {
3164 if (!mdOpenTextOutput(exporter, err)) {
3165 return FALSE;
3166 }
3167 } else {
3168 if ((cfg->ctime - exporter->last_rotate_ms) > exporter->rotate) {
3169 if (!mdTextFileRotate(exporter, cfg->ctime, err)) {
3170 exporter->last_rotate_ms = 0;
3171 return FALSE;
3172 }
3173 }
3174 }
3175
3176 } else {
3177 exporter->fbuf = mdOutputOpen(cfg, exporter, err);
3178 if (exporter->fbuf == NULL) {
3179 return FALSE;
3180 }
3181 if (!fBufEmit(exporter->fbuf, err)) {
3182 if (exporter->fbuf) fBufFree(exporter->fbuf);
3183 return FALSE;
3184 }
3185 }
3186
3187 g_debug("%s: Total Flows Exported Before Restart: %"PRIu64,
3188 exporter->name, exporter->exp_flows);
3189 /*if (exporter->exp_dns) {
3190 g_debug("%s: Total DNS Records Exported Before Restart: %llu",
3191 exporter->name, exporter->exp_dns);
3192 }*/
3193 g_message("%s: Exporter successfully restarted. Now active.",
3194 exporter->name);
3195 /* reset counters */
3196 exporter->exp_flows = 0;
3197 exporter->exp_stats = 0;
3198 exporter->exp_bytes = 0;
3199 exporter->active = TRUE;
3200 exporter->last_restart_ms = 0;
3201 /* note that exporter was restarted */
3202 exporter->time_started = g_timer_elapsed(mdStatGetTimer(), NULL);
3203 return TRUE;
3204
3205 }
3206
mdExporterUpdateStats(mdConfig_t * cfg,gboolean dedup)3207 void mdExporterUpdateStats(
3208 mdConfig_t *cfg,
3209 gboolean dedup)
3210 {
3211 md_export_node_t *cnode = NULL;
3212 uint64_t seconds = g_timer_elapsed(mdStatGetTimer(), NULL);
3213
3214 if (!seconds) seconds = 1;
3215
3216 for (cnode = cfg->flowexit; cnode; cnode = cnode->next) {
3217
3218 if (cnode->dedup) {
3219 if (dedup) {
3220 md_dedup_print_stats(cnode->dedup, cnode->exp->name);
3221 }
3222 continue;
3223 }
3224
3225 if (cnode->exp->exp_flows) {
3226 g_message("Exporter %s: %"PRIu64" records, %"PRIu64" stats, "
3227 "%.4f Mbps, %.2f bytes per record",
3228 cnode->exp->name, cnode->exp->exp_flows,
3229 cnode->exp->exp_stats,
3230 ((((double)cnode->exp->exp_bytes * 8.0) / 1000000) /
3231 seconds),
3232 ((double)cnode->exp->exp_bytes / cnode->exp->exp_flows));
3233 } else {
3234 g_message("Exporter %s: %"PRIu64" records, %"PRIu64" stats",
3235 cnode->exp->name, cnode->exp->exp_flows,
3236 cnode->exp->exp_stats);
3237 }
3238
3239 if (cnode->dns_dedup && dedup) {
3240 md_dns_dedup_print_stats(cnode->dns_dedup, cnode->exp->name);
3241 }
3242
3243 }
3244 }
3245
3246 /**
3247 * mdExporterDestroy
3248 *
3249 * loop through exporter list and remove the exporters
3250 * flush the DNS close queue, and destroy tables
3251 *
3252 */
mdExporterDestroy(mdConfig_t * cfg,GError ** err)3253 gboolean mdExporterDestroy(
3254 mdConfig_t *cfg,
3255 GError **err)
3256 {
3257 md_export_node_t *cnode = NULL;
3258 int loop;
3259
3260 mdExporterUpdateStats(cfg, FALSE);
3261
3262 for (cnode = cfg->flowexit; cnode; cnode = cnode->next) {
3263 detachHeadOfSLL((mdSLL_t **)&(cfg->flowexit), (mdSLL_t **)&cnode);
3264
3265 if (cnode->dns_dedup) {
3266 md_dns_flush_all_tab(cnode->dns_dedup, cfg->ctime, TRUE);
3267 if (!md_dns_flush_queue(cnode, cfg, err)) {
3268 return FALSE;
3269 }
3270 /* print final stats */
3271 md_dns_dedup_print_stats(cnode->dns_dedup, cnode->exp->name);
3272 if (!md_dns_dedup_free_state(cfg, cnode, err)) {
3273 return FALSE;
3274 }
3275 }
3276
3277 if (cnode->dedup) {
3278 md_dedup_flush_alltab(cnode, cfg->ctime, TRUE);
3279 if (!md_dedup_flush_queue(cnode, cfg, err)) {
3280 return FALSE;
3281 }
3282 md_dedup_print_stats(cnode->dedup, cnode->exp->name);
3283 if (!md_dedup_free_state(cfg, cnode, err)) {
3284 return FALSE;
3285 }
3286 if (cnode->exp->type == TEXT && !cnode->exp->json) {
3287 /* otherwise it will be freed below */
3288 g_free(cnode->exp->outspec);
3289 }
3290 }
3291
3292 if (cnode->ssl_dedup) {
3293 md_ssl_flush_tab(cnode->ssl_dedup, cfg->ctime, TRUE);
3294 if (!md_ssl_flush_queue(cnode, cfg, err)) {
3295 return FALSE;
3296 }
3297 md_ssl_dedup_print_stats(cnode->ssl_dedup, cnode->exp->name);
3298 if (!md_ssl_dedup_free_state(cfg, cnode, err)) {
3299 return FALSE;
3300 }
3301 }
3302
3303 if (cnode->exp->fbuf && cnode->exp->active) {
3304 if (!mdOutputClose(cnode->exp->fbuf, TRUE, err)) {
3305 return FALSE;
3306 }
3307 }
3308
3309 if (cnode->exp->spec.host) {
3310 g_free(cnode->exp->spec.host);
3311 }
3312
3313 if (cnode->exp->spec.svc) {
3314 g_free(cnode->exp->spec.svc);
3315 }
3316
3317 if (cnode->exp->ssl_config) {
3318 if (cnode->exp->ssl_config->issuer) {
3319 g_free(cnode->exp->ssl_config->issuer);
3320 }
3321 if (cnode->exp->ssl_config->subject) {
3322 g_free(cnode->exp->ssl_config->subject);
3323 }
3324 if (cnode->exp->ssl_config->other) {
3325 g_free(cnode->exp->ssl_config->other);
3326 }
3327 if (cnode->exp->ssl_config->extensions) {
3328 g_free(cnode->exp->ssl_config->extensions);
3329 }
3330 g_slice_free1(sizeof(cnode->exp->ssl_config),
3331 cnode->exp->ssl_config);
3332 }
3333
3334 if (cnode->exp->multi_files) {
3335 for (loop = 0; loop < num_tables; loop++) {
3336 if (table_info[loop]->table_file) {
3337 mdCloseAndUnlock(cnode->exp, table_info[loop]->table_file,
3338 table_info[loop]->file_name,
3339 table_info[loop]->table_name);
3340 /*fclose(table_info[loop]->table_file);
3341 if (cnode->exp->lock) {
3342 mdUnlockFile(table_info[loop]->file_name);
3343 }
3344 if (cnode->exp->mysql) {
3345 mdLoadFile(cnode->exp, table_info[loop]->table_name,
3346 table_info[loop]->file_name);
3347 }
3348 g_free(table_info[loop]->file_name);*/
3349 }
3350 g_free(table_info[loop]->table_name);
3351 g_slice_free(mdTableInfo_t, table_info[loop]);
3352 }
3353
3354 if (table_info) {
3355 g_free(table_info);
3356 g_hash_table_destroy(table_hash);
3357 num_tables = 0;
3358 }
3359
3360 g_free(cnode->exp->outspec);
3361
3362 } else if (cnode->exp->lfp) {
3363 if (cnode->exp->current_fname) {
3364 mdCloseAndUnlock(cnode->exp, cnode->exp->lfp,
3365 cnode->exp->current_fname, NULL);
3366 if (cnode->exp->outspec) {
3367 g_free(cnode->exp->outspec);
3368 }
3369 } else {
3370 mdCloseAndUnlock(cnode->exp, cnode->exp->lfp,
3371 cnode->exp->outspec, NULL);
3372 }
3373 }
3374
3375 if (cnode->exp->custom_list) {
3376 mdFieldList_t *list = cnode->exp->custom_list;
3377 mdFieldList_t *list2 = NULL;
3378 while (list) {
3379 detachHeadOfSLL((mdSLL_t **)&(cnode->exp->custom_list),
3380 (mdSLL_t **)&list);
3381 list2 = list->next;
3382 g_string_free(list->decorator, TRUE);
3383 g_slice_free(mdFieldList_t, list);
3384 list = list2;
3385 }
3386 }
3387
3388 if (cnode->exp->mysql) {
3389 g_free(cnode->exp->mysql->user);
3390 g_free(cnode->exp->mysql->password);
3391 g_free(cnode->exp->mysql->db_name);
3392 g_free(cnode->exp->mysql->db_host);
3393 g_free(cnode->exp->mysql->table);
3394 #if HAVE_MYSQL
3395 if (cnode->exp->mysql->conn) {
3396 mysql_close(cnode->exp->mysql->conn);
3397 }
3398 #endif
3399 g_slice_free(mdMySQLInfo_t, cnode->exp->mysql);
3400 }
3401
3402 /* free exporter name */
3403 g_free(cnode->exp->name);
3404
3405 if (cnode->exp->type == TEXT) {
3406 g_slice_free1(cnode->exp->buf->buflen, cnode->exp->buf->buf);
3407 g_slice_free(mdBuf_t, cnode->exp->buf);
3408 }
3409
3410 mdExporterFree(cnode->exp);
3411
3412 if (cnode->filter) {
3413 md_filter_t *cfil = cnode->filter;
3414 md_filter_t *nfil = NULL;
3415
3416 while (cfil) {
3417 detachHeadOfSLL((mdSLL_t **)&(cnode->filter),
3418 (mdSLL_t **)&cfil);
3419 nfil = cfil->next;
3420 #if ENABLE_SKIPSET
3421 if (cfil->ipset) {
3422 skIPSetDestroy(&(cfil->ipset));
3423 }
3424 #endif
3425 g_slice_free(md_filter_t, cfil);
3426 cfil = nfil;
3427 }
3428 #if ENABLE_SKIPSET
3429 skAppUnregister();
3430 #endif
3431 }
3432 /*g_slice_free(md_export_node_t, cnode);*/
3433
3434 }
3435
3436 return TRUE;
3437 }
3438
3439
3440 /**
3441 * mdExporterConnectionReset
3442 *
3443 * when a connection is reset via TCP, flush the DNS tables
3444 * and buffer so we don't hang on to records too long.
3445 * this also gets called every 5 minutes if we're not receiving
3446 * anything
3447 *
3448 */
mdExporterConnectionReset(mdConfig_t * cfg,GError ** err)3449 gboolean mdExporterConnectionReset(
3450 mdConfig_t *cfg,
3451 GError **err)
3452 {
3453
3454 md_export_node_t *cnode = NULL;
3455
3456 for (cnode = cfg->flowexit; cnode; cnode = cnode->next) {
3457
3458 if (!cnode->exp->active) {
3459 continue;
3460 }
3461
3462 if (cnode->dns_dedup) {
3463 md_dns_flush_all_tab(cnode->dns_dedup, cfg->ctime, FALSE);
3464 if (!md_dns_flush_queue(cnode, cfg, err)) {
3465 return FALSE;
3466 }
3467 }
3468
3469 if (cnode->dedup) {
3470 md_dedup_flush_alltab(cnode, cfg->ctime, FALSE);
3471 if (!md_dedup_flush_queue(cnode, cfg, err)) {
3472 return FALSE;
3473 }
3474 }
3475
3476 if (cnode->ssl_dedup) {
3477 md_ssl_flush_tab(cnode->ssl_dedup, cfg->ctime, FALSE);
3478 if (!md_ssl_flush_queue(cnode, cfg, err)) {
3479 return FALSE;
3480 }
3481 }
3482
3483 if (cnode->exp->fbuf) {
3484 if (!fBufEmit(cnode->exp->fbuf, err)) {
3485 g_warning("Error emitting buffer: %s", (*err)->message);
3486 g_warning("Deactivating Exporter %s.", cnode->exp->name);
3487 cnode->exp->active = FALSE;
3488 g_clear_error(err);
3489 }
3490
3491 if (cnode->exp->rotate) {
3492 if (cnode->exp->last_rotate_ms == 0) {
3493 cnode->exp->last_rotate_ms = cfg->ctime;
3494 } else if ((cfg->ctime - cnode->exp->last_rotate_ms) >
3495 cnode->exp->rotate)
3496 {
3497 cnode->exp->fbuf = (fBuf_t *)mdFileRotate(cnode->exp,
3498 cfg->ctime,
3499 err);
3500 }
3501 }
3502 }
3503
3504 if (cnode->exp->lfp) {
3505 fflush(cnode->exp->lfp);
3506 }
3507
3508 if (cnode->exp->type == TEXT) {
3509 if (cnode->exp->rotate) {
3510 if ((cfg->ctime - cnode->exp->last_rotate_ms) >
3511 cnode->exp->rotate)
3512 {
3513 if (!mdTextFileRotate(cnode->exp, cfg->ctime, err)) {
3514 cnode->exp->last_rotate_ms = 0;
3515 g_warning("Error rotating file: %s",(*err)->message);
3516 g_warning("Deactivating Exporter %s.",
3517 cnode->exp->name);
3518 cnode->exp->active = FALSE;
3519 g_clear_error(err);
3520 }
3521 }
3522 }
3523 }
3524 }
3525
3526 return TRUE;
3527
3528 }
3529
3530 /**
3531 * mdAppendDPIStrMultiFiles
3532 *
3533 */
mdAppendDPIStrMultiFiles(mdFlowExporter_t * exporter,uint8_t * buf,char * label,char * index_str,size_t index_len,uint16_t id,size_t buflen,gboolean hex)3534 gboolean mdAppendDPIStrMultiFiles(
3535 mdFlowExporter_t *exporter,
3536 uint8_t *buf,
3537 char *label,
3538 char *index_str,
3539 size_t index_len,
3540 uint16_t id,
3541 size_t buflen,
3542 gboolean hex)
3543 {
3544
3545 char delim = exporter->dpi_delimiter;
3546 mdBuf_t *mdbuf = exporter->buf;
3547 size_t brem = MD_REM_MSG(mdbuf);
3548 int ret;
3549 FILE *fp;
3550 size_t rc;
3551 GError *err;
3552
3553 if (buflen == 0) {
3554 return TRUE;
3555 }
3556
3557 if (table_hash) {
3558 label = mdGetTableItem(id);
3559 if (label == NULL) {
3560 return TRUE;
3561 }
3562 }
3563
3564 if (!md_util_append_buffer(mdbuf, &brem, (uint8_t*)index_str, index_len)) {
3565 return FALSE;
3566 }
3567
3568 ret = snprintf(mdbuf->cp, brem, "%d%c", id, delim);
3569 MD_CHECK_RET(mdbuf, ret, brem);
3570
3571 if (!mdPrintVariableLength(mdbuf, &brem, buf, buflen,
3572 delim, hex, exporter->escape_chars)) {
3573 return FALSE;
3574 }
3575
3576
3577 MD_APPEND_CHAR_CHECK(brem, mdbuf, '\n');
3578
3579 fp = mdGetTableFile(exporter, id);
3580
3581 if (fp == NULL) {
3582 g_warning("Error: File does not exist for id %d", id);
3583 return FALSE;
3584 }
3585
3586 rc = md_util_write_buffer(fp, mdbuf, exporter->name, &err);
3587
3588 if (!rc) {
3589 g_warning("Error writing file for id %d: %s",
3590 id, err->message);
3591 g_clear_error(&err);
3592 return FALSE;
3593 }
3594
3595 exporter->exp_bytes += rc;
3596
3597 return TRUE;
3598 }
3599
3600
3601 /**
3602 * mdAppendDPIStr
3603 *
3604 * append the given string and label to the given GString
3605 *
3606 */
mdAppendDPIStr(mdFlowExporter_t * exporter,uint8_t * buf,char * label,char * index_str,size_t index_len,uint16_t id,size_t buflen,gboolean hex)3607 gboolean mdAppendDPIStr(
3608 mdFlowExporter_t *exporter,
3609 uint8_t *buf,
3610 char *label,
3611 char *index_str,
3612 size_t index_len,
3613 uint16_t id,
3614 size_t buflen,
3615 gboolean hex)
3616 {
3617
3618 char delim = exporter->dpi_delimiter;
3619 mdBuf_t *mdbuf = exporter->buf;
3620 size_t brem = MD_REM_MSG(mdbuf);
3621 int ret;
3622
3623 if (buflen == 0) {
3624 return TRUE;
3625 }
3626
3627 if (exporter->dpi_field_table) {
3628 if (!mdGetDPIItem(exporter->dpi_field_table, id)) {
3629 return TRUE;
3630 }
3631 }
3632
3633 if (!exporter->no_index) {
3634 ret = snprintf(mdbuf->cp, brem, "%s%c", label, delim);
3635 MD_CHECK_RET(mdbuf, ret, brem);
3636 }
3637
3638 if (!md_util_append_buffer(mdbuf, &brem, (uint8_t*)index_str, index_len)) {
3639 return FALSE;
3640 }
3641
3642 ret = snprintf(mdbuf->cp, brem, "%d%c", id, delim);
3643 MD_CHECK_RET(mdbuf, ret, brem);
3644
3645 if (!mdPrintVariableLength(mdbuf, &brem, buf, buflen, delim, hex,
3646 exporter->escape_chars)) {
3647 return FALSE;
3648 }
3649
3650 MD_APPEND_CHAR_CHECK(brem, mdbuf, '\n');
3651
3652 return TRUE;
3653 }
3654
3655 /**
3656 * mdCustomFlowPrint
3657 *
3658 *
3659 */
mdCustomFlowPrint(mdFieldList_t * list,mdFullFlow_t * fflow,mdFlowExporter_t * exporter,GError ** err)3660 int mdCustomFlowPrint(
3661 mdFieldList_t *list,
3662 mdFullFlow_t *fflow,
3663 mdFlowExporter_t *exporter,
3664 GError **err)
3665 {
3666 mdFieldList_t *cnode = NULL;
3667 mdBuf_t *buf = exporter->buf;
3668 char *bufstart = buf->cp;
3669 size_t brem = MD_REM_MSG(buf);
3670 size_t buflen;
3671 size_t rc = 0;
3672 int ret;
3673
3674
3675 if (exporter->dpionly && !(fflow->app_tid || fflow->dhcpfp)) {
3676 return 1;
3677 }
3678
3679 if (exporter->json) {
3680 ret = snprintf(buf->cp, brem, "{\"flows\":{");
3681 MD_CHECK_RET(buf, ret, brem);
3682 }
3683
3684 for (cnode = list; cnode; cnode = cnode->next) {
3685 if (!cnode->print_fn(fflow, buf, &brem, cnode->decorator->str)) {
3686 return 0;
3687 }
3688 rc++;
3689 }
3690
3691 if (exporter->basic_list_dpi &&
3692 (fflow->app_tid || fflow->dhcpfp || fflow->stats))
3693 {
3694 ret = mdExporterDPIFlowPrint(exporter, fflow, NULL, 0, err);
3695 } else if (exporter->custom_list_dpi && fflow->app_tid) {
3696 /* reset buffer - since it will be copied for each DPI line */
3697 char *indexstr = NULL;
3698 buflen = MD_MSG_LEN(buf);
3699 indexstr = malloc(buflen);
3700 memcpy(indexstr, buf->buf, buflen);
3701 buf->cp = buf->buf;
3702 ret = mdExporterDPIFlowPrint(exporter, fflow, indexstr, buflen, err);
3703 free(indexstr);
3704 } else if (!exporter->dpionly) {
3705 /* remove last delimiter */
3706 buf->cp -= 1;
3707 brem += 1;
3708 if (exporter->json) {
3709 ret = snprintf(buf->cp, brem, "}}\n");
3710 MD_CHECK_RET(buf, ret, brem);
3711 } else {
3712 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
3713 }
3714 rc = md_util_write_buffer(exporter->lfp, buf, exporter->name, err);
3715
3716 if (!rc) {
3717 return -1;
3718 }
3719
3720 exporter->exp_bytes += rc;
3721
3722 ret = rc;
3723 } else {
3724 /* if DPI_ONLY - only print line if DPI is associated with it." */
3725 /* don't write anything - rewind buffer */
3726 buf->cp = bufstart;
3727 ret = 1;
3728 }
3729
3730 return ret;
3731 }
3732
3733
3734 /**
3735 * mdDPIIndex
3736 *
3737 * print an index line in the output file.
3738 *
3739 */
mdDPIIndex(mdFlowExporter_t * exporter,mdFullFlow_t * flow,char * label)3740 static gboolean mdDPIIndex(
3741 mdFlowExporter_t *exporter,
3742 mdFullFlow_t *flow,
3743 char *label)
3744 {
3745 mdBuf_t *buf = exporter->buf;
3746 size_t brem = MD_REM_MSG(buf);
3747 char strdec[10];
3748 char intdec[10];
3749 int ret;
3750
3751 if (exporter->json) {
3752 /* no for JSON */
3753 return TRUE;
3754 }
3755
3756 snprintf(strdec, sizeof(strdec), "%%s%c", exporter->dpi_delimiter);
3757 snprintf(intdec, sizeof(intdec), "%%u%c", exporter->dpi_delimiter);
3758
3759 if (table_hash) {
3760 label = mdGetTableItem(0);
3761 if (label == NULL) {
3762 return TRUE;
3763 }
3764 }
3765
3766 if (!exporter->multi_files) {
3767 ret = snprintf(buf->cp, brem, "%s%c", label, exporter->dpi_delimiter);
3768 MD_CHECK_RET(buf, ret, brem);
3769 }
3770
3771 if (!mdPrintFlowKeyHash(flow, buf, &brem, intdec)) return FALSE;
3772 if (!mdPrintSTIMEMS(flow, buf, &brem, "%llu")) return FALSE;
3773 MD_APPEND_CHAR_CHECK(brem, buf, exporter->dpi_delimiter);
3774 if (!mdPrintSIP(flow, buf, &brem, strdec)) return FALSE;
3775 if (!mdPrintDIP(flow, buf, &brem, strdec)) return FALSE;
3776 if (!mdPrintProto(flow, buf, &brem, intdec)) return FALSE;
3777 if (!mdPrintSPort(flow, buf, &brem, intdec)) return FALSE;
3778 if (!mdPrintDPort(flow, buf, &brem, intdec)) return FALSE;
3779 if (!mdPrintVLANINT(flow, buf, &brem, intdec)) return FALSE;
3780 if (!mdPrintOBDomain(flow, buf, &brem, "%u")) return FALSE;
3781
3782 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
3783
3784 if (exporter->multi_files) {
3785 FILE *fp = mdGetTableFile(exporter, 0);
3786 size_t rc;
3787 GError *err;
3788 if (fp == NULL) {
3789 g_warning("Error retrieving file for index records");
3790 return TRUE;
3791 }
3792 rc = md_util_write_buffer(fp, buf, exporter->name, &err);
3793 if (!rc) {
3794 g_warning("Error writing index records: %s",
3795 err->message);
3796 g_clear_error(&err);
3797 return FALSE;
3798
3799 }
3800 exporter->exp_bytes += rc;
3801 }
3802
3803 return TRUE;
3804
3805 }
3806
mdDPIExtendedIndex(mdFlowExporter_t * exporter,mdFullFlow_t * flow)3807 static gboolean mdDPIExtendedIndex(
3808 mdFlowExporter_t *exporter,
3809 mdFullFlow_t *flow)
3810 {
3811 mdBuf_t *buf = exporter->buf;
3812 size_t brem = MD_REM_MSG(buf);
3813 char delim = exporter->dpi_delimiter;
3814 char strdec[10];
3815 char intdec[10];
3816
3817 snprintf(strdec, sizeof(strdec), "%%s%c", delim);
3818 snprintf(intdec, sizeof(intdec), "%%d%c", delim);
3819
3820 if (!mdPrintSTIME(flow, buf, &brem, strdec)) return FALSE;
3821 if (!mdPrintSIP(flow, buf, &brem, strdec)) return FALSE;
3822 if (!mdPrintDIP(flow, buf, &brem, strdec)) return FALSE;
3823 if (!mdPrintProto(flow, buf, &brem, intdec)) return FALSE;
3824 if (!mdPrintSPort(flow, buf, &brem, intdec)) return FALSE;
3825 if (!mdPrintDPort(flow, buf, &brem, intdec)) return FALSE;
3826 if (!mdPrintVLANINT(flow, buf, &brem, intdec)) return FALSE;
3827 if (!mdPrintOBDomain(flow, buf, &brem, intdec)) return FALSE;
3828
3829 return TRUE;
3830 }
3831
3832 /**
3833 * mdExportFlowStats
3834 *
3835 */
mdExportFlowStats(mdFlowExporter_t * exporter,yaf_flow_stats_t * stats,char * index_str,size_t index_len,char * label,uint8_t rev)3836 static gboolean mdExportFlowStats(
3837 mdFlowExporter_t *exporter,
3838 yaf_flow_stats_t *stats,
3839 char *index_str,
3840 size_t index_len,
3841 char *label,
3842 uint8_t rev)
3843 {
3844 char delim = exporter->delimiter;
3845 mdBuf_t *buf = exporter->buf;
3846 size_t brem = MD_REM_MSG(buf);
3847 char *bufstart = buf->cp;
3848 int ret;
3849 GError *err;
3850
3851 if (exporter->no_flow_stats) {
3852 return TRUE;
3853 }
3854
3855 if (!exporter->no_index) {
3856 ret = snprintf(buf->cp, brem, "%s%c", label, delim);
3857 MD_CHECK_RET(buf, ret, brem);
3858 }
3859
3860 if (!md_util_append_buffer(buf, &brem, (uint8_t*)index_str, index_len)) {
3861 return FALSE;
3862 }
3863
3864 ret = snprintf(buf->cp, brem, "%u%c%u%c%u%c%"PRIu64,
3865 stats->tcpUrgTotalCount, delim,
3866 stats->smallPacketCount, delim,
3867 stats->nonEmptyPacketCount, delim,
3868 stats->dataByteCount);
3869 MD_CHECK_RET(buf, ret, brem);
3870 ret = snprintf(buf->cp, brem, "%c%"PRIu64"%c%d%c%d%c%d%c", delim,
3871 stats->averageInterarrivalTime, delim,
3872 stats->firstNonEmptyPacketSize, delim,
3873 stats->largePacketCount, delim,
3874 stats->maxPacketSize, delim);
3875 MD_CHECK_RET(buf, ret, brem);
3876 ret = snprintf(buf->cp, brem, "%02x%c%d%c%"PRIu64"%c",
3877 stats->firstEightNonEmptyPacketDirections, delim,
3878 stats->standardDeviationPayloadLength, delim,
3879 stats->standardDeviationInterarrivalTime, delim);
3880 MD_CHECK_RET(buf, ret, brem);
3881 if (stats->nonEmptyPacketCount) {
3882 ret = snprintf(buf->cp, brem, "%"PRIu64"%c",
3883 stats->dataByteCount/stats->nonEmptyPacketCount,
3884 delim);
3885 MD_CHECK_RET(buf, ret, brem);
3886 } else {
3887 MD_APPEND_CHAR_CHECK(brem, buf, '0');
3888 MD_APPEND_CHAR_CHECK(brem, buf, delim);
3889 }
3890
3891 if (rev) {
3892 ret = snprintf(buf->cp, brem, "%u%c%u%c%u%c%"PRIu64,
3893 stats->reverseTcpUrgTotalCount, delim,
3894 stats->reverseSmallPacketCount, delim,
3895 stats->reverseNonEmptyPacketCount, delim,
3896 stats->reverseDataByteCount);
3897 MD_CHECK_RET(buf, ret, brem);
3898 ret = snprintf(buf->cp, brem, "%c%"PRIu64"%c%d%c%d%c%d%c", delim,
3899 stats->reverseAverageInterarrivalTime, delim,
3900 stats->reverseFirstNonEmptyPacketSize, delim,
3901 stats->reverseLargePacketCount, delim,
3902 stats->reverseMaxPacketSize, delim);
3903 MD_CHECK_RET(buf, ret, brem);
3904 ret = snprintf(buf->cp, brem, "%d%c%"PRIu64"%c",
3905 stats->reverseStandardDeviationPayloadLength,
3906 delim,
3907 stats->reverseStandardDeviationInterarrivalTime,
3908 delim);
3909 MD_CHECK_RET(buf, ret, brem);
3910 if (stats->reverseNonEmptyPacketCount) {
3911 ret = snprintf(buf->cp, brem, "%"PRIu64,
3912 stats->reverseDataByteCount / stats->reverseNonEmptyPacketCount);
3913 MD_CHECK_RET(buf, ret, brem);
3914
3915 } else {
3916 MD_APPEND_CHAR_CHECK(brem, buf, '0');
3917 }
3918 } else {
3919 ret = snprintf(buf->cp, brem,"0%c0%c0%c0%c0%c0%c0%c0%c0%c0%c0%c0",
3920 delim, delim, delim, delim, delim, delim, delim,
3921 delim, delim, delim, delim);
3922 MD_CHECK_RET(buf, ret, brem);
3923 }
3924
3925 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
3926
3927
3928 if (exporter->multi_files) {
3929 FILE *fp = mdGetTableFile(exporter, 500);
3930 size_t rc;
3931 if (fp == NULL) {
3932 buf->cp = bufstart;
3933 return TRUE;
3934 }
3935
3936 rc = md_util_write_buffer(fp, buf, exporter->name, &err);
3937
3938 if (!rc) {
3939 g_warning("Error writing file for flowstats: %s",
3940 err->message);
3941 g_clear_error(&err);
3942 return FALSE;
3943 }
3944
3945 exporter->exp_bytes += rc;
3946 }
3947
3948 return TRUE;
3949 }
3950
mdJsonizeFlowStats(mdFlowExporter_t * exporter,yaf_flow_stats_t * stats,char * index_str,size_t index_len,uint8_t rev)3951 static gboolean mdJsonizeFlowStats(
3952 mdFlowExporter_t *exporter,
3953 yaf_flow_stats_t *stats,
3954 char *index_str,
3955 size_t index_len,
3956 uint8_t rev)
3957 {
3958 mdBuf_t *buf = exporter->buf;
3959 size_t brem = MD_REM_MSG(buf);
3960 int ret;
3961
3962 if (exporter->no_flow_stats) {
3963 return TRUE;
3964 }
3965
3966 if (exporter->no_index) {
3967 if (!md_util_append_buffer(buf, &brem, (uint8_t*)index_str, index_len)) {
3968 return FALSE;
3969 }
3970 }
3971
3972 ret = snprintf(buf->cp, brem, "\"tcpUrgTotalCount\":%u,",
3973 stats->tcpUrgTotalCount);
3974 MD_CHECK_RET(buf, ret, brem);
3975 ret = snprintf(buf->cp, brem, "\"smallPacketCount\":%u,",
3976 stats->smallPacketCount);
3977 MD_CHECK_RET(buf, ret, brem);
3978 ret = snprintf(buf->cp, brem, "\"nonEmptyPacketCount\":%u,",
3979 stats->nonEmptyPacketCount);
3980 MD_CHECK_RET(buf, ret, brem);
3981 ret = snprintf(buf->cp, brem, "\"dataByteCount\":%"PRIu64",",
3982 stats->dataByteCount);
3983 MD_CHECK_RET(buf, ret, brem);
3984 ret = snprintf(buf->cp, brem, "\"averageInterarrivalTime\":%"PRIu64",",
3985 stats->averageInterarrivalTime);
3986 MD_CHECK_RET(buf, ret, brem);
3987 ret = snprintf(buf->cp, brem, "\"firstNonEmptyPacketSize\":%d,",
3988 stats->firstNonEmptyPacketSize);
3989 MD_CHECK_RET(buf, ret, brem);
3990 ret = snprintf(buf->cp, brem, "\"largePacketCount\":%d,",
3991 stats->largePacketCount);
3992 MD_CHECK_RET(buf, ret, brem);
3993 ret = snprintf(buf->cp, brem, "\"maxPacketSize\":%d,",
3994 stats->maxPacketSize);
3995 MD_CHECK_RET(buf, ret, brem);
3996 ret = snprintf(buf->cp, brem, "\"firstEightNonEmptyPacketDirections\":\"%02x\",",
3997 stats->firstEightNonEmptyPacketDirections);
3998 MD_CHECK_RET(buf, ret, brem);
3999 ret = snprintf(buf->cp, brem, "\"standardDeviationPayloadLength\":%u,",
4000 stats->standardDeviationPayloadLength);
4001 MD_CHECK_RET(buf, ret, brem);
4002 ret = snprintf(buf->cp, brem, "\"standardDeviationInterarrivalTime\":%"PRIu64",",
4003 stats->standardDeviationInterarrivalTime);
4004 MD_CHECK_RET(buf, ret, brem);
4005
4006 if (stats->nonEmptyPacketCount) {
4007 ret = snprintf(buf->cp, brem, "\"bytesPerPacket\":%"PRIu64",",
4008 stats->dataByteCount/stats->nonEmptyPacketCount);
4009 MD_CHECK_RET(buf, ret, brem);
4010 }
4011
4012 if (rev) {
4013 ret = snprintf(buf->cp, brem, "\"reverseTcpUrgTotalCount\":%u,",
4014 stats->reverseTcpUrgTotalCount);
4015 MD_CHECK_RET(buf, ret, brem);
4016 ret = snprintf(buf->cp, brem, "\"reverseSmallPacketCount\":%u,",
4017 stats->reverseSmallPacketCount);
4018 MD_CHECK_RET(buf, ret, brem);
4019 ret = snprintf(buf->cp, brem, "\"reverseNonEmptyPacketCount\":%u,",
4020 stats->reverseNonEmptyPacketCount);
4021 MD_CHECK_RET(buf, ret, brem);
4022 ret = snprintf(buf->cp, brem, "\"reverseDataByteCount\":%"PRIu64",",
4023 stats->reverseDataByteCount);
4024 MD_CHECK_RET(buf, ret, brem);
4025 ret = snprintf(buf->cp, brem, "\"reverseAverageInterarrivalTime\":%"PRIu64",",
4026 stats->reverseAverageInterarrivalTime);
4027 MD_CHECK_RET(buf, ret, brem);
4028 ret = snprintf(buf->cp, brem, "\"reverseFirstNonEmptyPacketSize\":%d,",
4029 stats->reverseFirstNonEmptyPacketSize);
4030 MD_CHECK_RET(buf, ret, brem);
4031 ret = snprintf(buf->cp, brem, "\"reverseLargePacketCount\":%d,",
4032 stats->reverseLargePacketCount);
4033 MD_CHECK_RET(buf, ret, brem);
4034 ret = snprintf(buf->cp, brem, "\"reverseMaxPacketSize\":%d,",
4035 stats->reverseMaxPacketSize);
4036 MD_CHECK_RET(buf, ret, brem);
4037 ret = snprintf(buf->cp, brem, "\"reverseStandardDeviationPayloadLength\":%u,",
4038 stats->reverseStandardDeviationPayloadLength);
4039 MD_CHECK_RET(buf, ret, brem);
4040 ret = snprintf(buf->cp, brem,
4041 "\"reverseStandardDeviationInterarrivalTime\":%"PRIu64",",
4042 stats->reverseStandardDeviationInterarrivalTime);
4043 MD_CHECK_RET(buf, ret, brem);
4044
4045 if (stats->reverseNonEmptyPacketCount) {
4046 ret = snprintf(buf->cp, brem, "\"reverseBytesPerPacket\":%"PRIu64",",
4047 stats->reverseDataByteCount/
4048 stats->reverseNonEmptyPacketCount);
4049 MD_CHECK_RET(buf, ret, brem);
4050 }
4051 }
4052 return TRUE;
4053 }
4054
4055 /**
4056 * mdExportBLMultiFiles
4057 *
4058 */
4059
mdExportBLMultiFiles(mdFlowExporter_t * exporter,fbBasicList_t * bl,char * index_str,size_t index_len,char * label,gboolean hex)4060 gboolean mdExportBLMultiFiles(
4061 mdFlowExporter_t *exporter,
4062 fbBasicList_t *bl,
4063 char *index_str,
4064 size_t index_len,
4065 char *label,
4066 gboolean hex)
4067 {
4068 uint16_t elem_id;
4069 char delim = exporter->dpi_delimiter;
4070 GString *tstr = NULL;
4071 mdBuf_t *mdbuf = exporter->buf;
4072 GError *err;
4073
4074 if (bl->infoElement == NULL) {
4075 /* this item must not be included in infoModel. */
4076 return TRUE;
4077 }
4078
4079 elem_id = bl->infoElement->num;
4080
4081 if (table_hash) {
4082 label = mdGetTableItem(elem_id);
4083 if (label == NULL) {
4084 return TRUE;
4085 }
4086 }
4087
4088 tstr = g_string_new("");
4089
4090 g_string_append_len(tstr, index_str, index_len);
4091 g_string_append_printf(tstr, "%d%c", elem_id, delim);
4092
4093 if (exporter->dedup_per_flow) {
4094 if (!md_dedup_basic_list(bl, mdbuf, tstr, delim, hex, exporter->escape_chars)) {
4095 g_string_free(tstr, TRUE);
4096 return FALSE;
4097 }
4098 } else {
4099 if (!mdPrintBasicList(mdbuf, tstr, bl, delim, hex, exporter->escape_chars)) {
4100 g_string_free(tstr, TRUE);
4101 return FALSE;
4102 }
4103 }
4104
4105 g_string_free(tstr, TRUE);
4106
4107 if (MD_MSG_LEN(mdbuf)) {
4108 FILE *fp = mdGetTableFile(exporter, elem_id);
4109 size_t rc;
4110 if (fp == NULL) {
4111 g_debug("%s: Error retrieving file for id %d", exporter->name,
4112 elem_id);
4113 return FALSE;
4114 }
4115
4116 rc = md_util_write_buffer(fp, mdbuf, exporter->name, &err);
4117
4118 if (!rc) {
4119 g_warning("Error writing file for id %d: %s",
4120 elem_id, err->message);
4121 g_clear_error(&err);
4122 return FALSE;
4123 }
4124
4125 exporter->exp_bytes += rc;
4126 }
4127
4128 return TRUE;
4129 }
4130
4131
4132
4133 /**
4134 * mdExportBLCustomList
4135 *
4136 *
4137 */
mdExportBLCustomList(mdFlowExporter_t * exporter,fbBasicList_t * bl,char * index_str,size_t index_len,char * label,gboolean hex)4138 gboolean mdExportBLCustomList(
4139 mdFlowExporter_t *exporter,
4140 fbBasicList_t *bl,
4141 char *index_str,
4142 size_t index_len,
4143 char *label,
4144 gboolean hex)
4145 {
4146 uint16_t elem_id;
4147 char delim = exporter->dpi_delimiter;
4148 GString *tstr = NULL;
4149 mdBuf_t *mdbuf = exporter->buf;
4150
4151 if (bl->infoElement == NULL) {
4152 /* InfoElement must be in infoModel */
4153 return TRUE;
4154 }
4155
4156 elem_id = bl->infoElement->num;
4157
4158 if (exporter->dpi_field_table) {
4159 if (!mdGetDPIItem(exporter->dpi_field_table, elem_id)) {
4160 return TRUE;
4161 }
4162 }
4163
4164 tstr = g_string_new("");
4165
4166 g_string_append_len(tstr, index_str, index_len);
4167
4168 g_string_append_printf(tstr, "%d%c", elem_id, delim);
4169
4170 if (exporter->dedup_per_flow) {
4171 if (!md_dedup_basic_list(bl, mdbuf, tstr, delim, hex, exporter->escape_chars)) {
4172 g_string_free(tstr, TRUE);
4173 return FALSE;
4174 }
4175 } else {
4176 if (!mdPrintBasicList(mdbuf, tstr, bl, delim, hex, exporter->escape_chars)) {
4177 g_string_free(tstr, TRUE);
4178 return FALSE;
4179 }
4180 }
4181
4182 g_string_free(tstr, TRUE);
4183 return TRUE;
4184 }
4185
4186
4187
4188
4189 /**
4190 * mdExportBL
4191 *
4192 *
4193 *
4194 */
mdExportBL(mdFlowExporter_t * exporter,fbBasicList_t * bl,char * index_str,size_t index_len,char * label,gboolean hex)4195 gboolean mdExportBL(
4196 mdFlowExporter_t *exporter,
4197 fbBasicList_t *bl,
4198 char *index_str,
4199 size_t index_len,
4200 char *label,
4201 gboolean hex)
4202 {
4203 uint16_t elem_id;
4204 char delim = exporter->dpi_delimiter;
4205 mdBuf_t *mdbuf = exporter->buf;
4206 GString *tstr = NULL;
4207
4208 if (bl->infoElement == NULL) {
4209 /* InfoElement must be in infoModel */
4210 return TRUE;
4211 }
4212
4213 elem_id = bl->infoElement->num;
4214
4215 if (exporter->dpi_field_table) {
4216 if (!mdGetDPIItem(exporter->dpi_field_table, elem_id)) {
4217 return TRUE;
4218 }
4219 }
4220
4221 tstr = g_string_new("");
4222
4223 if (index_len) {
4224 if (!exporter->no_index) {
4225 /* print label */
4226 g_string_append_printf(tstr, "%s%c", label, delim);
4227 }
4228 g_string_append_len(tstr, index_str, index_len);
4229 g_string_append_printf(tstr, "%d%c", elem_id, delim);
4230 } else {
4231 g_string_append_printf(tstr, "%s%c%d%c", label, delim, elem_id, delim);
4232 }
4233
4234 if (exporter->dedup_per_flow) {
4235 if (!md_dedup_basic_list(bl, mdbuf, tstr, delim, hex, exporter->escape_chars)) {
4236 g_string_free(tstr, TRUE);
4237 return FALSE;
4238 }
4239 } else {
4240 if (!mdPrintBasicList(mdbuf, tstr, bl, delim, hex, exporter->escape_chars)) {
4241 g_string_free(tstr, TRUE);
4242 return FALSE;
4243 }
4244 }
4245
4246 g_string_free(tstr, TRUE);
4247 return TRUE;
4248 }
4249
4250 /**
4251 * mdJsonizeBLElement
4252 *
4253 */
mdJsonizeBLElement(mdFlowExporter_t * exporter,fbBasicList_t * bl,char * index_str,size_t index_len,char * label,gboolean hex)4254 gboolean mdJsonizeBLElement(
4255 mdFlowExporter_t *exporter,
4256 fbBasicList_t *bl,
4257 char *index_str,
4258 size_t index_len,
4259 char *label,
4260 gboolean hex)
4261 {
4262
4263 uint16_t elem_id;
4264 const fbInfoElement_t *ie = NULL;
4265 uint16_t w = 0;
4266 fbVarfield_t *var = NULL;
4267 char hexdump[65534];
4268 size_t hexlen = sizeof(hexdump);
4269 size_t buflen;
4270 GString *tstr = NULL;
4271 mdBuf_t *mdbuf = exporter->buf;
4272 char *bufstart = mdbuf->cp;
4273 char *blstart;
4274 size_t brem = MD_REM_MSG(mdbuf);
4275 int ret;
4276
4277 if (bl->infoElement == NULL) {
4278 return TRUE;
4279 }
4280
4281 /* Get the IE name from fixbuf */
4282
4283 elem_id = bl->infoElement->num;
4284 ie = fbInfoModelGetElementByID(mdInfoModel(), elem_id, CERT_PEN);
4285
4286 if (exporter->dpi_field_table) {
4287 if (!mdGetDPIItem(exporter->dpi_field_table, elem_id)) {
4288 return TRUE;
4289 }
4290 }
4291
4292 if (exporter->dedup_per_flow) {
4293 tstr = md_dedup_basic_list_no_count(bl, ',', TRUE, hex,
4294 exporter->escape_chars);
4295 if (tstr) {
4296 ret = snprintf(mdbuf->cp, brem, "\"%s\":[%s],", ie->ref.name,
4297 tstr->str);
4298 if ((ret < 0) || ((size_t)ret >= brem)) {
4299 g_string_free(tstr, TRUE);
4300 return FALSE;
4301 }
4302 mdbuf->cp += ret;
4303 brem -= ret;
4304 g_string_free(tstr, TRUE);
4305 }
4306 } else {
4307 ret = snprintf(mdbuf->cp, brem, "\"%s\":[\"", ie->ref.name);
4308 MD_CHECK_RET(mdbuf, ret, brem);
4309 blstart = mdbuf->cp;
4310 for (w = 0;
4311 (var = (fbVarfield_t *)fbBasicListGetIndexedDataPtr(bl, w));
4312 w++) {
4313
4314 if (hex) {
4315 buflen = var->len;
4316 if (buflen > sizeof(hexdump)) {
4317 buflen = sizeof(hexdump);
4318 }
4319 ret = md_util_hexdump_append(hexdump, &hexlen,
4320 var->buf, buflen);
4321 if (!ret) return FALSE;
4322 if (!md_util_append_buffer(mdbuf, &brem, (uint8_t*)hexdump, ret)) {
4323 return FALSE;
4324 }
4325 } else {
4326 if (!mdJsonifyEscapeChars(mdbuf, &brem, var->buf, var->len)) {
4327 return FALSE;
4328 }
4329 }
4330 ret = snprintf(mdbuf->cp, brem, "\", \"");
4331 MD_CHECK_RET(mdbuf, ret, brem);
4332 }
4333
4334 if (mdbuf->cp > blstart) {
4335 mdbuf->cp -= 3;
4336 brem += 3;
4337 if (brem > 2) {
4338 MD_APPEND_CHAR(mdbuf, ']');
4339 MD_APPEND_CHAR(mdbuf, ',');
4340 } else {
4341 return FALSE;
4342 }
4343 } else {
4344 mdbuf->cp = bufstart;
4345 }
4346 }
4347
4348 return TRUE;
4349 }
4350
4351 /**
4352 * mdJsonizeVLElement
4353 *
4354 */
mdJsonizeVLElement(mdFlowExporter_t * exporter,uint8_t * buf,char * label,char * index_str,size_t index_len,uint16_t id,size_t buflen,gboolean hex)4355 gboolean mdJsonizeVLElement(
4356 mdFlowExporter_t *exporter,
4357 uint8_t *buf,
4358 char *label,
4359 char *index_str,
4360 size_t index_len,
4361 uint16_t id,
4362 size_t buflen,
4363 gboolean hex)
4364 {
4365
4366 const fbInfoElement_t *ie = NULL;
4367 mdBuf_t *mdbuf = exporter->buf;
4368 size_t brem = MD_REM_MSG(mdbuf);
4369 int ret;
4370 char hexdump[65534];
4371 size_t hexlen = sizeof(hexdump);
4372
4373 if (exporter->dpi_field_table) {
4374 if (!mdGetDPIItem(exporter->dpi_field_table, id)) {
4375 return TRUE;
4376 }
4377 }
4378
4379 /* Get the IE name from fixbuf */
4380
4381 ie = fbInfoModelGetElementByID(mdInfoModel(), id, CERT_PEN);
4382
4383 if (ie->type != 0 && ie->type < 11) {
4384 /* use fixbuf 1.4 to get type information, and if integer,
4385 * don't quote the value: (0 is octet array, 1-10 are integers, floats) */
4386 ret = snprintf(mdbuf->cp, brem, "\"%s\":", ie->ref.name);
4387 MD_CHECK_RET(mdbuf, ret, brem);
4388
4389 if (!mdPrintVariableLength(mdbuf, &brem, buf, buflen, '"', hex,
4390 exporter->escape_chars))
4391 {
4392 return FALSE;
4393 }
4394
4395
4396 } else {
4397 /* quote because it's an octet array, string, or other */
4398
4399 ret = snprintf(mdbuf->cp, brem, "\"%s\":\"", ie->ref.name);
4400 MD_CHECK_RET(mdbuf, ret, brem);
4401 if (hex) {
4402 ret = md_util_hexdump_append(hexdump, &hexlen, buf, buflen);
4403 if (!md_util_append_buffer(mdbuf, &brem, (uint8_t*)hexdump, ret)) {
4404 return FALSE;
4405 }
4406 } else {
4407 mdJsonifyEscapeChars(mdbuf, &brem, buf, buflen);
4408 }
4409 MD_APPEND_CHAR_CHECK(brem, mdbuf, '\"');
4410 }
4411
4412 MD_APPEND_CHAR_CHECK(brem, mdbuf, ',');
4413
4414 return TRUE;
4415 }
4416
4417 /**
4418 * mdExporterDPIGetIndexStr
4419 *
4420 *
4421 */
mdExporterDPIGetIndexStr(mdFlowExporter_t * exporter,mdFullFlow_t * flow)4422 gboolean mdExporterDPIGetIndexStr(
4423 mdFlowExporter_t *exporter,
4424 mdFullFlow_t *flow)
4425 {
4426 char delim = exporter->dpi_delimiter;
4427 int ret;
4428 mdBuf_t *buf = exporter->buf;
4429 size_t brem = MD_REM_MSG(buf);
4430
4431 if (flow->app_tid == 0 && !flow->p0f && !flow->dhcpfp && !flow->stats) {
4432 return FALSE;
4433 }
4434
4435 /* this prints the index record for this flow */
4436 if (exporter->dpionly) {
4437 if (exporter->no_index) {
4438 /* if this fails - it will fail in the DPI Flow Print and realloc*/
4439 mdDPIExtendedIndex(exporter, flow);
4440 } else {
4441 ret = snprintf(buf->cp, brem, "%u%c%"PRIu64"%c%u%c",
4442 md_util_flow_key_hash(flow->rec), delim,
4443 flow->rec->flowStartMilliseconds, delim,
4444 flow->rec->observationDomainId, delim);
4445 MD_CHECK_RET(buf, ret, brem);
4446 }
4447 }
4448
4449 return TRUE;
4450 }
4451
4452
4453
mdExporterSSLCertHash(mdFlowExporter_t * exporter,fbVarfield_t * ct,char * index_str,size_t index_len,int cert_no)4454 int mdExporterSSLCertHash(
4455 mdFlowExporter_t *exporter,
4456 fbVarfield_t *ct,
4457 char *index_str,
4458 size_t index_len,
4459 int cert_no)
4460 {
4461 #if HAVE_OPENSSL
4462 mdBuf_t *buf = exporter->buf;
4463 size_t brem = MD_REM_MSG(buf);
4464 char ssl_buffer[4096];
4465 char delim = exporter->dpi_delimiter;
4466 size_t bufsz = sizeof(ssl_buffer);
4467 int i, ret;
4468 unsigned char md5[MD5_DIGEST_LENGTH];
4469 unsigned char sha1[SHA_DIGEST_LENGTH];
4470
4471 if (exporter->json) {
4472 /* remove '},' */
4473 buf->cp -= 2;
4474 brem += 2;
4475 if (exporter->md5_hash || mdExporterCheckSSLConfig(exporter, 299, 3)) {
4476 md_ssl_md5_hash(md5, ct->buf, ct->len);
4477 ret = snprintf(buf->cp, brem, ",\"sslCertificateMD5\":\"");
4478 MD_CHECK_RET(buf, ret, brem);
4479 ret = md_util_hexdump_append_nospace(buf->cp, &brem,
4480 md5, MD5_DIGEST_LENGTH);
4481 if (!ret) {
4482 return 0;
4483 }
4484 buf->cp += ret;
4485 MD_APPEND_CHAR_CHECK(brem, buf, '"');
4486 }
4487 if (exporter->sha1_hash || mdExporterCheckSSLConfig(exporter, 298, 3)) {
4488 md_ssl_sha1_hash(sha1, ct->buf, ct->len);
4489 ret = snprintf(buf->cp, brem, ",\"sslCertificateSHA1\":\"");
4490 MD_CHECK_RET(buf, ret, brem);
4491 ret = md_util_hexdump_append_nospace(buf->cp, &brem,
4492 sha1, SHA_DIGEST_LENGTH);
4493 if (!ret) {
4494 return 0;
4495 }
4496 buf->cp += ret;
4497 MD_APPEND_CHAR_CHECK(brem, buf, '"');
4498 }
4499
4500 MD_APPEND_CHAR_CHECK(brem, buf, '}');
4501 MD_APPEND_CHAR_CHECK(brem, buf, ',');
4502
4503 } else {
4504 if (exporter->md5_hash || mdExporterCheckSSLConfig(exporter, 299, 3)) {
4505 md_ssl_md5_hash(md5, ct->buf, ct->len);
4506 i = snprintf(ssl_buffer, bufsz, "I%c%d%c", delim, cert_no, delim);
4507 bufsz -= i;
4508 i += md_util_hexdump_append(ssl_buffer + i, &bufsz, md5,
4509 MD5_DIGEST_LENGTH);
4510 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
4511 SSL_DEFAULT, index_str, index_len, 299,
4512 i, FALSE);
4513 }
4514 bufsz = sizeof(ssl_buffer);
4515 if (exporter->sha1_hash || mdExporterCheckSSLConfig(exporter, 298, 3)) {
4516 md_ssl_sha1_hash(sha1, ct->buf, ct->len);
4517 i = snprintf(ssl_buffer, bufsz, "I%c%d%c", delim, cert_no, delim);
4518 bufsz -= i;
4519 i += md_util_hexdump_append(ssl_buffer + i, &bufsz, sha1,
4520 SHA_DIGEST_LENGTH);
4521 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
4522 SSL_DEFAULT, index_str, index_len, 298,
4523 i, FALSE);
4524 }
4525 }
4526
4527 #endif
4528
4529 return 1;
4530
4531 }
4532
4533 /**
4534 * mdExporterSSLBase64Encode
4535 *
4536 */
mdExporterSSLBase64Encode(mdFlowExporter_t * exporter,fbVarfield_t * ct,char * index_str,size_t index_len,int cert_no)4537 gboolean mdExporterSSLBase64Encode(
4538 mdFlowExporter_t *exporter,
4539 fbVarfield_t *ct,
4540 char *index_str,
4541 size_t index_len,
4542 int cert_no)
4543 {
4544 char delim = exporter->dpi_delimiter;
4545 char ssl_buffer[4096];
4546 int ret;
4547 gchar *base1 = NULL;
4548 size_t bufsz = sizeof(ssl_buffer);
4549
4550 ret = snprintf(ssl_buffer, bufsz, "I%c%d%c", delim, cert_no, delim);
4551 bufsz -= ret;
4552
4553 base1 = g_base64_encode((const guchar *)ct->buf, ct->len);
4554
4555 if (strlen(base1) < bufsz) {
4556 memcpy(ssl_buffer + ret, base1, strlen(base1));
4557 bufsz = strlen(base1) + ret;
4558 } else {
4559 g_free(base1);
4560 return TRUE;
4561 }
4562
4563 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
4564 SSL_DEFAULT, index_str, index_len, 296,
4565 bufsz, FALSE);
4566 if (base1) {
4567 g_free(base1);
4568 }
4569
4570 return TRUE;
4571 }
4572
4573 /**
4574 * mdExporterDPIFlowPrint
4575 *
4576 * writes all the DPI data to the given FILE.
4577 *
4578 */
mdExporterDPIFlowPrint(mdFlowExporter_t * exporter,mdFullFlow_t * flow,char * index_str,size_t index_len,GError ** err)4579 int mdExporterDPIFlowPrint(
4580 mdFlowExporter_t *exporter,
4581 mdFullFlow_t *flow,
4582 char *index_str,
4583 size_t index_len,
4584 GError **err)
4585 {
4586
4587 mdBuf_t *buf = exporter->buf;
4588 char *bufstart = buf->cp;
4589 gboolean rev = FALSE;
4590 int loop;
4591 size_t rc;
4592 char delim = exporter->dpi_delimiter;
4593 fbBasicList_t *bl = NULL;
4594 int ret;
4595 size_t brem = MD_REM_MSG(buf);
4596 gboolean rv = TRUE;
4597
4598 if (flow->rec->reversePacketTotalCount) {
4599 rev = TRUE;
4600 }
4601
4602 if (exporter->json) {
4603 /* since index_str is technically already in the buffer, just move up
4604 buf->cp index_len */
4605 buf->cp += index_len;
4606 bufstart = buf->cp;
4607 }
4608
4609 if (flow->p0f) {
4610 rv = exporter->VLprint_fn(exporter, flow->p0f->osName.buf, P0F_DEFAULT,
4611 index_str, index_len, 36, flow->p0f->osName.len,
4612 FALSE);
4613 MD_RET0(rv);
4614 rv = exporter->VLprint_fn(exporter, flow->p0f->osVersion.buf,
4615 P0F_DEFAULT, index_str, index_len, 37,
4616 flow->p0f->osVersion.len, FALSE);
4617 MD_RET0(rv);
4618 rv = exporter->VLprint_fn(exporter, flow->p0f->osFingerPrint.buf,
4619 P0F_DEFAULT, index_str, index_len, 107,
4620 flow->p0f->osFingerPrint.len, FALSE);
4621 MD_RET0(rv);
4622 if (rev) {
4623 rv = exporter->VLprint_fn(exporter, flow->p0f->reverseOsName.buf,
4624 P0F_DEFAULT, index_str, index_len,
4625 36|FB_IE_VENDOR_BIT_REVERSE,
4626 flow->p0f->reverseOsName.len, FALSE);
4627 MD_RET0(rv);
4628 rv = exporter->VLprint_fn(exporter, flow->p0f->reverseOsVersion.buf,
4629 P0F_DEFAULT, index_str, index_len,
4630 37|FB_IE_VENDOR_BIT_REVERSE,
4631 flow->p0f->reverseOsVersion.len, FALSE);
4632 MD_RET0(rv);
4633 rv = exporter->VLprint_fn(exporter,
4634 flow->p0f->reverseOsFingerPrint.buf,
4635 P0F_DEFAULT, index_str, index_len,
4636 107|FB_IE_VENDOR_BIT_REVERSE,
4637 flow->p0f->reverseOsFingerPrint.len, FALSE);
4638 MD_RET0(rv);
4639 }
4640 }
4641
4642 switch (flow->app_tid & YTF_BIF) {
4643 case YAF_HTTP_FLOW_TID:
4644 bl = (fbBasicList_t *)flow->app;
4645 for (loop = 0; loop < flow->app_elements; loop++) {
4646 rv = exporter->BLprint_fn(exporter, bl, index_str, index_len,
4647 HTTP_DEFAULT, FALSE);
4648 MD_RET0(rv);
4649 bl++;
4650 }
4651 break;
4652
4653 case YAF_POP3_FLOW_TID:
4654 rv = exporter->BLprint_fn(exporter, (fbBasicList_t *)flow->app,
4655 index_str, index_len, POP3_DEFAULT, FALSE);
4656 MD_RET0(rv);
4657 break;
4658 case YAF_IRC_FLOW_TID:
4659 rv = exporter->BLprint_fn(exporter, (fbBasicList_t *)flow->app,
4660 index_str, index_len, IRC_DEFAULT, FALSE);
4661 MD_RET0(rv);
4662 break;
4663 case YAF_TFTP_FLOW_TID:
4664 {
4665 yaf_tftp_t *tftp = (yaf_tftp_t *)flow->app;
4666 rv = exporter->VLprint_fn(exporter, tftp->tftpMode.buf,
4667 TFTP_DEFAULT, index_str, index_len, 127,
4668 tftp->tftpMode.len, FALSE);
4669 MD_RET0(rv);
4670 rv = exporter->VLprint_fn(exporter, tftp->tftpFilename.buf,
4671 TFTP_DEFAULT, index_str, index_len, 126,
4672 tftp->tftpFilename.len, FALSE);
4673 MD_RET0(rv);
4674 break;
4675 }
4676 case YAF_SLP_FLOW_TID:
4677 {
4678 yaf_slp_t *slp = (yaf_slp_t *)flow->app;
4679 char slp_buffer[20];
4680
4681 snprintf(slp_buffer, sizeof(slp_buffer), "%d", slp->slpVersion);
4682 rv = exporter->VLprint_fn(exporter, (uint8_t *)slp_buffer,
4683 SLP_DEFAULT, index_str, index_len, 128,
4684 strlen(slp_buffer), FALSE);
4685 MD_RET0(rv);
4686 snprintf(slp_buffer, sizeof(slp_buffer), "%d", slp->slpMessageType);
4687 rv = exporter->VLprint_fn(exporter, (uint8_t *)slp_buffer,
4688 SLP_DEFAULT, index_str, index_len, 129,
4689 strlen(slp_buffer), FALSE);
4690 MD_RET0(rv);
4691 rv = exporter->BLprint_fn(exporter, (fbBasicList_t *)flow->app,
4692 index_str, index_len, SLP_DEFAULT, FALSE);
4693 MD_RET0(rv);
4694 break;
4695 }
4696 case YAF_FTP_FLOW_TID:
4697 bl = (fbBasicList_t *)flow->app;
4698 for (loop = 0; loop < flow->app_elements; loop++) {
4699 rv = exporter->BLprint_fn(exporter, bl, index_str, index_len,
4700 FTP_DEFAULT, 0);
4701 MD_RET0(rv);
4702 bl++;
4703 }
4704 break;
4705 case YAF_IMAP_FLOW_TID:
4706 bl = (fbBasicList_t *)flow->app;
4707 for (loop = 0; loop < flow->app_elements; loop++) {
4708 rv = exporter->BLprint_fn(exporter, bl, index_str,
4709 index_len, IMAP_DEFAULT,0);
4710 MD_RET0(rv);
4711 bl++;
4712 }
4713 break;
4714 case YAF_SIP_FLOW_TID:
4715 bl = (fbBasicList_t *)flow->app;
4716 for (loop = 0; loop < flow->app_elements; loop++) {
4717 rv = exporter->BLprint_fn(exporter, bl, index_str,
4718 index_len, SIP_DEFAULT, 0);
4719 MD_RET0(rv);
4720 bl++;
4721 }
4722 break;
4723 case YAF_SMTP_FLOW_TID:
4724 bl = (fbBasicList_t *)flow->app;
4725 for (loop = 0; loop < flow->app_elements; loop++) {
4726 rv = exporter->BLprint_fn(exporter, bl, index_str,
4727 index_len, SMTP_DEFAULT,0);
4728 MD_RET0(rv);
4729 bl++;
4730 }
4731 break;
4732 case YAF_SSH_FLOW_TID:
4733 bl = (fbBasicList_t *)flow->app;
4734 for (loop = 0; loop < flow->app_elements; loop++) {
4735 rv=exporter->BLprint_fn(exporter, bl, index_str, index_len, SSH_DEFAULT, 0);
4736 MD_RET0(rv);
4737 bl++;
4738 }
4739 break;
4740 case YAF_NNTP_FLOW_TID:
4741 bl = (fbBasicList_t *)flow->app;
4742 for (loop = 0; loop < flow->app_elements; loop++) {
4743 rv = exporter->BLprint_fn(exporter, bl, index_str,
4744 index_len, NNTP_DEFAULT,0);
4745 bl++;
4746 }
4747 break;
4748 case SM_INTSSL_FLOW_TID:
4749 {
4750 yaf_newssl_t *sslflow = (yaf_newssl_t *)flow->app;
4751 if (exporter->json) {
4752 if (!mdJsonifyNewSSLRecord(exporter, sslflow, FALSE,
4753 exporter->escape_chars)) {
4754 return 0;
4755 }
4756 } else {
4757 if (!mdExporterTextNewSSLPrint(exporter, sslflow, index_str,
4758 index_len)) {
4759 return 0;
4760 }
4761 }
4762 break;
4763 }
4764 case YAF_SSL_FLOW_TID:
4765 {
4766 yaf_ssl_t *sslflow = (yaf_ssl_t *)flow->app;
4767 yaf_ssl_cert_t *cert = NULL;
4768
4769 char ssl_buffer[20];
4770
4771 if (flow->cert == NULL) {
4772 break;
4773 }
4774
4775 if (sslflow->sslServerCipher) {
4776 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d", sslflow->sslServerCipher);
4777 rv = exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
4778 SSL_DEFAULT,index_str, index_len, 187, strlen(ssl_buffer),
4779 FALSE);
4780 MD_RET0(rv);
4781 }
4782
4783 if (sslflow->sslCompressionMethod) {
4784 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d", sslflow->sslCompressionMethod);
4785 rv = exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
4786 SSL_DEFAULT,index_str, index_len, 188, strlen(ssl_buffer),
4787 FALSE);
4788 MD_RET0(rv);
4789 }
4790
4791 if (sslflow->sslClientVersion) {
4792 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d", sslflow->sslClientVersion);
4793 rv = exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
4794 SSL_DEFAULT,index_str, index_len, 186, strlen(ssl_buffer),
4795 FALSE);
4796 MD_RET0(rv);
4797 }
4798
4799 while ((cert = fbSubTemplateMultiListEntryNextDataPtr(flow->cert,
4800 cert)))
4801 {
4802
4803 /* g_string_append_printf(str, "SSL Cert:");
4804 if (cert->sslSignature.len) {
4805 g_string_append_printf(str, "0x");
4806 }
4807 for (w = 0; w < cert->sslSignature.len; w++) {
4808 g_string_append_printf(str, "%02x",
4809 *(cert->sslSignature.buf + w));
4810 }
4811 g_string_append_printf(str, "%c", delim);*/
4812 rv = exporter->VLprint_fn(exporter, cert->sslIssuerCountryName.buf,
4813 SSL_DEFAULT, index_str, index_len, 191,
4814 cert->sslIssuerCountryName.len, FALSE);
4815 MD_RET0(rv);
4816 rv = exporter->VLprint_fn(exporter, cert->sslIssuerOrgName.buf,
4817 SSL_DEFAULT, index_str, index_len, 192,
4818 cert->sslIssuerOrgName.len, FALSE);
4819 MD_RET0(rv);
4820 rv = exporter->VLprint_fn(exporter, cert->sslIssuerOrgUnitName.buf,
4821 SSL_DEFAULT, index_str, index_len, 193,
4822 cert->sslIssuerOrgUnitName.len, FALSE);
4823 MD_RET0(rv);
4824 rv = exporter->VLprint_fn(exporter, cert->sslIssuerZipCode.buf,
4825 SSL_DEFAULT, index_str, index_len, 194,
4826 cert->sslIssuerZipCode.len, FALSE);
4827 MD_RET0(rv);
4828 rv = exporter->VLprint_fn(exporter, cert->sslIssuerState.buf,
4829 SSL_DEFAULT, index_str, index_len, 195,
4830 cert->sslIssuerState.len, FALSE);
4831 MD_RET0(rv);
4832 rv = exporter->VLprint_fn(exporter, cert->sslIssuerCommonName.buf,
4833 SSL_DEFAULT, index_str, index_len, 196,
4834 cert->sslIssuerCommonName.len, FALSE);
4835 MD_RET0(rv);
4836 rv = exporter->VLprint_fn(exporter, cert->sslIssuerLocalityName.buf,
4837 SSL_DEFAULT, index_str, index_len, 197,
4838 cert->sslIssuerLocalityName.len, FALSE);
4839 MD_RET0(rv);
4840 rv = exporter->VLprint_fn(exporter, cert->sslIssuerStreetAddress.buf,
4841 SSL_DEFAULT, index_str, index_len, 198,
4842 cert->sslIssuerStreetAddress.len, FALSE);
4843 MD_RET0(rv);
4844 rv = exporter->VLprint_fn(exporter, cert->sslSubCountryName.buf,
4845 SSL_DEFAULT, index_str, index_len, 200,
4846 cert->sslSubCountryName.len, FALSE);
4847 MD_RET0(rv);
4848 rv = exporter->VLprint_fn(exporter, cert->sslSubOrgName.buf,
4849 SSL_DEFAULT, index_str, index_len, 201,
4850 cert->sslSubOrgName.len, FALSE);
4851 MD_RET0(rv);
4852 rv = exporter->VLprint_fn(exporter, cert->sslSubOrgUnitName.buf,
4853 SSL_DEFAULT, index_str, index_len, 202,
4854 cert->sslSubOrgUnitName.len, FALSE);
4855 MD_RET0(rv);
4856 rv = exporter->VLprint_fn(exporter, cert->sslSubZipCode.buf,
4857 SSL_DEFAULT, index_str, index_len, 203,
4858 cert->sslSubZipCode.len, FALSE);
4859 MD_RET0(rv);
4860 rv = exporter->VLprint_fn(exporter, cert->sslSubState.buf,
4861 SSL_DEFAULT, index_str, index_len, 204,
4862 cert->sslSubState.len, FALSE);
4863 MD_RET0(rv);
4864 rv = exporter->VLprint_fn(exporter, cert->sslSubCommonName.buf,
4865 SSL_DEFAULT, index_str, index_len, 205,
4866 cert->sslSubCommonName.len, FALSE);
4867 MD_RET0(rv);
4868 rv = exporter->VLprint_fn(exporter, cert->sslSubLocalityName.buf,
4869 SSL_DEFAULT, index_str, index_len, 206,
4870 cert->sslSubLocalityName.len, FALSE);
4871 MD_RET0(rv);
4872 rv = exporter->VLprint_fn(exporter, cert->sslSubStreetAddress.buf,
4873 SSL_DEFAULT, index_str, index_len, 207,
4874 cert->sslSubStreetAddress.len, FALSE);
4875 MD_RET0(rv);
4876 }
4877
4878 break;
4879 }
4880 case YAF_MYSQL_FLOW_TID:
4881 {
4882 yaf_mysql_t *mflow = (yaf_mysql_t *)flow->app;
4883 yaf_mysql_txt_t *mtxt = NULL;
4884 rv = exporter->VLprint_fn(exporter, mflow->mysqlUsername.buf,
4885 MYSQL_DEFAULT, index_str, index_len, 223,
4886 mflow->mysqlUsername.len, FALSE);
4887 MD_RET0(rv);
4888 while ((mtxt = (yaf_mysql_txt_t *)FBSTLNEXT(&(mflow->mysqlList),
4889 mtxt)))
4890 {
4891 rv = exporter->VLprint_fn(exporter, mtxt->mysqlCommandText.buf,
4892 MYSQL_DEFAULT, index_str, index_len, 225,
4893 mtxt->mysqlCommandText.len, FALSE);
4894 MD_RET0(rv);
4895 }
4896 break;
4897 }
4898 case YAF_DNS_FLOW_TID:
4899 {
4900 yaf_dns_t *dnsflow = (yaf_dns_t *)flow->app;
4901 yaf_dnsQR_t *dnsqrflow = NULL;
4902 char *label = DNS_DEFAULT;
4903 size_t buftest = 0;
4904 uint16_t uid;
4905
4906 if (exporter->json) {
4907 ret = snprintf(buf->cp, brem, "\"dnsRecord\":[");
4908 MD_CHECK_RET(buf, ret, brem);
4909 buftest = MD_REM_MSG(buf);
4910 }
4911
4912 while((dnsqrflow =(yaf_dnsQR_t *)FBSTLNEXT(&(dnsflow->dnsQRList),
4913 dnsqrflow)))
4914 {
4915 uid = dnsqrflow->dnsQRType > 51 ? 53 : dnsqrflow->dnsQRType;
4916
4917 if (exporter->dns_resp_only) {
4918 if (dnsqrflow->dnsQueryResponse == 0) continue;
4919 }
4920
4921 if (exporter->dpi_field_table) {
4922 if (!mdGetDPIItem(exporter->dpi_field_table,uid)) {
4923 continue;
4924 }
4925 }
4926 if (exporter->json) {
4927
4928 MD_APPEND_CHAR_CHECK(brem, buf, '{');
4929 if (!mdJsonifyDNSRecord(dnsqrflow, buf)) {
4930 return FALSE;
4931 }
4932 brem = MD_REM_MSG(buf);
4933 MD_APPEND_CHAR_CHECK(brem, buf, '}');
4934 MD_APPEND_CHAR_CHECK(brem, buf, ',');
4935
4936 } else if (exporter->multi_files) {
4937 FILE *fp = NULL;
4938 if (table_hash) {
4939 label = mdGetTableItem(uid);
4940 if (label == NULL) {
4941 continue;
4942 }
4943 }
4944
4945 if (!md_util_append_buffer(buf, &brem, (uint8_t*)index_str, index_len)) {
4946 return 0;
4947 }
4948
4949 if (!mdExporterTextDNSPrint(exporter, dnsqrflow)) {
4950 return 0;
4951 }
4952
4953 fp = mdGetTableFile(exporter, uid);
4954 if (fp == NULL) {
4955 g_warning("Error: File does not exist for DNS "
4956 "Type: %d", dnsqrflow->dnsQRType);
4957 continue;
4958 }
4959
4960 rc = md_util_write_buffer(fp, buf, exporter->name, err);
4961 if (!rc) {
4962 return -1;
4963 }
4964 exporter->exp_bytes += rc;
4965 } else {
4966 if (exporter->custom_list_dpi) {
4967 if (!md_util_append_buffer(buf, &brem, (uint8_t*)index_str, index_len)) {
4968 return 0;
4969 }
4970 } else {
4971 if (!exporter->no_index) {
4972 ret = snprintf(buf->cp, brem, "%s%c", label, delim);
4973 MD_CHECK_RET(buf, ret, brem);
4974 }
4975 if (index_str) {
4976 if (!md_util_append_buffer(buf, &brem, (uint8_t*)index_str, index_len)) {
4977 return 0;
4978 }
4979 }
4980 }
4981 if (!mdExporterTextDNSPrint(exporter, dnsqrflow)) {
4982 return 0;
4983 }
4984 }
4985 } /* record loop */
4986
4987 if (exporter->json) {
4988 brem = MD_REM_MSG(buf);
4989 if (buftest == brem) {
4990 buf->cp -= 13;
4991 brem += 13;
4992 } else {
4993 buf->cp -= 1;
4994 brem += 1;
4995 MD_APPEND_CHAR_CHECK(brem, buf, ']');
4996 MD_APPEND_CHAR_CHECK(brem, buf, ',');
4997 }
4998 }
4999
5000 } /* dns */
5001 break;
5002 case YAF_RTSP_FLOW_TID:
5003 bl = (fbBasicList_t *)flow->app;
5004 for (loop = 0; loop < flow->app_elements; loop++) {
5005 rv = exporter->BLprint_fn(exporter, bl, index_str, index_len, RTSP_DEFAULT,0);
5006 MD_RET0(rv);
5007 bl++;
5008 }
5009 break;
5010 case YAF_RTP_FLOW_TID:
5011 {
5012 yaf_rtp_t *rtp = (yaf_rtp_t *)flow->app;
5013 char rtp_buffer[20];
5014
5015 if (rtp) {
5016 snprintf(rtp_buffer, sizeof(rtp_buffer), "%d",
5017 rtp->rtpPayloadType);
5018 rv = exporter->VLprint_fn(exporter, (uint8_t *)rtp_buffer,
5019 RTP_DEFAULT, index_str, index_len, 287,
5020 strlen(rtp_buffer), FALSE);
5021 MD_RET0(rv);
5022 if (rtp->reverseRtpPayloadType) {
5023 snprintf(rtp_buffer, sizeof(rtp_buffer), "%d",
5024 rtp->reverseRtpPayloadType);
5025 rv = exporter->VLprint_fn(exporter, (uint8_t *)rtp_buffer,
5026 RTP_DEFAULT, index_str, index_len, 287,
5027 strlen(rtp_buffer), FALSE);
5028 MD_RET0(rv);
5029 }
5030 }
5031 }
5032 break;
5033 case YAF_DNP3_FLOW_TID:
5034 {
5035 yaf_dnp_t *dnp = (yaf_dnp_t *)flow->app;
5036 yaf_dnp_rec_t *rec = NULL;
5037 char dnp_buffer[65535];
5038 size_t buflen;
5039 size_t bufsz;
5040 int i;
5041
5042 if (dnp) {
5043 while ((rec = (yaf_dnp_rec_t *)FBSTLNEXT(&(dnp->dnp_list), rec)))
5044 {
5045 i = 0;
5046 bufsz = sizeof(dnp_buffer);
5047 if (!exporter->json) {
5048 i = snprintf(dnp_buffer, bufsz, "%d%c%d%c%d%c",
5049 rec->dnp3SourceAddress,
5050 delim, rec->dnp3DestinationAddress, delim, rec->dnp3Function,
5051 delim);
5052 }
5053 buflen = rec->dnp3ObjectData.len;
5054 bufsz -= i;
5055 if (buflen > bufsz) {
5056 buflen = bufsz;
5057 }
5058 i += md_util_hexdump_append(dnp_buffer + i,
5059 &bufsz, rec->dnp3ObjectData.buf, buflen);
5060 rv = exporter->VLprint_fn(exporter, (uint8_t *)dnp_buffer,
5061 DNP_DEFAULT, index_str, index_len, 284, i, FALSE);
5062 MD_RET0(rv);
5063 }
5064
5065 }
5066 }
5067 break;
5068 case YAF_MODBUS_FLOW_TID:
5069 rv = exporter->BLprint_fn(exporter, (fbBasicList_t *)flow->app,
5070 index_str, index_len, MODBUS_DEFAULT, TRUE);
5071 MD_RET0(rv);
5072 break;
5073 case YAF_ENIP_FLOW_TID:
5074 rv = exporter->BLprint_fn(exporter, (fbBasicList_t *)flow->app,
5075 index_str, index_len, ENIP_DEFAULT, TRUE);
5076 MD_RET0(rv);
5077 break;
5078 default:
5079 break;
5080 }
5081
5082 if (flow->fullcert) {
5083 yaf_newssl_cert_t *cert = NULL;
5084 size_t buftest;
5085 fbVarfield_t *ct = NULL;
5086 int i = 0;
5087 int cert_no = 0;
5088
5089 brem = MD_REM_MSG(buf);
5090
5091 if (exporter->json) {
5092 if (flow->sslcerts) {
5093 ret = snprintf(buf->cp, brem, "\"sslCertList\":[");
5094 MD_CHECK_RET(buf, ret, brem);
5095 }
5096 buftest = brem;
5097
5098 while ((cert = flow->sslcerts[i])) {
5099 if (!mdJsonifyNewSSLCertRecord(exporter, cert, cert_no)) {
5100 return 0;
5101 }
5102 ct = (fbVarfield_t *)FBBLNP(&(flow->fullcert->cert), ct);
5103 if (ct->len == 0) {
5104 continue;
5105 }
5106 if (exporter->md5_hash || exporter->sha1_hash ||
5107 mdExporterCheckSSLConfig(exporter, 299, 3) ||
5108 mdExporterCheckSSLConfig(exporter, 298, 3))
5109 {
5110 if (!mdExporterSSLCertHash(exporter, ct, NULL, 0, cert_no)) {
5111 return 0;
5112 }
5113 brem = MD_REM_MSG(buf);
5114 }
5115 if (mdExporterCheckSSLConfig(exporter, 296, 3)) {
5116 if (!mdJsonifySSLCertBase64(exporter->buf, ct)) {
5117 return FALSE;
5118 }
5119 }
5120 cert_no++;
5121 i++;
5122 } /* cert list loop */
5123
5124 brem = MD_REM_MSG(buf);
5125
5126 if (flow->sslcerts) {
5127 if (brem != buftest) {
5128 /* remove comma if sslCertList array is not empty*/
5129 buf->cp -= 1;
5130 brem += 1;
5131 }
5132 MD_APPEND_CHAR_CHECK(brem, exporter->buf, ']');
5133 MD_APPEND_CHAR_CHECK(brem, exporter->buf, ',');
5134 }
5135 } else {
5136 while ((cert = flow->sslcerts[i]))
5137 {
5138 if (!mdExporterTextNewSSLCertPrint(exporter, cert, index_str,
5139 index_len, cert_no)) {
5140 return 0;
5141 }
5142 if (exporter->md5_hash || exporter->sha1_hash ||
5143 mdExporterCheckSSLConfig(exporter, 299, 3) ||
5144 mdExporterCheckSSLConfig(exporter, 298, 3) ||
5145 mdExporterCheckSSLConfig(exporter, 296, 3))
5146 {
5147 ct = (fbVarfield_t *)FBBLNP(&(flow->fullcert->cert), ct);
5148 if (ct->len == 0) {
5149 continue;
5150 }
5151 if (!mdExporterSSLCertHash(exporter, ct, index_str, index_len, cert_no)) {
5152 return 0;
5153 }
5154 if (mdExporterCheckSSLConfig(exporter, 296, 3)) {
5155 if (!mdExporterSSLBase64Encode(exporter, ct, index_str, index_len, cert_no)) {
5156 return FALSE;
5157 }
5158 }
5159
5160 }
5161 cert_no++;
5162 i++;
5163 }
5164 }
5165 }
5166
5167 if (flow->stats && exporter->basic_list_dpi) {
5168 if (exporter->json) {
5169 rv = mdJsonizeFlowStats(exporter, flow->stats, index_str, index_len,
5170 rev);
5171 MD_RET0(rv);
5172 } else {
5173 rv = mdExportFlowStats(exporter, flow->stats, index_str, index_len,
5174 FLOW_STATS_DEFAULT, rev);
5175 MD_RET0(rv);
5176 }
5177 }
5178
5179
5180 if (flow->dhcpfp) {
5181 if ((flow->dhcpfp->tmplID & YTF_BIF) == YAF_DHCP_FLOW_TID) {
5182 yaf_dhcp_fp_t *dhcp = NULL;
5183 dhcp = (yaf_dhcp_fp_t *)fbSubTemplateMultiListEntryNextDataPtr(flow->dhcpfp, dhcp);
5184 rv = exporter->VLprint_fn(exporter, dhcp->dhcpFingerPrint.buf,
5185 DHCP_DEFAULT, index_str, index_len, 242,
5186 dhcp->dhcpFingerPrint.len, FALSE);
5187 MD_RET0(rv);
5188 rv = exporter->VLprint_fn(exporter, dhcp->dhcpVendorCode.buf,
5189 DHCP_DEFAULT, index_str, index_len, 243,
5190 dhcp->dhcpVendorCode.len, FALSE);
5191 MD_RET0(rv);
5192 if (flow->dhcpfp->tmplID & YTF_REV) {
5193 rv = exporter->VLprint_fn(exporter,
5194 dhcp->reverseDhcpFingerPrint.buf,
5195 DHCP_DEFAULT, index_str, index_len, 242,
5196 dhcp->reverseDhcpFingerPrint.len, FALSE);
5197 MD_RET0(rv);
5198 rv = exporter->VLprint_fn(exporter,
5199 dhcp->reverseDhcpVendorCode.buf,
5200 DHCP_DEFAULT, index_str, index_len, 243,
5201 dhcp->reverseDhcpVendorCode.len, FALSE);
5202 MD_RET0(rv);
5203 }
5204 } else if ((flow->dhcpfp->tmplID & YTF_BIF) == YAF_DHCP_OP_TID) {
5205 yaf_dhcp_options_t *dhcp = NULL;
5206 uint8_t *option;
5207 char dhcp_buffer[4096];
5208 size_t dhcpbuflen = 0;
5209 dhcp = (yaf_dhcp_options_t *)fbSubTemplateMultiListEntryNextDataPtr(flow->dhcpfp, dhcp);
5210 /* print options basiclist */
5211 for (loop = 0; (option =
5212 (uint8_t*)fbBasicListGetIndexedDataPtr(&(dhcp->options), loop));
5213 loop++)
5214 {
5215 dhcpbuflen += snprintf(dhcp_buffer+dhcpbuflen,
5216 sizeof(dhcp_buffer) - dhcpbuflen, "%d, ", *option);
5217 }
5218
5219 if (dhcpbuflen > 2) {
5220 if (exporter->json) {
5221 if (exporter->dpi_field_table) {
5222 if (!mdGetDPIItem(exporter->dpi_field_table, 297)) {
5223 /* fix me */
5224 return 0;
5225 }
5226 }
5227 ret = snprintf(buf->cp, brem,
5228 "\"dhcpOptionsList\":[");
5229 MD_CHECK_RET(buf, ret, brem);
5230 if (!md_util_append_buffer(buf, &brem, (uint8_t*)dhcp_buffer,
5231 dhcpbuflen-2)) {
5232 return 0;
5233 }
5234 MD_APPEND_CHAR(buf, ']');
5235 MD_APPEND_CHAR(buf, ',');
5236 } else {
5237 rv = exporter->VLprint_fn(exporter, (uint8_t*)dhcp_buffer,
5238 DHCP_DEFAULT, index_str, index_len, 297,
5239 dhcpbuflen-2, FALSE);
5240 MD_RET0(rv);
5241 }
5242 }
5243 rv = exporter->VLprint_fn(exporter, dhcp->dhcpVendorCode.buf,
5244 DHCP_DEFAULT, index_str, index_len, 243,
5245 dhcp->dhcpVendorCode.len, FALSE);
5246 MD_RET0(rv);
5247 if (flow->dhcpfp->tmplID & YTF_REV) {
5248 /*print reverse options basiclist */
5249 dhcpbuflen = 0;
5250 /* print options basiclist */
5251 for (loop = 0; (option =
5252 (uint8_t*)fbBasicListGetIndexedDataPtr(&(dhcp->revOptions), loop));
5253 loop++)
5254 {
5255 dhcpbuflen += snprintf(dhcp_buffer+dhcpbuflen,
5256 sizeof(dhcp_buffer) - dhcpbuflen, "%d, ", *option);
5257 }
5258
5259 if (dhcpbuflen > 2) {
5260 if (exporter->json) {
5261 if (exporter->dpi_field_table) {
5262 if (!mdGetDPIItem(exporter->dpi_field_table, 297)) {
5263 /* fix me */
5264 return 0;
5265 }
5266 }
5267 ret = snprintf(buf->cp, brem,
5268 "\"reverseDhcpOptionsList\":[");
5269 MD_CHECK_RET(buf, ret, brem);
5270 if (!md_util_append_buffer(buf, &brem, (uint8_t*)dhcp_buffer,
5271 dhcpbuflen-2)) {
5272 return 0;
5273 }
5274 MD_APPEND_CHAR(buf, ']');
5275 MD_APPEND_CHAR(buf, ',');
5276 } else {
5277 rv = exporter->VLprint_fn(exporter, (uint8_t*)dhcp_buffer,
5278 DHCP_DEFAULT, index_str, index_len, 297,
5279 dhcpbuflen-2, FALSE);
5280 MD_RET0(rv);
5281 }
5282 }
5283
5284 rv = exporter->VLprint_fn(exporter,
5285 dhcp->reverseDhcpVendorCode.buf,
5286 DHCP_DEFAULT, index_str, index_len, 243,
5287 dhcp->reverseDhcpVendorCode.len, FALSE);
5288 MD_RET0(rv);
5289 }
5290 }
5291
5292 }
5293
5294 brem = MD_REM_MSG(buf);
5295
5296 if (exporter->json) {
5297 buf->cp -= 1;
5298 brem++;
5299 if (brem > 3) {
5300 MD_APPEND_CHAR(buf, '}');
5301 MD_APPEND_CHAR(buf, '}');
5302 MD_APPEND_CHAR(buf, '\n');
5303 }
5304 }
5305
5306 /* this prints the index record for this flow */
5307 if (exporter->dpionly && !exporter->no_index) {
5308 if (!mdDPIIndex(exporter, flow, INDEX_DEFAULT)) {
5309 return 0;
5310 }
5311 }
5312
5313 /* only write if we have something.
5314 The exception is if we have a custom list.
5315 We want to print the data even if we don't have relevant DPI data
5316 since we don't print to the file in mdCustomFlowPrint. */
5317
5318 if (buf->cp > bufstart) {
5319
5320 rc = md_util_write_buffer(exporter->lfp, buf, exporter->name, err);
5321
5322 if (!rc) {
5323 return -1;
5324 }
5325
5326 exporter->exp_bytes += rc;
5327
5328 } else if (exporter->custom_list_dpi || exporter->basic_list_dpi) {
5329
5330 if (exporter->dpionly && (buf->cp == bufstart)) {
5331 /* didn't add any DPI and we only want dpi...*/
5332 return 1;
5333 } else if (buf->cp == bufstart) {
5334 /* if custom_list_dpi - add the index_str back on the buffer */
5335 /* this can happen if DPI_FIELD_LIST or SSL_CONFIG is configured */
5336 buf->cp += index_len;
5337 }
5338
5339 rc = md_util_write_buffer(exporter->lfp, buf, exporter->name, err);
5340
5341 if (!rc) {
5342 return -1;
5343 }
5344
5345 exporter->exp_bytes += rc;
5346 } else {
5347 rc = 1;
5348 }
5349
5350 return rc;
5351
5352 }
5353
5354 /**
5355 * mdExporterTextDNSPrint
5356 *
5357 * Returns DNS elements from DPI suitable for text output
5358 *
5359 */
mdExporterTextDNSPrint(mdFlowExporter_t * exporter,yaf_dnsQR_t * dns)5360 gboolean mdExporterTextDNSPrint(
5361 mdFlowExporter_t *exporter,
5362 yaf_dnsQR_t *dns)
5363 {
5364 char delim = exporter->dpi_delimiter;
5365 mdBuf_t *buf = exporter->buf;
5366 size_t brem = MD_REM_MSG(buf);
5367 int ret;
5368
5369 if (dns->dnsQueryResponse) {
5370 /* this is a response */
5371 ret = snprintf(buf->cp, brem, "R%c%d%c", delim, dns->dnsID, delim);
5372 } else {
5373 ret = snprintf(buf->cp, brem, "Q%c%d%c", delim, dns->dnsID, delim);
5374 }
5375
5376 MD_CHECK_RET(buf, ret, brem);
5377
5378
5379 ret = snprintf(buf->cp, brem, "%d%c%d%c", dns->dnsRRSection, delim,
5380 dns->dnsNXDomain, delim);
5381 MD_CHECK_RET(buf, ret, brem);
5382
5383 if (dns->dnsAuthoritative) {
5384 ret = snprintf(buf->cp, brem, "1%c", delim);
5385 } else {
5386 ret = snprintf(buf->cp, brem, "0%c", delim);
5387 }
5388
5389 MD_CHECK_RET(buf, ret, brem);
5390
5391 ret = snprintf(buf->cp, brem, "%d%c%u%c", dns->dnsQRType, delim,
5392 dns->dnsTTL, delim);
5393 MD_CHECK_RET(buf, ret, brem);
5394
5395 if (dns->dnsQName.buf) {
5396 if (!md_util_append_varfield(buf, &brem, &(dns->dnsQName))) {
5397 return FALSE;
5398 }
5399 } /* else - query may be for the root server which is NULL*/
5400
5401 if (dns->dnsQueryResponse == 0) {
5402 ret = snprintf(buf->cp, brem, "%c\n", delim);
5403 MD_CHECK_RET(buf, ret, brem);
5404 return TRUE;
5405 }
5406
5407 MD_APPEND_CHAR_CHECK(brem, buf, delim);
5408
5409 switch (dns->dnsQRType) {
5410 case 1:
5411 {
5412 yaf_dnsA_t *a = NULL;
5413 char ipaddr[20];
5414 while ((a = (yaf_dnsA_t *)FBSTLNEXT(&(dns->dnsRRList), a))) {
5415 if (a->sourceIPv4Address) {
5416 md_util_print_ip4_addr(ipaddr, a->sourceIPv4Address);
5417 ret = snprintf(buf->cp, brem, "%s", ipaddr);
5418 MD_CHECK_RET(buf, ret, brem);
5419 }
5420 }
5421
5422 MD_APPEND_CHAR(buf, '\n');
5423
5424 break;
5425 }
5426 case 2:
5427 {
5428 yaf_dnsNS_t *ns = NULL;
5429 while ((ns = (yaf_dnsNS_t *)FBSTLNEXT(&(dns->dnsRRList), ns))) {
5430
5431 mdPrintVariableLength(buf, &brem, ns->dnsNSDName.buf,
5432 ns->dnsNSDName.len, delim, 0,
5433 exporter->escape_chars);
5434 }
5435 MD_APPEND_CHAR(buf, '\n');
5436 break;
5437
5438 }
5439 case 5:
5440 {
5441 yaf_dns_CNAME_t *c = NULL;
5442 while ((c = (yaf_dns_CNAME_t *)FBSTLNEXT(&(dns->dnsRRList), c)))
5443 {
5444 mdPrintVariableLength(buf, &brem, c->dnsCName.buf,
5445 c->dnsCName.len, delim, 0,
5446 exporter->escape_chars);
5447 }
5448 MD_APPEND_CHAR(buf, '\n');
5449 break;
5450 }
5451 case 6:
5452 {
5453 yaf_dnsSOA_t *soa = NULL;
5454 while ((soa=(yaf_dnsSOA_t *)FBSTLNEXT(&(dns->dnsRRList), soa))) {
5455 mdPrintVariableLength(buf, &brem, soa->dnsSOAMName.buf,
5456 soa->dnsSOAMName.len, delim, 0,
5457 exporter->escape_chars);
5458
5459 /*g_string_append_len(str, soa->rname.buf,
5460 soa->rname.len);
5461 g_string_append_printf(str,
5462 "%u%c%u%c%u%c%u%c%u",
5463 soa->serial, delim,
5464 soa->refresh, delim,
5465 soa->retry, delim,
5466 soa->expire, delim,
5467 soa->minimum);*/
5468 }
5469 MD_APPEND_CHAR(buf, '\n');
5470 break;
5471 }
5472 case 12:
5473 {
5474 yaf_dnsPTR_t *ptr = NULL;
5475 while ((ptr = (yaf_dnsPTR_t *)FBSTLNEXT(&(dns->dnsRRList), ptr)))
5476 {
5477 mdPrintVariableLength(buf, &brem, ptr->dnsPTRDName.buf,
5478 ptr->dnsPTRDName.len, delim, 0,
5479 exporter->escape_chars);
5480 }
5481 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5482 break;
5483 }
5484 case 15:
5485 {
5486 yaf_dnsMX_t *mx = NULL;
5487 while (( mx = (yaf_dnsMX_t *)FBSTLNEXT(&(dns->dnsRRList), mx)))
5488 {
5489 mdPrintVariableLength(buf, &brem, mx->dnsMXExchange.buf,
5490 mx->dnsMXExchange.len, delim, 0,
5491 exporter->escape_chars);
5492 }
5493 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5494 break;
5495
5496 }
5497 case 16:
5498 {
5499 yaf_dnsTXT_t *txt = NULL;
5500 while ((txt = (yaf_dnsTXT_t *)FBSTLNEXT(&(dns->dnsRRList), txt)))
5501 {
5502
5503 mdPrintVariableLength(buf, &brem, txt->dnsTXTData.buf,
5504 txt->dnsTXTData.len,delim, 0,
5505 exporter->escape_chars);
5506 }
5507 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5508 break;
5509 }
5510 case 28:
5511 {
5512 yaf_dnsAAAA_t *aa = NULL;
5513 char ipaddr[40];
5514 while ((aa = (yaf_dnsAAAA_t *)FBSTLNEXT(&(dns->dnsRRList), aa)))
5515 {
5516 md_util_print_ip6_addr(ipaddr,(uint8_t *)&(aa->sourceIPv6Address));
5517 ret = snprintf(buf->cp, brem, "%s", ipaddr);
5518 MD_CHECK_RET(buf, ret, brem);
5519 }
5520 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5521 break;
5522 }
5523 case 33:
5524 {
5525 yaf_dnsSRV_t *srv = NULL;
5526 while ((srv = (yaf_dnsSRV_t *)FBSTLNEXT(&(dns->dnsRRList), srv)))
5527 {
5528 mdPrintVariableLength(buf, &brem, srv->dnsSRVTarget.buf,
5529 srv->dnsSRVTarget.len, delim, 0,
5530 exporter->escape_chars);
5531
5532 /*g_string_append_printf(str, "%c%d%c%d%c%d",
5533 delim, srv->dnsPriority,
5534 delim, srv->dnsWeight,
5535 delim, srv->dnsPort);*/
5536 }
5537 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5538 break;
5539 }
5540 case 46:
5541 {
5542 yaf_dnsRRSig_t *rr = NULL;
5543 while ((rr =(yaf_dnsRRSig_t *)FBSTLNEXT(&(dns->dnsRRList), rr))){
5544 mdPrintVariableLength(buf, &brem, rr->dnsSigner.buf,
5545 rr->dnsSigner.len, delim, 0,
5546 exporter->escape_chars);
5547 }
5548 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5549 break;
5550 }
5551 case 47:
5552 {
5553 yaf_dnsNSEC_t *nsec = NULL;
5554 while ((nsec = (yaf_dnsNSEC_t *)FBSTLNEXT(&(dns->dnsRRList), nsec))) {
5555 mdPrintVariableLength(buf, &brem, nsec->dnsHashData.buf,
5556 nsec->dnsHashData.len,delim, 0,
5557 exporter->escape_chars);
5558 }
5559 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5560 break;
5561 }
5562 default:
5563 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
5564 }
5565
5566 return TRUE;
5567
5568 }
5569
mdExporterCheckSSLConfig(mdFlowExporter_t * exporter,int obj_id,uint8_t type)5570 static gboolean mdExporterCheckSSLConfig(
5571 mdFlowExporter_t *exporter,
5572 int obj_id,
5573 uint8_t type)
5574 {
5575
5576 if (exporter->ssl_config) {
5577 if (type == 1) {
5578 if (exporter->ssl_config->issuer) {
5579 if (exporter->ssl_config->issuer[obj_id] == 1) {
5580 return TRUE;
5581 }
5582 }
5583 } else if (type == 2) {
5584 if (exporter->ssl_config->subject) {
5585 if (exporter->ssl_config->subject[obj_id] == 1) {
5586 return TRUE;
5587 }
5588 }
5589 } else if (type == 3) {
5590 if (exporter->ssl_config->other) {
5591 if (exporter->ssl_config->other[obj_id] == 1) {
5592 return TRUE;
5593 }
5594 }
5595 } else if (type == 4) {
5596 if (exporter->ssl_config->extensions) {
5597 if (exporter->ssl_config->extensions[obj_id] == 1) {
5598 return TRUE;
5599 }
5600 }
5601 }
5602
5603 return FALSE;
5604 }
5605
5606 /* extensions & SHA1/MD5 have to be enabled manually! */
5607 if (type == 4 || ((obj_id == 298) || (obj_id == 299))) {
5608 return FALSE;
5609 }
5610
5611 return TRUE;
5612 }
5613
5614
mdJsonifyNewSSLCertRecord(mdFlowExporter_t * exporter,yaf_newssl_cert_t * cert,uint8_t cert_no)5615 static gboolean mdJsonifyNewSSLCertRecord(
5616 mdFlowExporter_t *exporter,
5617 yaf_newssl_cert_t *cert,
5618 uint8_t cert_no)
5619 {
5620
5621 GString *ou_str = g_string_new("\"sslCertIssuerOrgUnitName\":[");
5622 mdBuf_t *buf = exporter->buf;
5623 size_t brem = MD_REM_MSG(buf);
5624 int ret;
5625 yaf_subssl_t *obj = NULL;
5626 char ssl_buffer[4096];
5627 size_t buflen;
5628 gboolean rv = TRUE;
5629
5630 /* print issuer label and opening delimiter, only if issuer list
5631 * exists.
5632 */
5633 if (fbSubTemplateListGetIndexedDataPtr(&cert->issuer, 0)) {
5634 ret = snprintf(buf->cp, brem, "{\"sslCertIssuer\":{");
5635 MD_CHECK_RET(buf, ret, brem);
5636 }
5637
5638 while ((obj = (yaf_subssl_t *)FBSTLNEXT(&(cert->issuer),
5639 obj)))
5640 {
5641 if (!mdExporterCheckSSLConfig(exporter, obj->sslObjectType, 1)) {
5642 continue;
5643 }
5644
5645 if (obj->sslObjectValue.len) {
5646
5647 switch (obj->sslObjectType) {
5648
5649 case 3: /* Common Name */
5650
5651 ret = snprintf(buf->cp, brem,"\"sslCertIssuerCommonName\":\"");
5652 break;
5653
5654 case 6: /* Country Name */
5655
5656 ret = snprintf(buf->cp, brem,
5657 "\"sslCertIssuerCountryName\":\"");
5658 break;
5659
5660 case 7: /* Locality Name */
5661
5662 ret = snprintf(buf->cp, brem,
5663 "\"sslCertIssuerLocalityName\":\"");
5664 break;
5665
5666 case 8: /* State Name */
5667
5668 ret = snprintf(buf->cp, brem, "\"sslCertIssuerState\":\"");
5669 break;
5670
5671 case 9: /* Street Address */
5672
5673 ret = snprintf(buf->cp, brem,
5674 "\"sslCertIssuerStreetAddress\":\"");
5675 break;
5676
5677 case 10: /* Organization Name */
5678
5679 ret = snprintf(buf->cp, brem,
5680 "\"sslCertIssuerOrgName\":\"");
5681 break;
5682
5683 case 11: /* Organization Unit Name -- note, multi! */
5684 g_string_append_c(ou_str, '\"');
5685 g_string_append_len(ou_str, (char *)obj->sslObjectValue.buf,
5686 obj->sslObjectValue.len);
5687 g_string_append_printf(ou_str, "\",");
5688 continue;
5689 case 17: /* Zip Code */
5690 ret = snprintf(buf->cp, brem, "\"sslCertIssuerZipCode\":\"");
5691 break;
5692 default: /* We don't know */
5693 ret = snprintf(buf->cp, brem, "\"sslObjectID%d\":\"",
5694 obj->sslObjectType);
5695 } /* switch */
5696
5697 MD_CHECK_RET(buf, ret, brem);
5698
5699 if (!md_util_append_varfield(buf, &brem, &(obj->sslObjectValue))) {
5700 return FALSE;
5701 }
5702
5703 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
5704 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5705 } /* if obj valid */
5706 } /* issuer loop */
5707
5708 /* print OU string if configured, and closing delimiters if issuer
5709 * exists.
5710 */
5711 if (fbSubTemplateListGetIndexedDataPtr(&cert->issuer, 0)) {
5712
5713 if (mdExporterCheckSSLConfig(exporter, 11, 1)) {
5714 if (ou_str->str[ou_str->len-1] == ',') {
5715 g_string_truncate(ou_str, ou_str->len-1);
5716 }
5717 g_string_append(ou_str, "]");
5718 if (!md_util_append_gstr(buf, &brem, ou_str)) {
5719 return FALSE;
5720 }
5721 } else {
5722 /* we need to snip the trailing comma since we aren't appending
5723 * the ou string
5724 */
5725 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
5726 buf->cp -= 1;
5727 brem += 1;
5728 }
5729 }
5730 MD_APPEND_CHAR_CHECK(brem, buf, '}');
5731 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5732 }
5733
5734 /* print subject key and opening delimiter, only if subject lists
5735 * exists.
5736 */
5737 if (fbSubTemplateListGetIndexedDataPtr(&cert->subject, 0)) {
5738 ret = snprintf(buf->cp, brem, "\"sslCertSubject\":{");
5739 MD_CHECK_RET(buf, ret, brem);
5740 }
5741
5742 g_string_truncate(ou_str, 0);
5743 g_string_append(ou_str, "\"sslCertSubjectOrgUnitName\":[");
5744 obj = NULL;
5745
5746 while ((obj = (yaf_subssl_t *)FBSTLNEXT(&(cert->subject),
5747 obj)))
5748 {
5749 if (!mdExporterCheckSSLConfig(exporter, obj->sslObjectType, 2)) {
5750 continue;
5751 }
5752
5753 if (obj->sslObjectValue.len) {
5754 switch (obj->sslObjectType) {
5755
5756 case 3: /* Common Name */
5757
5758 ret = snprintf(buf->cp, brem,
5759 "\"sslCertSubCommonName\":\"");
5760 break;
5761
5762 case 6: /* Country Name */
5763
5764 ret = snprintf(buf->cp, brem,
5765 "\"sslCertSubCountryName\":\"");
5766 break;
5767
5768 case 7: /* Locality Name */
5769
5770 ret = snprintf(buf->cp, brem,
5771 "\"sslCertSubLocalityName\":\"");
5772 break;
5773
5774 case 8: /* State Name */
5775
5776 ret = snprintf(buf->cp, brem,
5777 "\"sslCertSubState\":\"");
5778 break;
5779
5780 case 9: /* Street Address */
5781
5782 ret = snprintf(buf->cp, brem,
5783 "\"sslCertSubStreetAddress\":\"");
5784 break;
5785
5786 case 10: /* Organization Name */
5787
5788 ret = snprintf(buf->cp, brem,
5789 "\"sslCertSubOrgName\":\"");
5790 break;
5791
5792 case 11: /* Organization Unit Name -- note, multi! */
5793 g_string_append_c(ou_str, '\"');
5794 g_string_append_len(ou_str, (char *)obj->sslObjectValue.buf,
5795 obj->sslObjectValue.len);
5796 g_string_append_printf(ou_str, "\",");
5797 continue;
5798 case 17: /* Zip Code */
5799 ret = snprintf(buf->cp, brem,
5800 "\"sslCertSubZipCode\":\"");
5801 break;
5802
5803 default: /* We don't know */
5804 ret = snprintf(buf->cp, brem, "\"sslObjectID%d\":\"",
5805 obj->sslObjectType);
5806 } /* switch */
5807
5808 MD_CHECK_RET(buf, ret, brem);
5809
5810 if (!md_util_append_varfield(buf, &brem, &(obj->sslObjectValue))) {
5811 return FALSE;
5812 }
5813 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
5814 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5815 } /* if obj valid */
5816 } /* cert subject loop */
5817
5818 /* append OU string if configured, and closing delimiters if subject
5819 * exists
5820 */
5821
5822 if (fbSubTemplateListGetIndexedDataPtr(&cert->subject, 0)) {
5823
5824 if (mdExporterCheckSSLConfig(exporter, 11, 2)) {
5825
5826 if (ou_str->str[ou_str->len-1] == ',') {
5827 g_string_truncate(ou_str, ou_str->len-1);
5828 }
5829 g_string_append(ou_str, "]");
5830 if (!md_util_append_gstr(buf, &brem, ou_str)) {
5831 return FALSE;
5832 }
5833 } else {
5834 /* we need to snip the trailing comma since we aren't appending
5835 * the ou string
5836 */
5837 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
5838 buf->cp -= 1;
5839 brem += 1;
5840 }
5841 }
5842
5843 MD_APPEND_CHAR_CHECK(brem, buf, '}');
5844 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5845 }
5846
5847 /* print cert version */
5848
5849 if (mdExporterCheckSSLConfig(exporter, 189, 3)) {
5850 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d", cert->sslCertVersion);
5851 rv = exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
5852 SSL_DEFAULT, NULL, 0, 189, strlen(ssl_buffer),
5853 FALSE);
5854 MD_RET0(rv);
5855 }
5856
5857 if (cert->sslCertSerialNumber.len && (mdExporterCheckSSLConfig(exporter, 244, 3))) {
5858 size_t bufsz = sizeof(ssl_buffer);
5859 buflen = cert->sslCertSerialNumber.len > bufsz ? bufsz : cert->sslCertSerialNumber.len;
5860 ret = md_util_hexdump_append(ssl_buffer, &bufsz,
5861 cert->sslCertSerialNumber.buf, buflen);
5862 rv = exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
5863 SSL_DEFAULT, NULL, 0, 244, ret, FALSE);
5864 MD_RET0(rv);
5865 }
5866 if (cert->sslCertValidityNotBefore.len &&
5867 (mdExporterCheckSSLConfig(exporter, 247, 3)))
5868 {
5869 rv = exporter->VLprint_fn(exporter, cert->sslCertValidityNotBefore.buf,
5870 SSL_DEFAULT, NULL, 0, 247,
5871 cert->sslCertValidityNotBefore.len, FALSE);
5872 MD_RET0(rv);
5873 }
5874 if (cert->sslCertValidityNotAfter.len && (mdExporterCheckSSLConfig(exporter, 248, 3)))
5875 {
5876 rv = exporter->VLprint_fn(exporter, cert->sslCertValidityNotAfter.buf,
5877 SSL_DEFAULT, NULL, 0, 248,
5878 cert->sslCertValidityNotAfter.len, FALSE);
5879 MD_RET0(rv);
5880 }
5881
5882 if (cert->sslPublicKeyLength && (mdExporterCheckSSLConfig(exporter, 250, 3))) {
5883 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d", cert->sslPublicKeyLength);
5884 rv = exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
5885 SSL_DEFAULT, NULL, 0, 250,
5886 strlen(ssl_buffer), FALSE);
5887 MD_RET0(rv);
5888 }
5889
5890 /* this element was added in yaf 2.8 */
5891 if (cert->sslCertificateHash.len && (mdExporterCheckSSLConfig(exporter, 295, 3)))
5892 {
5893 size_t bufsz = sizeof(ssl_buffer);
5894 buflen = cert->sslCertificateHash.len > bufsz ? bufsz : cert->sslCertificateHash.len;
5895 ret = md_util_hexdump_append(ssl_buffer, &bufsz,
5896 cert->sslCertificateHash.buf, buflen);
5897 rv = exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
5898 SSL_DEFAULT, NULL, 0, 295,
5899 ret, FALSE);
5900 MD_RET0(rv);
5901 }
5902
5903 brem = MD_REM_MSG(buf);
5904
5905 /* print extension and opening delimiter, only if ext lists
5906 * exists.
5907 */
5908 if (fbSubTemplateListGetIndexedDataPtr(&cert->extension, 0)) {
5909 ret = snprintf(buf->cp, brem, "\"sslExtensions\":{");
5910 MD_CHECK_RET(buf, ret, brem);
5911 }
5912 obj = NULL;
5913 while ((obj = (yaf_subssl_t *)FBSTLNEXT(&(cert->extension),
5914 obj)))
5915 {
5916 if (!mdExporterCheckSSLConfig(exporter, obj->sslObjectType, 4)) {
5917 continue;
5918 }
5919
5920 if (obj->sslObjectValue.len) {
5921 switch (obj->sslObjectType) {
5922 case 14:
5923 {
5924 ret = snprintf(buf->cp, brem,
5925 "\"sslSubjectKeyIdentifier\":\"");
5926 MD_CHECK_RET(buf, ret, brem);
5927 ret = md_util_hexdump_append(buf->cp, &brem, obj->sslObjectValue.buf,
5928 obj->sslObjectValue.len);
5929 if (!ret) return FALSE;
5930 buf->cp += ret;
5931 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
5932 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5933 }
5934 break;
5935 case 15:
5936 {
5937 ret = snprintf(buf->cp, brem, "\"sslKeyUsage\":\"");
5938 MD_CHECK_RET(buf, ret, brem);
5939 ret = md_util_hexdump_append(buf->cp, &brem, obj->sslObjectValue.buf,
5940 obj->sslObjectValue.len);
5941 if (!ret) return FALSE;
5942 buf->cp += ret;
5943 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
5944 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5945 }
5946 break;
5947 case 16:
5948 {
5949 ret = snprintf(buf->cp, brem,
5950 "\"sslPrivateKeyUsagePeriod\":\"");
5951 MD_CHECK_RET(buf, ret, brem);
5952 ret = md_util_hexdump_append(buf->cp, &brem, obj->sslObjectValue.buf,
5953 obj->sslObjectValue.len);
5954 if (!ret) return FALSE;
5955 buf->cp += ret;
5956 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
5957 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5958 }
5959 break;
5960 case 17:
5961 /* subject/issuer alt name can be a list */
5962 {
5963 uint8_t *buffer = obj->sslObjectValue.buf;
5964 size_t len = obj->sslObjectValue.len;
5965 uint16_t newlen;
5966 ret = snprintf(buf->cp, brem, "\"sslSubjectAltName\":[");
5967 MD_CHECK_RET(buf, ret, brem);
5968 while ((newlen = md_util_decode_asn1_sequence(&buffer,
5969 &len)))
5970 {
5971 if (*buffer == 0x30) {
5972 /* this is a sequence - ignore */
5973 break;
5974 }
5975 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
5976 if (!md_util_append_buffer(buf, &brem, buffer, newlen))
5977 {
5978 return FALSE;
5979 }
5980 buffer += newlen;
5981 len -= newlen;
5982
5983 ret = snprintf(buf->cp, brem, "\",");
5984 MD_CHECK_RET(buf, ret, brem);
5985 }
5986 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
5987 buf->cp -= 1;
5988 brem += 1;
5989 }
5990 MD_APPEND_CHAR_CHECK(brem, buf, ']');
5991 MD_APPEND_CHAR_CHECK(brem, buf, ',');
5992 }
5993 break;
5994 case 18:
5995 /* subject/issuer alt name can be a list */
5996 {
5997 uint8_t *buffer = obj->sslObjectValue.buf;
5998 size_t len = obj->sslObjectValue.len;
5999 uint16_t newlen;
6000 ret = snprintf(buf->cp, brem, "\"sslIssuerAltName\":[");
6001 MD_CHECK_RET(buf, ret, brem);
6002 while ((newlen = md_util_decode_asn1_sequence(&buffer,
6003 &len)))
6004 {
6005 if (*buffer == 0x30) {
6006 /* this is a sequence - ignore */
6007 break;
6008 }
6009 MD_APPEND_CHAR_CHECK(brem, buf,'\"');
6010 if (!md_util_append_buffer(buf,&brem, buffer, newlen))
6011 {
6012 return FALSE;
6013 }
6014 buffer += newlen;
6015 len -= newlen;
6016 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
6017 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6018 }
6019 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
6020 buf->cp -= 1;
6021 brem += 1;
6022 }
6023 MD_APPEND_CHAR_CHECK(brem, buf, ']');
6024 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6025 }
6026 break;
6027 case 29:
6028 /* subject/issuer alt name can be a list */
6029 {
6030 uint8_t *buffer = obj->sslObjectValue.buf;
6031 size_t len = obj->sslObjectValue.len;
6032 uint16_t newlen;
6033 ret = snprintf(buf->cp, brem,"\"sslCertificateIssuer\":[");
6034 MD_CHECK_RET(buf, ret, brem);
6035 while ((newlen = md_util_decode_asn1_sequence(&buffer,
6036 &len)))
6037 {
6038 if (*buffer == 0x30) {
6039 /* this is a sequence - ignore */
6040 break;
6041 }
6042 MD_APPEND_CHAR_CHECK(brem, buf,'\"');
6043 if (!md_util_append_buffer(buf,&brem, buffer, newlen))
6044 {
6045 return FALSE;
6046 }
6047 buffer += newlen;
6048 len -= newlen;
6049 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
6050 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6051 }
6052 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
6053 buf->cp -= 1;
6054 brem += 1;
6055 }
6056 MD_APPEND_CHAR_CHECK(brem, buf, ']');
6057 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6058 }
6059 break;
6060 case 31:
6061 {
6062 uint8_t *buffer = obj->sslObjectValue.buf;
6063 size_t len = obj->sslObjectValue.len;
6064 uint16_t newlen;
6065 gboolean a;
6066 ret = snprintf(buf->cp, brem,
6067 "\"sslCRLDistributionPoints\":[");
6068 MD_CHECK_RET(buf, ret, brem);
6069 while ((newlen = md_util_decode_asn1_sequence(&buffer, &len))) {
6070 a = FALSE;
6071 while (*buffer == 0xa0) {
6072 buffer++;
6073 len -= 1;
6074 md_util_decode_asn1_length(&buffer, &len);
6075 a = TRUE;
6076 }
6077 if (a) continue; /* start over */
6078 MD_APPEND_CHAR_CHECK(brem, buf,'\"');
6079 if (!md_util_append_buffer(buf,&brem, buffer, newlen))
6080 {
6081 return FALSE;
6082 }
6083 buffer += newlen;
6084 len -= newlen;
6085 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
6086 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6087 }
6088 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
6089 buf->cp -= 1;
6090 brem += 1;
6091 }
6092 MD_APPEND_CHAR_CHECK(brem, buf, ']');
6093 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6094 }
6095 break;
6096 case 32:
6097 {
6098 uint8_t *buffer = obj->sslObjectValue.buf;
6099 size_t len = obj->sslObjectValue.len;
6100 uint16_t newlen;
6101 newlen = md_util_decode_asn1_sequence(&buffer, &len);
6102 if (*buffer == 0x06) {
6103 /* OID */
6104 buffer++;
6105 newlen = (uint16_t)*buffer;
6106 buffer++;
6107 ret = snprintf(buf->cp, brem,
6108 "\"sslCertificatePolicyID\":\"");
6109 MD_CHECK_RET(buf, ret, brem);
6110 /* subject key identifier - just an octet string*/
6111 ret = md_util_hexdump_append(buf->cp, &brem, buffer,
6112 newlen);
6113 if (!ret) return FALSE;
6114 buf->cp += ret;
6115 buffer += newlen;
6116 len -= newlen;
6117 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
6118 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6119 }
6120 /* now to a sequqnece {policyQualifierID, qualifier} */
6121 if (*buffer == 0x30) {
6122 /* string */
6123 len = len - newlen - 2;
6124 newlen = md_util_decode_asn1_sequence(&buffer, &len);
6125 if (*buffer == 0x06) {
6126 /* OID */
6127 buffer++;
6128 newlen = (uint16_t)*buffer;
6129 buffer += newlen + 1;
6130 if (*buffer == 0x16) {
6131 buffer++;
6132 newlen = (uint16_t)*buffer;
6133 buffer++;
6134
6135 ret = snprintf(buf->cp, brem,
6136 "\"sslCertificatePolicy\":\"");
6137 MD_CHECK_RET(buf, ret, brem);
6138 if (!md_util_append_buffer(buf, &brem, buffer, newlen))
6139 {
6140 return FALSE;
6141 }
6142 MD_APPEND_CHAR_CHECK(brem, buf, '\"');
6143 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6144 }
6145 }
6146 }
6147 }
6148 break;
6149 default:
6150 break;
6151 }
6152 }
6153 }
6154
6155 if (fbSubTemplateListGetIndexedDataPtr(&cert->extension, 0)) {
6156 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
6157 buf->cp -= 1;
6158 brem += 1;
6159 }
6160 /* close extensions... */
6161 MD_APPEND_CHAR_CHECK(brem, buf, '}');
6162 }
6163
6164 if (buf->buf[MD_MSG_LEN(buf)-1] == ',') {
6165 buf->cp -= 1;
6166 brem += 1;
6167 }
6168
6169 /* close this certificate in the chain */
6170 MD_APPEND_CHAR_CHECK(brem, buf, '}');
6171 MD_APPEND_CHAR_CHECK(brem, buf, ',');
6172
6173 g_string_free(ou_str, TRUE);
6174 return TRUE;
6175 }
6176
6177
6178
mdJsonifyNewSSLRecord(mdFlowExporter_t * exporter,yaf_newssl_t * sslflow,gboolean hex,gboolean escape)6179 gboolean mdJsonifyNewSSLRecord(
6180 mdFlowExporter_t *exporter,
6181 yaf_newssl_t *sslflow,
6182 gboolean hex,
6183 gboolean escape)
6184 {
6185
6186 yaf_newssl_cert_t *cert = NULL;
6187 fbSubTemplateList_t *stl = &(sslflow->sslCertList);
6188 char ssl_buffer[500];
6189 int cert_no = 0, ret;
6190 size_t brem = MD_REM_MSG(exporter->buf);
6191
6192 if (!sslflow) {
6193 return TRUE;
6194 }
6195
6196 if (fbSubTemplateListGetIndexedDataPtr(stl, 0)) {
6197 ret = snprintf(exporter->buf->cp, brem, "\"sslCertList\":[");
6198 MD_CHECK_RET(exporter->buf, ret, brem);
6199 }
6200
6201 while ((cert = (yaf_newssl_cert_t *)FBSTLNEXT(stl, cert)))
6202 {
6203 if(!mdJsonifyNewSSLCertRecord(exporter, cert, cert_no)) {
6204 return FALSE;
6205 }
6206 cert_no++;
6207 } /* cert list loop */
6208
6209 brem = MD_REM_MSG(exporter->buf);
6210
6211 if (fbSubTemplateListGetIndexedDataPtr(stl, 0)) {
6212 exporter->buf->cp -= 1;
6213 brem += 1;
6214 MD_APPEND_CHAR_CHECK(brem, exporter->buf, ']');
6215 MD_APPEND_CHAR_CHECK(brem, exporter->buf, ',');
6216 }
6217
6218 if (sslflow->sslServerCipher &&
6219 (mdExporterCheckSSLConfig(exporter, 187, 3)))
6220 {
6221 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d",
6222 sslflow->sslServerCipher);
6223 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6224 SSL_DEFAULT, NULL, 0, 187,
6225 strlen(ssl_buffer), FALSE);
6226 }
6227
6228 if (sslflow->sslCompressionMethod &&
6229 (mdExporterCheckSSLConfig(exporter, 188, 3)))
6230 {
6231 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d",
6232 sslflow->sslCompressionMethod);
6233 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6234 SSL_DEFAULT, NULL, 0, 188,
6235 strlen(ssl_buffer), FALSE);
6236 }
6237
6238 if (sslflow->sslClientVersion &&
6239 (mdExporterCheckSSLConfig(exporter, 186, 3)))
6240 {
6241 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d",
6242 sslflow->sslClientVersion);
6243 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6244 SSL_DEFAULT, NULL, 0, 186,
6245 strlen(ssl_buffer), FALSE);
6246 }
6247
6248 if (sslflow->sslRecordVersion &&
6249 (mdExporterCheckSSLConfig(exporter, 288, 3)))
6250 {
6251 snprintf(ssl_buffer, sizeof(ssl_buffer), "%d",
6252 sslflow->sslRecordVersion);
6253 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6254 SSL_DEFAULT, NULL, 0, 288,
6255 strlen(ssl_buffer), FALSE);
6256 }
6257
6258 /* this element was added in yaf 2.8 */
6259 if (sslflow->sslServerName.buf &&
6260 (mdExporterCheckSSLConfig(exporter, 294, 3)))
6261 {
6262 exporter->VLprint_fn(exporter, sslflow->sslServerName.buf,
6263 SSL_DEFAULT, NULL, 0, 294,
6264 sslflow->sslServerName.len, FALSE);
6265 }
6266
6267 return TRUE;
6268 }
6269
6270
mdExporterTextNewSSLCertObjectPrint(mdFlowExporter_t * exporter,yaf_subssl_t * obj,char * index_str,size_t index_len,uint8_t section,uint8_t cert_no,char ise,char delim)6271 static gboolean mdExporterTextNewSSLCertObjectPrint(
6272 mdFlowExporter_t *exporter,
6273 yaf_subssl_t *obj,
6274 char *index_str,
6275 size_t index_len,
6276 uint8_t section,
6277 uint8_t cert_no,
6278 char ise,
6279 char delim)
6280 {
6281
6282 char *label = SSL_DEFAULT;
6283 mdBuf_t *buf = exporter->buf;
6284 size_t brem = MD_REM_MSG(buf);
6285 int ret;
6286
6287 if (!mdExporterCheckSSLConfig(exporter, obj->sslObjectType, section)) {
6288 return FALSE;
6289 }
6290
6291 if (obj->sslObjectValue.len == 0) {
6292 return FALSE;
6293 }
6294
6295 if (!exporter->no_index && !exporter->multi_files) {
6296 /* print label */
6297 ret = snprintf(buf->cp, brem, "%s%c", label, delim);
6298 MD_CHECK_RET(buf, ret, brem);
6299 }
6300
6301 if (!md_util_append_buffer(buf, &brem, (uint8_t*)index_str, index_len)) {
6302 g_warning("Error %s: error appending index (%zu) to buffer (%zu)",
6303 exporter->name, index_len, brem);
6304 return FALSE;
6305 }
6306
6307 ret = snprintf(buf->cp, brem, "%d%c%c%c%d%c", obj->sslObjectType, delim, ise,
6308 delim, cert_no, delim);
6309
6310 MD_CHECK_RET(buf, ret, brem);
6311
6312 if (section == 4) {
6313 return TRUE;
6314 }
6315
6316 if (exporter->escape_chars) {
6317 if (!mdPrintEscapeChars(buf, &brem, obj->sslObjectValue.buf,
6318 obj->sslObjectValue.len, delim)) {
6319 g_warning("Error %s: error appending escape buf (%zu) to buffer "
6320 "(%zu)", exporter->name, obj->sslObjectValue.len, brem);
6321 return FALSE;
6322 }
6323 } else {
6324 if (!md_util_append_varfield(buf, &brem, &(obj->sslObjectValue))) {
6325 g_warning("Error %s: error appending varfield (%zu) to buffer "
6326 "(%zu)", exporter->name, obj->sslObjectValue.len, brem);
6327 return FALSE;
6328 }
6329 }
6330
6331 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
6332
6333 return TRUE;
6334 }
6335
mdExporterTextNewSSLCertPrint(mdFlowExporter_t * exporter,yaf_newssl_cert_t * cert,char * index_str,size_t index_len,uint8_t cert_no)6336 static gboolean mdExporterTextNewSSLCertPrint(
6337 mdFlowExporter_t *exporter,
6338 yaf_newssl_cert_t *cert,
6339 char *index_str,
6340 size_t index_len,
6341 uint8_t cert_no)
6342 {
6343
6344 yaf_subssl_t *obj = NULL;
6345 char delim = exporter->dpi_delimiter;
6346 char ssl_buffer[2500];
6347 char new_index[500];
6348 size_t buflen, afterlen, brem;
6349 char *bufstart = NULL;
6350 mdBuf_t *buf = exporter->buf;
6351 int ret;
6352
6353 while ((obj = (yaf_subssl_t *)FBSTLNEXT(&(cert->issuer),
6354 obj)))
6355 {
6356 mdExporterTextNewSSLCertObjectPrint(exporter, obj, index_str,
6357 index_len, 1, cert_no, 'I',
6358 delim);
6359 }
6360
6361 obj = NULL;
6362 while ((obj = (yaf_subssl_t *)FBSTLNEXT(&(cert->subject),
6363 obj)))
6364 {
6365 mdExporterTextNewSSLCertObjectPrint(exporter, obj, index_str,
6366 index_len, 2, cert_no, 'S',
6367 delim);
6368 }
6369
6370 obj = NULL;
6371
6372 /* Extensions have to be manually set in the SSL_CONFIG -
6373 they will not print in any default configuration */
6374 while ((obj = (yaf_subssl_t *)FBSTLNEXT(&(cert->extension), obj))) {
6375
6376 bufstart = buf->cp;
6377 if (!mdExporterTextNewSSLCertObjectPrint(exporter, obj,
6378 index_str, index_len, 4,
6379 cert_no, 'E', delim))
6380 {
6381 continue;
6382 }
6383 afterlen = buf->cp - bufstart;
6384 if (afterlen < sizeof(new_index)) {
6385 memcpy(new_index, bufstart, afterlen);
6386 } else {
6387 memcpy(new_index, bufstart, sizeof(new_index));
6388 }
6389 buf->cp = bufstart;
6390 if (obj->sslObjectValue.len) {
6391 switch (obj->sslObjectType) {
6392 case 14:
6393 case 15:
6394 case 16:
6395 /* push buffer up */
6396 buf->cp += afterlen;
6397 /* subject key identifier - just an octet string*/
6398 brem = MD_REM_MSG(buf);
6399 ret = md_util_hexdump_append(buf->cp, &brem,
6400 obj->sslObjectValue.buf,
6401 obj->sslObjectValue.len);
6402 if (!ret) return FALSE;
6403 buf->cp += ret;
6404 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
6405 continue;
6406 case 17:
6407 case 18:
6408 case 29:
6409 /* subject/issuer alt name can be a list */
6410 {
6411 uint8_t *buffer = obj->sslObjectValue.buf;
6412 size_t len = obj->sslObjectValue.len;
6413 uint16_t newlen;
6414
6415 brem = MD_REM_MSG(buf);
6416 while ((newlen = md_util_decode_asn1_sequence(&buffer, &len))) {
6417 if (*buffer == 0x30) {
6418 /* this is a sequence - ignore */
6419 break;
6420 }
6421 if (!md_util_append_buffer(buf, &brem,
6422 (uint8_t*)new_index,
6423 afterlen))
6424 {
6425 g_warning("Error %s: error appending index (%zu) to"
6426 " buffer (%zu)", exporter->name, afterlen,
6427 brem);
6428 return FALSE;
6429 }
6430 if (exporter->escape_chars) {
6431 if (!mdPrintEscapeChars(buf, &brem, buffer,
6432 newlen, delim)) {
6433 g_warning("Error %s: error appending escape "
6434 "(%d) to buffer (%zu)",
6435 exporter->name, newlen, brem);
6436 return FALSE;
6437 }
6438 } else {
6439 if (!md_util_append_buffer(buf, &brem, buffer,
6440 newlen)) {
6441 g_warning("Error %s: error appending data "
6442 "(%d) to buffer (%zu)",
6443 exporter->name, newlen, brem);
6444 return FALSE;
6445 }
6446 }
6447 buffer += newlen;
6448 len -= newlen;
6449 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
6450 }
6451 }
6452 continue;
6453 case 31:
6454 {
6455 uint8_t *buffer = obj->sslObjectValue.buf;
6456 size_t len = obj->sslObjectValue.len;
6457 uint16_t newlen;
6458 gboolean a;
6459 brem = MD_REM_MSG(buf);
6460 while ((newlen = md_util_decode_asn1_sequence(&buffer, &len))) {
6461 a = FALSE;
6462 while (*buffer == 0xa0) {
6463 buffer++;
6464 len -= 1;
6465 md_util_decode_asn1_length(&buffer, &len);
6466 a = TRUE;
6467 }
6468 if (a) continue; /* start over */
6469 if (!md_util_append_buffer(buf, &brem,
6470 (uint8_t*)new_index,
6471 afterlen))
6472 {
6473 return FALSE;
6474 }
6475 if (exporter->escape_chars) {
6476 if (!mdPrintEscapeChars(buf, &brem, buffer,
6477 newlen, delim)) {
6478 return FALSE;
6479 }
6480 } else {
6481 if (!md_util_append_buffer(buf, &brem, buffer,
6482 newlen)) {
6483 return FALSE;
6484 }
6485 }
6486 buffer += newlen;
6487 len -= newlen;
6488 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
6489 }
6490 }
6491 continue;
6492 case 32:
6493 {
6494 uint8_t *buffer = obj->sslObjectValue.buf;
6495 size_t len = obj->sslObjectValue.len;
6496 uint16_t newlen;
6497
6498 brem = MD_REM_MSG(buf);
6499 newlen = md_util_decode_asn1_sequence(&buffer, &len);
6500 if (*buffer == 0x06) {
6501 /* OID */
6502 buffer++;
6503 newlen = (uint16_t)*buffer;
6504 buffer++;
6505 if (!md_util_append_buffer(buf, &brem,
6506 (uint8_t*)new_index,
6507 afterlen))
6508 {
6509 return FALSE;
6510 }
6511
6512 /* subject key identifier - just an octet string*/
6513 if (newlen > sizeof(ssl_buffer)) {
6514 newlen = sizeof(ssl_buffer);
6515 }
6516 ret = md_util_hexdump_append(buf->cp, &brem, buffer,
6517 newlen);
6518 if (!ret) return FALSE;
6519 buf->cp += ret;
6520 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
6521 buffer += newlen;
6522 }
6523 /* now to a sequqnece {policyQualifierID, qualifier} */
6524 if (*buffer == 0x30) {
6525 /* string */
6526 len = len - newlen - 2;
6527 newlen = md_util_decode_asn1_sequence(&buffer, &len);
6528 if (*buffer == 0x06) {
6529 /* OID */
6530 buffer++;
6531 newlen = (uint16_t)*buffer;
6532 buffer += newlen + 1;
6533 if (*buffer == 0x16) {
6534 buffer++;
6535 newlen = (uint16_t)*buffer;
6536 buffer++;
6537 if (!md_util_append_buffer(buf, &brem,
6538 (uint8_t*)new_index,
6539 afterlen))
6540 {
6541 return FALSE;
6542 }
6543 if (!md_util_append_buffer(buf, &brem, buffer,
6544 newlen))
6545 {
6546 return FALSE;
6547 }
6548 MD_APPEND_CHAR_CHECK(brem, buf, '\n');
6549 }
6550 }
6551 }
6552 }
6553 continue;
6554 default:
6555 continue;
6556 }
6557 }
6558 }
6559
6560 if (exporter->multi_files) {
6561 FILE *fp = mdGetTableFile(exporter, 443);
6562 GError *err;
6563
6564 if (fp == NULL) {
6565 g_warning("Error: File does not exist for 443");
6566 return FALSE;
6567 }
6568
6569 ret = md_util_write_buffer(fp, buf, exporter->name, &err);
6570
6571 if (!ret) {
6572 g_warning("Error writing file for id 443: %s",
6573 err->message);
6574 g_clear_error(&err);
6575 }
6576
6577 exporter->exp_bytes += ret;
6578 }
6579
6580 /* print cert version */
6581 if (mdExporterCheckSSLConfig(exporter, 189, 3)) {
6582 ret = snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c%d", delim,
6583 cert_no, delim, cert->sslCertVersion);
6584 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6585 SSL_DEFAULT, index_str, index_len,
6586 189, ret, FALSE);
6587 }
6588 if (cert->sslCertSerialNumber.len && (mdExporterCheckSSLConfig(exporter, 244, 3))) {
6589 int i;
6590 size_t bufsz = sizeof(ssl_buffer);
6591 i = snprintf(ssl_buffer, bufsz, "I%c%d%c", delim, cert_no, delim);
6592 bufsz -= i;
6593 if (cert->sslCertSerialNumber.len > bufsz) {
6594 buflen = bufsz;
6595 } else {
6596 buflen = cert->sslCertSerialNumber.len;
6597 }
6598 i += md_util_hexdump_append(ssl_buffer + i, &bufsz, cert->sslCertSerialNumber.buf,
6599 buflen);
6600 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6601 SSL_DEFAULT, index_str, index_len, 244, i, FALSE);
6602 }
6603 if (cert->sslCertValidityNotBefore.len && (mdExporterCheckSSLConfig(exporter, 247, 3)))
6604 {
6605 ret = snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c", delim,
6606 cert_no, delim);
6607 strncat(ssl_buffer,(char *)cert->sslCertValidityNotBefore.buf,
6608 cert->sslCertValidityNotBefore.len);
6609 ret += cert->sslCertValidityNotBefore.len;
6610 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6611 SSL_DEFAULT, index_str, index_len, 247,
6612 ret, FALSE);
6613 }
6614
6615 if (cert->sslCertValidityNotAfter.len &&(mdExporterCheckSSLConfig(exporter, 248, 3)))
6616 {
6617 ret = snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c", delim,
6618 cert_no, delim);
6619 strncat(ssl_buffer, (char *)cert->sslCertValidityNotAfter.buf,
6620 cert->sslCertValidityNotAfter.len);
6621 ret += cert->sslCertValidityNotAfter.len;
6622 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6623 SSL_DEFAULT, index_str, index_len, 248,
6624 ret, FALSE);
6625 }
6626
6627 if (cert->sslPublicKeyLength && (mdExporterCheckSSLConfig(exporter, 250, 3))) {
6628 ret = snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c%d", delim,
6629 cert_no, delim, cert->sslPublicKeyLength);
6630 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6631 SSL_DEFAULT, index_str, index_len, 250,
6632 ret, FALSE);
6633 }
6634
6635 if (cert->sslCertificateHash.len && (mdExporterCheckSSLConfig(exporter, 295, 3))) {
6636 int i;
6637 size_t bufsz = sizeof(ssl_buffer);
6638 i = snprintf(ssl_buffer, bufsz, "I%c%d%c", delim, cert_no, delim);
6639 bufsz -= i;
6640 i += md_util_hexdump_append(ssl_buffer + i, &bufsz, cert->sslCertificateHash.buf,
6641 cert->sslCertificateHash.len);
6642 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6643 SSL_DEFAULT, index_str, index_len, 295,
6644 i, FALSE);
6645 }
6646
6647 if (cert->sslCertSignature.len && (mdExporterCheckSSLConfig(exporter, 190, 3))) {
6648 int i;
6649 size_t bufsz = sizeof(ssl_buffer);
6650 i = snprintf(ssl_buffer, bufsz, "I%c%d%c", delim, cert_no, delim);
6651 bufsz -= i;
6652 i += md_util_hexdump_append(ssl_buffer + i, &bufsz, cert->sslCertSignature.buf,
6653 cert->sslCertSignature.len);
6654 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6655 SSL_DEFAULT, index_str, index_len, 190,
6656 i, FALSE);
6657 }
6658
6659 return TRUE;
6660
6661 }
6662
6663
mdExporterTextNewSSLPrint(mdFlowExporter_t * exporter,yaf_newssl_t * sslflow,char * index_str,size_t index_len)6664 gboolean mdExporterTextNewSSLPrint(
6665 mdFlowExporter_t *exporter,
6666 yaf_newssl_t *sslflow,
6667 char *index_str,
6668 size_t index_len)
6669 {
6670 char delim = exporter->dpi_delimiter;
6671 yaf_newssl_cert_t *cert = NULL;
6672 fbSubTemplateList_t *stl = &(sslflow->sslCertList);
6673 int cert_no = 0;
6674 char ssl_buffer[500];
6675 size_t buflen;
6676
6677
6678 while ((cert = (yaf_newssl_cert_t *)FBSTLNEXT(stl, cert)))
6679 {
6680 if (!mdExporterTextNewSSLCertPrint(exporter, cert, index_str,
6681 index_len, cert_no)) {
6682 return FALSE;
6683 }
6684 cert_no++;
6685 }
6686
6687 if (sslflow->sslServerCipher &&(mdExporterCheckSSLConfig(exporter, 187,3)))
6688 {
6689 snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c0x%04x",
6690 delim, 0, delim, sslflow->sslServerCipher);
6691 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6692 SSL_DEFAULT, index_str, index_len, 187,
6693 strlen(ssl_buffer), FALSE);
6694 }
6695
6696 if (sslflow->sslCompressionMethod &&
6697 (mdExporterCheckSSLConfig(exporter, 188, 3)))
6698 {
6699 snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c%d", delim, 0, delim,
6700 sslflow->sslCompressionMethod);
6701 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6702 SSL_DEFAULT, index_str, index_len, 188,
6703 strlen(ssl_buffer), FALSE);
6704 }
6705
6706 if (sslflow->sslClientVersion &&
6707 (mdExporterCheckSSLConfig(exporter, 186, 3))) {
6708 snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c%d", delim, 0, delim,
6709 sslflow->sslClientVersion);
6710 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6711 SSL_DEFAULT, index_str, index_len, 186,
6712 strlen(ssl_buffer), FALSE);
6713 }
6714
6715 if (sslflow->sslRecordVersion &&
6716 (mdExporterCheckSSLConfig(exporter, 288, 3)))
6717 {
6718 snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c0x%04x", delim,
6719 0, delim, sslflow->sslRecordVersion);
6720 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6721 SSL_DEFAULT, index_str, index_len, 288,
6722 strlen(ssl_buffer), FALSE);
6723 }
6724
6725 if (sslflow->sslServerName.buf &&
6726 (mdExporterCheckSSLConfig(exporter, 294, 3)))
6727 {
6728 snprintf(ssl_buffer, sizeof(ssl_buffer), "I%c%d%c", delim, 0,
6729 delim);
6730 if ((sslflow->sslServerName.len + strlen(ssl_buffer)) < sizeof(ssl_buffer))
6731 {
6732 buflen = strlen(ssl_buffer);
6733 memcpy(ssl_buffer + buflen, sslflow->sslServerName.buf,
6734 sslflow->sslServerName.len);
6735 buflen += sslflow->sslServerName.len;
6736 exporter->VLprint_fn(exporter, (uint8_t *)ssl_buffer,
6737 SSL_DEFAULT, index_str, index_len, 294,
6738 buflen, FALSE);
6739 }
6740 }
6741
6742 return TRUE;
6743 }
6744
mdExporterWriteDNSRRRecord(mdConfig_t * cfg,mdFlowExporter_t * exporter,uint16_t tid,uint8_t * rec,size_t rec_length,GError ** err)6745 gboolean mdExporterWriteDNSRRRecord(
6746 mdConfig_t *cfg,
6747 mdFlowExporter_t *exporter,
6748 uint16_t tid,
6749 uint8_t *rec,
6750 size_t rec_length,
6751 GError **err)
6752 {
6753 int ret;
6754 size_t bytes;
6755 gboolean expand = FALSE;
6756 md_dns_rr_t *dns = (md_dns_rr_t *)rec;
6757
6758 if (exporter == NULL) {
6759 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
6760 "Exporter Node Exists, but No Type\n");
6761 return FALSE;
6762 }
6763
6764 if (exporter->multi_files || (exporter->no_stats == 2)) {
6765 /* don't export anything for these exporters */
6766 return TRUE;
6767 }
6768
6769 if (!exporter->active) {
6770 if (cfg->ctime - exporter->last_restart_ms > MD_RESTART_MS) {
6771 if (!mdExporterRestart(cfg, exporter, err)) {
6772 g_message("Error restarting exporter %s: %s",
6773 exporter->name, (*err)->message);
6774 g_clear_error(err);
6775 return TRUE;
6776 }
6777 } else {
6778 return TRUE;
6779 }
6780
6781 }
6782
6783 if (exporter->fbuf) {
6784
6785 if (!mdExporterfBufSetup(cfg, exporter, NULL, err,
6786 mdInitExporterSessionDNSRROnly, MD_DNSRR, tid))
6787 {
6788 return FALSE;
6789 }
6790
6791 if (!fBufAppend(exporter->fbuf, (uint8_t *)rec, rec_length, err)) {
6792 fBufFree(exporter->fbuf);
6793 goto err;
6794 }
6795 /* update stats */
6796 exporter->exp_bytes += rec_length;
6797
6798 }
6799
6800 if (exporter->type == TEXT) {
6801 if (exporter->rotate) {
6802 if ((cfg->ctime - exporter->last_rotate_ms) > exporter->rotate) {
6803 if (!mdTextFileRotate(exporter, cfg->ctime, err)) {
6804 exporter->last_rotate_ms = 0;
6805 goto err;
6806 }
6807 }
6808 }
6809 if (exporter->custom_list && !exporter->basic_list_dpi) {
6810 mdFullFlow_t fflow;
6811 md_main_template_t mdrec;
6812 mdFieldList_t *cnode = NULL;
6813 size_t buflen = MD_REM_MSG(exporter->buf);
6814
6815 memset(&fflow, 0, sizeof(mdFullFlow_t));
6816 memset(&mdrec, 0, sizeof(md_main_template_t));
6817
6818 mdrec.flowStartMilliseconds = dns->flowStartMilliseconds;
6819 mdrec.flowEndMilliseconds = dns->flowStartMilliseconds;
6820 if (dns->sourceIPv4Address || dns->destinationIPv4Address) {
6821 mdrec.sourceIPv4Address = dns->sourceIPv4Address;
6822 mdrec.destinationIPv4Address = dns->destinationIPv4Address;
6823 } else {
6824 memcpy(&(mdrec.sourceIPv6Address), dns->sourceIPv6Address, 16);
6825 memcpy(&(mdrec.destinationIPv6Address), dns->destinationIPv6Address, 16);
6826 }
6827 mdrec.silkAppLabel = 53;
6828 mdrec.observationDomainId = dns->observationDomainId;
6829 mdrec.sourceTransportPort = dns->sourceTransportPort;
6830 mdrec.destinationTransportPort = dns->destinationTransportPort;
6831 mdrec.vlanId = dns->vlanId;
6832 mdrec.protocolIdentifier = dns->protocolIdentifier;
6833 fflow.rec = &mdrec;
6834
6835 for (cnode = exporter->custom_list; cnode; cnode = cnode->next) {
6836 if (!cnode->print_fn(&fflow, exporter->buf, &buflen,
6837 cnode->decorator->str))
6838 {
6839 if (!expand) {
6840 if (!mdExporterExpandBuf(exporter)) {
6841 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
6842 "Error allocating memory for exporter %s",
6843 exporter->name);
6844 return FALSE;
6845 }
6846 expand = TRUE;
6847 /* start over */
6848 cnode = exporter->custom_list;
6849 } else {
6850 /* already tried this - ABORT! */
6851 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
6852 "Error writing to buffer for exporter %s",
6853 exporter->name);
6854 goto err;
6855 }
6856 }
6857 }
6858 exporter->buf->cp -= 1;
6859 buflen += 1;
6860 MD_APPEND_CHAR(exporter->buf, '\n');
6861 bytes = md_util_write_buffer(exporter->lfp, exporter->buf,
6862 exporter->name, err);
6863 if (!bytes) {
6864 goto err;
6865 }
6866
6867 } else if (exporter->json) {
6868 /* here will call printDNS function already implemented in JSON code */
6869 if (!mdJsonifyDNSRRRecord((md_dns_rr_t *)rec, exporter->buf)) {
6870 goto err;
6871 }
6872 bytes = md_util_write_buffer(exporter->lfp, exporter->buf,
6873 exporter->name, err);
6874 if (!bytes) {
6875 goto err;
6876 }
6877
6878
6879 } else {
6880
6881 ret = mdPrintDNSRRRecord(exporter->buf, exporter->lfp,
6882 exporter->delimiter, rec,
6883 cfg->dns_base64_encode,
6884 exporter->escape_chars, err);
6885 if (ret < 0) {
6886 goto err;
6887 } else if (ret == 0) {
6888 if (!mdExporterExpandBuf(exporter)) {
6889 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
6890 "Error allocating memory for exporter %s",
6891 exporter->name);
6892 return FALSE;
6893 }
6894 ret = mdPrintDNSRRRecord(exporter->buf, exporter->lfp,
6895 exporter->delimiter, rec,
6896 cfg->dns_base64_encode,
6897 exporter->escape_chars, err);
6898 if (ret < 0) {
6899 goto err;
6900 } else if (ret == 0) {
6901 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
6902 "Error writing to buffer for exporter %s",
6903 exporter->name);
6904 goto err;
6905 }
6906 }
6907 bytes = ret;
6908 }
6909
6910 exporter->exp_bytes += bytes;
6911 }
6912
6913 ++(exporter->exp_flows);
6914
6915 return TRUE;
6916
6917 err:
6918
6919 g_warning("Error writing DNS Record: %s", (*err)->message);
6920 g_clear_error(err);
6921 g_warning("Deactivating Exporter %s.", exporter->name);
6922 exporter->active = FALSE;
6923 if (!mdExporterRestart(cfg, exporter, err)) {
6924 g_warning("Error restarting exporter %s: %s",
6925 exporter->name, (*err)->message);
6926 g_clear_error(err);
6927 }
6928
6929 return TRUE;
6930 }
6931
mdExportDNSRR(mdConfig_t * cfg,mdFlowExporter_t * exporter,mdFullFlow_t * flow,uint16_t tid,GError ** err)6932 gboolean mdExportDNSRR(
6933 mdConfig_t *cfg,
6934 mdFlowExporter_t *exporter,
6935 mdFullFlow_t *flow,
6936 uint16_t tid,
6937 GError **err)
6938 {
6939
6940 md_dns_rr_t dns;
6941 yaf_dns_t *dnsflow = (yaf_dns_t *)flow->app;
6942 yaf_dnsQR_t *dnsqr = NULL;
6943
6944 if ((flow->rec->silkAppLabel != 53) || (dnsflow == NULL)) {
6945 return TRUE;
6946 }
6947
6948 if (exporter->deduponly || exporter->dnsdeduponly ||
6949 exporter->ssldeduponly || exporter->flowonly || exporter->multi_files)
6950 {
6951 return TRUE;
6952 }
6953
6954 memset(&dns, 0, sizeof(dns));
6955
6956 if (exporter->dns_rr_only == 2) {
6957 /* check if flow is v4 or v6 */
6958 if (tid & YTF_IP4) {
6959 dns.sourceIPv4Address = flow->rec->sourceIPv4Address;
6960 dns.destinationIPv4Address = flow->rec->destinationIPv4Address;
6961 tid = MD_DNSRR;
6962 tid |= MD_DNSRR_FULL;
6963 tid |= YTF_IP4;
6964 } else {
6965 memcpy(dns.sourceIPv6Address, flow->rec->sourceIPv6Address, 16);
6966 memcpy(dns.destinationIPv6Address, flow->rec->destinationIPv6Address, 16);
6967 tid = MD_DNSRR;
6968 tid |= MD_DNSRR_FULL;
6969 tid |= YTF_IP6;
6970 }
6971 dns.sourceTransportPort = flow->rec->sourceTransportPort;
6972 dns.destinationTransportPort = flow->rec->destinationTransportPort;
6973 dns.vlanId = flow->rec->vlanId;
6974 dns.protocolIdentifier = flow->rec->protocolIdentifier;
6975 } else {
6976 tid = MD_DNSRR;
6977 }
6978
6979
6980 dns.yafFlowKeyHash = md_util_flow_key_hash(flow->rec);
6981 dns.flowStartMilliseconds = flow->rec->flowStartMilliseconds;
6982 dns.observationDomainId = flow->rec->observationDomainId;
6983
6984 while ((dnsqr =(yaf_dnsQR_t *)FBSTLNEXT(&(dnsflow->dnsQRList),
6985 dnsqr)))
6986 {
6987 dns.dnsTTL = dnsqr->dnsTTL;
6988 dns.dnsQRType = dnsqr->dnsQRType;
6989 dns.dnsID = dnsqr->dnsID;
6990 dns.dnsQueryResponse = dnsqr->dnsQueryResponse;
6991 dns.dnsAuthoritative = dnsqr->dnsAuthoritative;
6992 dns.dnsNXDomain = dnsqr->dnsNXDomain;
6993 dns.rrname.buf = dnsqr->dnsQName.buf;
6994 dns.rrname.len = dnsqr->dnsQName.len;
6995
6996 if (exporter->dns_resp_only) {
6997 if (dns.dnsQueryResponse == 0) continue;
6998 }
6999
7000 if (flow->rec->flowEndReason == UDP_FORCE && dns.dnsQueryResponse == 1) {
7001 dns.yafFlowKeyHash = md_util_rev_flow_key_hash(flow->rec);
7002 }
7003
7004 dns.rrdata.buf = NULL;
7005 dns.rrdata.len = 0;
7006
7007 if (dns.dnsQueryResponse) {
7008 switch (dnsqr->dnsQRType) {
7009 case 1:
7010 {
7011 yaf_dnsA_t *a = NULL;
7012 while ((a = (yaf_dnsA_t *)FBSTLNEXT(&(dnsqr->dnsRRList), a))) {
7013 dns.rrdata.buf = (uint8_t *)&(a->sourceIPv4Address);
7014 dns.rrdata.len = sizeof(flow->rec->sourceIPv4Address);
7015 }
7016 break;
7017 }
7018 case 2:
7019 {
7020 yaf_dnsNS_t *ns = NULL;
7021 while ((ns = (yaf_dnsNS_t *)FBSTLNEXT(&(dnsqr->dnsRRList), ns)))
7022 {
7023 dns.rrdata.buf = ns->dnsNSDName.buf;
7024 dns.rrdata.len = ns->dnsNSDName.len;
7025 }
7026 break;
7027
7028 }
7029 case 5:
7030 {
7031 yaf_dns_CNAME_t *c = NULL;
7032 while ((c = (yaf_dns_CNAME_t *)FBSTLNEXT(&(dnsqr->dnsRRList), c)))
7033 {
7034 dns.rrdata.buf = c->dnsCName.buf;
7035 dns.rrdata.len = c->dnsCName.len;
7036 }
7037 break;
7038 }
7039 case 6:
7040 {
7041 yaf_dnsSOA_t *soa = NULL;
7042 while ((soa=(yaf_dnsSOA_t *)FBSTLNEXT(&(dnsqr->dnsRRList), soa))) {
7043 dns.rrdata.buf = soa->dnsSOAMName.buf;
7044 dns.rrdata.len = soa->dnsSOAMName.len;
7045 }
7046 break;
7047 }
7048 case 12:
7049 {
7050 yaf_dnsPTR_t *ptr = NULL;
7051 while ((ptr = (yaf_dnsPTR_t *)FBSTLNEXT(&(dnsqr->dnsRRList), ptr)))
7052 {
7053 dns.rrdata.buf = ptr->dnsPTRDName.buf;
7054 dns.rrdata.len = ptr->dnsPTRDName.len;
7055 }
7056 break;
7057 }
7058 case 15:
7059 {
7060 yaf_dnsMX_t *mx = NULL;
7061 while (( mx = (yaf_dnsMX_t *)FBSTLNEXT(&(dnsqr->dnsRRList), mx)))
7062 {
7063 dns.rrdata.buf = mx->dnsMXExchange.buf;
7064 dns.rrdata.len = mx->dnsMXExchange.len;
7065 }
7066 break;
7067
7068 }
7069 case 16:
7070 {
7071 yaf_dnsTXT_t *txt = NULL;
7072 while ((txt = (yaf_dnsTXT_t *)FBSTLNEXT(&(dnsqr->dnsRRList), txt)))
7073 {
7074 dns.rrdata.buf = txt->dnsTXTData.buf;
7075 dns.rrdata.len = txt->dnsTXTData.len;
7076 }
7077 break;
7078 }
7079 case 28:
7080 {
7081 yaf_dnsAAAA_t *aa = NULL;
7082 while ((aa = (yaf_dnsAAAA_t *)FBSTLNEXT(&(dnsqr->dnsRRList), aa)))
7083 {
7084 dns.rrdata.buf = (uint8_t *)&(aa->sourceIPv6Address);
7085 dns.rrdata.len = sizeof(flow->rec->sourceIPv6Address);
7086 }
7087 break;
7088 }
7089 case 33:
7090 {
7091 yaf_dnsSRV_t *srv = NULL;
7092 while ((srv = (yaf_dnsSRV_t *)FBSTLNEXT(&(dnsqr->dnsRRList), srv)))
7093 {
7094 dns.rrdata.buf = srv->dnsSRVTarget.buf;
7095 dns.rrdata.len = srv->dnsSRVTarget.len;
7096 }
7097 break;
7098 }
7099 case 46:
7100 {
7101 yaf_dnsRRSig_t *rr = NULL;
7102 while ((rr =(yaf_dnsRRSig_t *)FBSTLNEXT(&(dnsqr->dnsRRList), rr))){
7103 dns.rrdata.buf = rr->dnsSigner.buf;
7104 dns.rrdata.len = rr->dnsSigner.len;
7105 }
7106 break;
7107 }
7108 case 47:
7109 {
7110 yaf_dnsNSEC_t *nsec = NULL;
7111 while ((nsec = (yaf_dnsNSEC_t *)FBSTLNEXT(&(dnsqr->dnsRRList), nsec))) {
7112 dns.rrdata.buf = nsec->dnsHashData.buf;
7113 dns.rrdata.len = nsec->dnsHashData.len;
7114 }
7115 break;
7116 }
7117 default:
7118 dns.rrdata.buf = NULL;
7119 dns.rrdata.len = 0;
7120 }
7121 }
7122
7123 if (!mdExporterWriteDNSRRRecord(cfg, exporter, tid, (uint8_t *)&dns,
7124 sizeof(dns), err)) {
7125 return FALSE;
7126 }
7127 }
7128
7129 return TRUE;
7130 }
7131
mdExporterDedupFileClose(mdFlowExporter_t * exporter,FILE * fp,char * last_file)7132 void mdExporterDedupFileClose(
7133 mdFlowExporter_t *exporter,
7134 FILE *fp,
7135 char *last_file)
7136 {
7137 mdCloseAndUnlock(exporter, fp, last_file, NULL);
7138 /*if (fp) {
7139 fclose(fp);
7140 }
7141
7142 if (last_file) {
7143 if (exporter->lock) {
7144 mdUnlockFile(last_file);
7145 }
7146 g_free(last_file);
7147
7148 }*/
7149 }
7150
mdExporterDedupFileOpen(mdConfig_t * cfg,mdFlowExporter_t * exporter,FILE ** file,char ** last_file,char * prefix,uint64_t * rotate)7151 gboolean mdExporterDedupFileOpen(
7152 mdConfig_t *cfg,
7153 mdFlowExporter_t *exporter,
7154 FILE **file,
7155 char **last_file,
7156 char *prefix,
7157 uint64_t *rotate)
7158 {
7159
7160 GString *file_name;
7161 uint64_t start_secs;
7162 FILE *fp = *file;
7163
7164 if (exporter->type != TEXT) {
7165 return TRUE;
7166 }
7167
7168 if (exporter->json) {
7169 fp = exporter->lfp;
7170 }
7171
7172 if (fp && !exporter->rotate) {
7173 return TRUE;
7174 }
7175
7176 if (fp && exporter->rotate) {
7177 if ((cfg->ctime - *rotate) < exporter->rotate) {
7178 return TRUE;
7179 } else {
7180 if (exporter->json) {
7181 mdExporterDedupFileClose(exporter, fp, exporter->current_fname);
7182 } else {
7183 mdExporterDedupFileClose(exporter, fp, *last_file);
7184 }
7185 }
7186 }
7187
7188 file_name = g_string_new("");
7189
7190 if (exporter->dedupconfig) {
7191 g_string_assign(file_name, exporter->outspec);
7192 }
7193
7194 if (exporter->rotate) {
7195 start_secs = cfg->ctime / 1000;
7196
7197 if (!exporter->json) {
7198 /* 1 FILE for JSON */
7199 g_string_append_printf(file_name, "%s.", prefix);
7200 }
7201 md_util_time_g_string_append(file_name, start_secs, TIME_FMT);
7202
7203 if (exporter->json) {
7204 g_string_append_printf(file_name, ".json");
7205 } else {
7206 g_string_append_printf(file_name, ".txt");
7207 }
7208 } else if (!exporter->json) {
7209 if (!exporter->dedupconfig) {
7210 g_string_append_printf(file_name, "%s", prefix);
7211 } else {
7212 g_string_append_printf(file_name, "%s.txt", prefix);
7213 }
7214 }
7215
7216 if (exporter->json) {
7217 exporter->current_fname = g_strdup(file_name->str);
7218 } else {
7219 *last_file = g_strdup(file_name->str);
7220 }
7221 if (exporter->lock) {
7222 mdLockFile(file_name);
7223 }
7224
7225 fp = fopen(file_name->str, "w");
7226 if (fp == NULL) {
7227 g_warning("%s: Error Opening File (%d) %s", exporter->name, errno,
7228 file_name->str);
7229 return FALSE;
7230 }
7231 g_debug("%s: Opening Text File: %s", exporter->name,
7232 file_name->str);
7233 g_string_free(file_name, TRUE);
7234
7235 if (exporter->rotate) {
7236 *rotate = cfg->ctime;
7237 }
7238 if (exporter->json) {
7239 exporter->lfp = fp;
7240 } else {
7241 *file = fp;
7242 }
7243
7244 return TRUE;
7245 }
7246
mdExporterWriteDedupRecord(mdConfig_t * cfg,md_export_node_t * enode,FILE * fp,md_dedup_t * rec,char * prefix,uint16_t int_tid,uint16_t ext_tid,GError ** err)7247 gboolean mdExporterWriteDedupRecord(
7248 mdConfig_t *cfg,
7249 md_export_node_t *enode,
7250 FILE *fp,
7251 md_dedup_t *rec,
7252 char *prefix,
7253 uint16_t int_tid,
7254 uint16_t ext_tid,
7255 GError **err)
7256 {
7257 mdFlowExporter_t *exporter = enode->exp;
7258 size_t rec_length = sizeof(md_dedup_t);
7259 int ret = 0;
7260
7261 if (exporter == NULL) {
7262 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
7263 "Exporter Node Exists, but No Type\n");
7264 return FALSE;
7265 }
7266
7267 if (exporter->ssldeduponly || exporter->dnsdeduponly ||
7268 exporter->flowonly || exporter->multi_files)
7269 {
7270 return TRUE;
7271 }
7272
7273 if (!exporter->active) {
7274 if (cfg->ctime - exporter->last_restart_ms > MD_RESTART_MS) {
7275 if (!mdExporterRestart(cfg, exporter, err)) {
7276 g_message("Error restarting exporter %s: %s",
7277 exporter->name, (*err)->message);
7278 g_clear_error(err);
7279 return TRUE;
7280 }
7281 } else {
7282 return TRUE;
7283 }
7284 }
7285
7286 if (exporter->fbuf) {
7287 if (int_tid == 0) {
7288 int_tid = MD_DEDUP_FULL;
7289 }
7290
7291 if (!mdExporterfBufSetup(cfg, exporter, NULL, err,
7292 mdInitExporterSessionDedupOnly,
7293 int_tid, ext_tid))
7294 {
7295 /* if this fails, it's probably because the internal dedup
7296 templates have not been added to the session. Add them
7297 and try again */
7298
7299 if (!g_error_matches(*err, FB_ERROR_DOMAIN, FB_ERROR_TMPL)) {
7300 return FALSE;
7301 }
7302
7303 g_clear_error(err);
7304 if (!md_dedup_add_templates(enode->dedup, exporter->fbuf, err)) {
7305 return FALSE;
7306 }
7307 if (!mdExporterfBufSetup(cfg, exporter, NULL, err,
7308 mdInitExporterSessionDedupOnly,
7309 int_tid, ext_tid))
7310 {
7311 return FALSE;
7312 }
7313 }
7314 if (!fBufAppend(exporter->fbuf, (uint8_t *)rec, rec_length, err)) {
7315 fBufFree(exporter->fbuf);
7316 goto err;
7317 }
7318 /* update stats */
7319 exporter->exp_bytes += rec_length;
7320
7321 }
7322
7323 if (exporter->type == TEXT) {
7324
7325 if (!fp) {
7326 /* for collectors OR JSON exporters */
7327 fp = exporter->lfp;
7328 }
7329
7330 if (exporter->json) {
7331 ret = mdJsonifyDedupRecord(fp, exporter->buf, prefix,
7332 rec, err);
7333
7334 } else {
7335 ret = mdPrintDedupRecord(fp, exporter->buf, rec,
7336 exporter->delimiter, err);
7337 }
7338
7339 if (ret < 0) {
7340 goto err;
7341 } else if (ret == 0) {
7342 if (!mdExporterExpandBuf(exporter)) {
7343 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
7344 "Error allocating memory for exporter %s",
7345 exporter->name);
7346 return FALSE;
7347 }
7348
7349 if (exporter->json) {
7350 ret = mdJsonifyDedupRecord(fp, exporter->buf, prefix,
7351 rec, err);
7352 } else {
7353 ret = mdPrintDedupRecord(fp, exporter->buf, rec,
7354 exporter->delimiter, err);
7355 }
7356
7357 if (ret< 0) {
7358 goto err;
7359 } else if (ret == 0) {
7360 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
7361 "Error writing to buffer for exporter %s",
7362 exporter->name);
7363 goto err;
7364 }
7365 }
7366
7367 exporter->exp_bytes += ret;
7368 }
7369
7370 ++(exporter->exp_flows);
7371
7372 return TRUE;
7373
7374 err:
7375
7376 g_warning("Error writing Dedup Record: %s", (*err)->message);
7377 g_clear_error(err);
7378 g_warning("Deactivating Exporter %s.", exporter->name);
7379 exporter->active = FALSE;
7380 if (!mdExporterRestart(cfg, exporter, err)) {
7381 g_warning("Error restarting exporter %s: %s",
7382 exporter->name, (*err)->message);
7383 g_clear_error(err);
7384 }
7385
7386 return TRUE;
7387 }
7388
mdExporterSSLCertRecord(mdConfig_t * cfg,mdFlowExporter_t * exporter,FILE * cert_file,yaf_newssl_cert_t * ssl,yfSSLFullCert_t * fullcert,uint8_t * issuer,size_t issuer_len,uint8_t cert_no,GError ** err)7389 gboolean mdExporterSSLCertRecord(
7390 mdConfig_t *cfg,
7391 mdFlowExporter_t *exporter,
7392 FILE *cert_file,
7393 yaf_newssl_cert_t *ssl,
7394 yfSSLFullCert_t *fullcert,
7395 uint8_t *issuer,
7396 size_t issuer_len,
7397 uint8_t cert_no,
7398 GError **err)
7399 {
7400 size_t rc;
7401 size_t brem;
7402 int ret;
7403 uint64_t start_secs = cfg->ctime/1000;
7404 uint32_t start_rem = cfg->ctime %1000;
7405
7406 if (exporter->deduponly || exporter->dnsdeduponly ||
7407 exporter->dns_rr_only || exporter->flowonly || exporter->multi_files)
7408 {
7409 return TRUE;
7410 }
7411
7412 if (!exporter->active) {
7413 if (cfg->ctime - exporter->last_restart_ms > MD_RESTART_MS) {
7414 if (!mdExporterRestart(cfg, exporter, err)) {
7415 g_message("Error restarting exporter %s: %s",
7416 exporter->name, (*err)->message);
7417 g_clear_error(err);
7418 return TRUE;
7419 }
7420 } else {
7421 return TRUE;
7422 }
7423
7424 }
7425
7426 if (exporter->fbuf) {
7427
7428 if (!mdExporterfBufSetup(cfg, exporter, NULL, err,
7429 mdInitExporterSessionSSLDedupOnly,
7430 YAF_NEW_SSL_CERT_TID, YAF_NEW_SSL_CERT_TID))
7431 {
7432 return FALSE;
7433 }
7434
7435 if (!fBufAppend(exporter->fbuf, (uint8_t *)ssl,
7436 sizeof(yaf_newssl_cert_t), err))
7437 {
7438 fBufFree(exporter->fbuf);
7439 goto err;
7440 }
7441 /* update stats */
7442 exporter->exp_bytes += sizeof(yaf_newssl_cert_t);
7443
7444 }
7445
7446 if (exporter->type == TEXT) {
7447
7448 brem = MD_REM_MSG(exporter->buf);
7449
7450 if (exporter->json) {
7451
7452 ret = snprintf(exporter->buf->cp, brem, "{\"sslCert\":");
7453 MD_CHECK_RET(exporter->buf, ret, brem);
7454 if (!mdJsonifyNewSSLCertRecord(exporter, ssl, cert_no)) {
7455 /* expand buffer */
7456 if (!mdExporterExpandBuf(exporter)) {
7457 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
7458 "Error allocating memory for exporter %s",
7459 exporter->name);
7460 return -1;
7461 }
7462 brem = MD_REM_MSG(exporter->buf);
7463 /* try again with expanded buffer */
7464 ret = snprintf(exporter->buf->cp, brem,
7465 "{\"sslCert\":");
7466 MD_CHECK_RET(exporter->buf, ret, brem);
7467 if (!mdJsonifyNewSSLCertRecord(exporter, ssl, cert_no)) {
7468 return FALSE;
7469 }
7470 }
7471 if (fullcert && (exporter->md5_hash || exporter->sha1_hash ||
7472 mdExporterCheckSSLConfig(exporter, 299, 3) ||
7473 mdExporterCheckSSLConfig(exporter, 298, 3) ||
7474 mdExporterCheckSSLConfig(exporter, 296, 3)))
7475 {
7476 fbVarfield_t *ct =
7477 fbBasicListGetIndexedDataPtr(&(fullcert->cert), cert_no);
7478
7479 if (ct->len) {
7480 if (!mdExporterSSLCertHash(exporter, ct, NULL, 0, cert_no)) {
7481 return FALSE;
7482 }
7483 if (!mdJsonifySSLCertBase64(exporter->buf, ct)) {
7484 return FALSE;
7485 }
7486 }
7487 }
7488 /* remove '},' */
7489 exporter->buf->cp -= 2;
7490 brem += 2;
7491 ret = snprintf(exporter->buf->cp, brem,
7492 ",\"flowStartMilliseconds\":\"");
7493 MD_CHECK_RET(exporter->buf, ret, brem);
7494 md_util_time_buf_append(exporter->buf, &brem, start_secs,
7495 PRINT_TIME_FMT);
7496
7497 ret = snprintf(exporter->buf->cp, brem, ".%03u\"}}\n", start_rem);
7498 MD_CHECK_RET(exporter->buf, ret, brem);
7499 } else {
7500 char *bufstart = exporter->buf->cp;
7501 size_t afterlen;
7502 gboolean index_config = exporter->no_index;
7503
7504 /* set temporarily */
7505 exporter->no_index = TRUE;
7506
7507 ret = md_util_hexdump_append_nospace(exporter->buf->cp, &brem,
7508 ssl->sslCertSerialNumber.buf, ssl->sslCertSerialNumber.len);
7509 if (!ret) {
7510 exporter->no_index = index_config;
7511 g_warning("Error %s: error appending serial (%zu) to buffer",
7512 exporter->name, ssl->sslCertSerialNumber.len);
7513 return FALSE;
7514 }
7515 exporter->buf->cp += ret;
7516 MD_APPEND_CHAR_CHECK(brem, exporter->buf, exporter->delimiter);
7517
7518 if (issuer) {
7519 if (!md_util_append_buffer(exporter->buf, &brem, issuer,
7520 issuer_len))
7521 {
7522 g_warning("Error %s: error appending issuer (%zu) to buffer",
7523 exporter->name, issuer_len);
7524 exporter->no_index = index_config;
7525 return FALSE;
7526 }
7527 }
7528 MD_APPEND_CHAR_CHECK(brem, exporter->buf, exporter->delimiter);
7529
7530 if (start_secs) {
7531 if (!md_util_time_buf_append(exporter->buf, &brem, start_secs,
7532 PRINT_TIME_FMT))
7533 {
7534 g_warning("Error %s: error appending time to buffer (%zu)",
7535 exporter->name, brem);
7536 exporter->no_index = index_config;
7537 return FALSE;
7538 }
7539
7540 ret = snprintf(exporter->buf->cp, brem, ".%03u%c",
7541 start_rem, exporter->delimiter);
7542 MD_CHECK_RET(exporter->buf, ret, brem);
7543 } else {
7544 MD_APPEND_CHAR_CHECK(brem, exporter->buf, exporter->delimiter);
7545 }
7546 afterlen = exporter->buf->cp - bufstart;
7547 /* reset buffer */
7548 exporter->buf->cp = bufstart;
7549 bufstart = malloc(afterlen);
7550 memcpy(bufstart, exporter->buf->cp, afterlen);
7551 if (!mdExporterTextNewSSLCertPrint(exporter, ssl, bufstart,
7552 afterlen, cert_no)) {
7553 /* expand buffer */
7554 if (!mdExporterExpandBuf(exporter)) {
7555 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
7556 "Error allocating memory for exporter %s",
7557 exporter->name);
7558 exporter->no_index = index_config;
7559 free(bufstart);
7560 return FALSE;
7561 }
7562 /* try again with expanded buffer */
7563 brem = MD_REM_MSG(exporter->buf);
7564 if (!md_util_append_buffer(exporter->buf, &brem,
7565 (uint8_t*)bufstart, afterlen))
7566 {
7567 exporter->no_index = index_config;
7568 free(bufstart);
7569 return FALSE;
7570 }
7571 if (!mdExporterTextNewSSLCertPrint(exporter, ssl, bufstart,
7572 afterlen, cert_no)) {
7573
7574 exporter->no_index = index_config;
7575 free(bufstart);
7576 return FALSE;
7577 }
7578 }
7579 if (fullcert && (exporter->md5_hash || exporter->sha1_hash ||
7580 mdExporterCheckSSLConfig(exporter, 299, 3) ||
7581 mdExporterCheckSSLConfig(exporter, 298, 3) ||
7582 mdExporterCheckSSLConfig(exporter, 296, 3)))
7583 {
7584 fbVarfield_t *ct = fbBasicListGetIndexedDataPtr(&(fullcert->cert), cert_no);
7585 if (ct->len) {
7586 if (!mdExporterSSLCertHash(exporter, ct, bufstart, afterlen, cert_no)) {
7587 return FALSE;
7588 }
7589 if (mdExporterCheckSSLConfig(exporter, 296, 3)) {
7590 if (!mdExporterSSLBase64Encode(exporter, ct, bufstart, afterlen, cert_no)) {
7591 return FALSE;
7592 }
7593 }
7594 }
7595 }
7596
7597 exporter->no_index = index_config;
7598 free(bufstart);
7599 }
7600
7601 /* write to file */
7602 if (!cert_file && exporter->rotate) {
7603 if ((cfg->ctime - exporter->last_rotate_ms) > exporter->rotate) {
7604 if (!mdTextFileRotate(exporter, cfg->ctime, err)) {
7605 exporter->last_rotate_ms = 0;
7606 return FALSE;
7607 }
7608 }
7609 }
7610
7611 if (MD_MSG_LEN(exporter->buf) == 0) {
7612 /* Nothing to write */
7613 return TRUE;
7614 }
7615
7616 if (cert_file) {
7617 rc = md_util_write_buffer(cert_file, exporter->buf,
7618 exporter->name, err);
7619 } else {
7620 rc = md_util_write_buffer(exporter->lfp, exporter->buf,
7621 exporter->name, err);
7622 }
7623
7624 if (!rc) {
7625 goto err;
7626 }
7627
7628 exporter->exp_bytes += rc;
7629 }
7630
7631 return TRUE;
7632
7633 err:
7634
7635 g_warning("Error writing SSL CERT Record: %s", (*err)->message);
7636 g_clear_error(err);
7637 g_warning("Deactivating Exporter %s.", exporter->name);
7638 exporter->active = FALSE;
7639 if (!mdExporterRestart(cfg, exporter, err)) {
7640 g_warning("Error restarting exporter %s: %s",
7641 exporter->name, (*err)->message);
7642 g_clear_error(err);
7643 }
7644
7645 return TRUE;
7646 }
7647
7648 /**
7649 * mdExporterWriteSSLDedupRecord
7650 *
7651 * write a SSL de-duplicated record to the given exporter
7652 *
7653 * @param cfg - mediator configuration options
7654 * @param exporter - exporter to write to
7655 * @param tid - template id
7656 * @param rec - the record to write
7657 * @param rec_length - length of record to write
7658 * @param err
7659 * @return TRUE if no errors
7660 */
mdExporterWriteSSLDedupRecord(mdConfig_t * cfg,mdFlowExporter_t * exporter,uint16_t tid,uint8_t * rec,size_t rec_length,GError ** err)7661 gboolean mdExporterWriteSSLDedupRecord(
7662 mdConfig_t *cfg,
7663 mdFlowExporter_t *exporter,
7664 uint16_t tid,
7665 uint8_t *rec,
7666 size_t rec_length,
7667 GError **err)
7668 {
7669 int ret;
7670
7671 if (exporter == NULL) {
7672 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
7673 "Exporter Node Exists, but No Type\n");
7674 return FALSE;
7675 }
7676
7677 if (exporter->deduponly || exporter->dnsdeduponly ||
7678 exporter->dns_rr_only || exporter->flowonly || exporter->multi_files)
7679 {
7680 return TRUE;
7681 }
7682
7683 if (!exporter->active) {
7684 if (cfg->ctime - exporter->last_restart_ms > MD_RESTART_MS) {
7685 if (!mdExporterRestart(cfg, exporter, err)) {
7686 g_message("Error restarting exporter %s: %s",
7687 exporter->name, (*err)->message);
7688 g_clear_error(err);
7689 return TRUE;
7690 }
7691 } else {
7692 return TRUE;
7693 }
7694
7695 }
7696
7697 if (exporter->fbuf) {
7698
7699 if (!mdExporterfBufSetup(cfg, exporter, NULL, err,
7700 mdInitExporterSessionSSLDedupOnly, tid, tid))
7701 {
7702 return FALSE;
7703 }
7704
7705 if (!fBufAppend(exporter->fbuf, (uint8_t *)rec, rec_length, err)) {
7706 fBufFree(exporter->fbuf);
7707 goto err;
7708 }
7709 /* update stats */
7710 exporter->exp_bytes += rec_length;
7711
7712 }
7713
7714 if (exporter->type == TEXT) {
7715
7716 if (exporter->rotate) {
7717 if ((cfg->ctime - exporter->last_rotate_ms) > exporter->rotate) {
7718 if (!mdTextFileRotate(exporter, cfg->ctime, err)) {
7719 exporter->last_rotate_ms = 0;
7720 goto err;
7721 }
7722 }
7723 }
7724
7725 if (exporter->json) {
7726 ret = mdJsonifySSLDedupRecord(exporter->lfp, exporter->buf, rec,
7727 err);
7728 } else {
7729 ret = mdPrintSSLDedupRecord(exporter->lfp, exporter->buf, rec,
7730 exporter->delimiter, err);
7731 }
7732
7733 if (ret < 0) {
7734 goto err;
7735 } else if (ret == 0) {
7736 if (!mdExporterExpandBuf(exporter)) {
7737 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_MEM,
7738 "Error allocating memory for exporter %s",
7739 exporter->name);
7740 return FALSE;
7741 }
7742 /* try again with expanded buffer */
7743 if (exporter->json) {
7744 ret = mdJsonifySSLDedupRecord(exporter->lfp, exporter->buf,
7745 rec, err);
7746 } else {
7747 ret = mdPrintSSLDedupRecord(exporter->lfp, exporter->buf, rec,
7748 exporter->delimiter, err);
7749 }
7750 if (ret < 0) {
7751 goto err;
7752 } else if (ret == 0) {
7753 g_set_error(err, MD_ERROR_DOMAIN, MD_ERROR_IO,
7754 "Error writing to buffer for exporter %s",
7755 exporter->name);
7756 goto err;
7757 }
7758 }
7759 exporter->exp_bytes += ret;
7760 }
7761
7762 ++(exporter->exp_flows);
7763
7764 return TRUE;
7765
7766 err:
7767
7768 g_warning("Error writing SSL Dedup Record: %s", (*err)->message);
7769 g_clear_error(err);
7770 g_warning("Deactivating Exporter %s.", exporter->name);
7771 exporter->active = FALSE;
7772 if (!mdExporterRestart(cfg, exporter, err)) {
7773 g_warning("Error restarting exporter %s: %s",
7774 exporter->name, (*err)->message);
7775 g_clear_error(err);
7776 }
7777
7778 return TRUE;
7779 }
7780