1 /**
2 * @file mediator_dedup.c
3 *
4 * deduplication code.
5 *
6 *
7 ** ------------------------------------------------------------------------
8 ** Copyright (C) 2006-2018 Carnegie Mellon University. All Rights Reserved.
9 * -----------------------------------------------------------
10 * Authors: Emily Sarneso <netsa-help@cert.org>
11 * -----------------------------------------------------------
12 * @OPENSOURCE_HEADER_START@
13 * Use of this (and related) source code is subject to the terms
14 * of the following licenses:
15 *
16 * GNU Public License (GPL) Rights pursuant to Version 2, June 1991
17 * Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013
18 *
19 *
20 * This material is based upon work funded and supported by
21 * the Department of Defense under Contract FA8721-05-C-0003 with
22 * Carnegie Mellon University for the operation of the Software Engineering
23 * Institue, a federally funded research and development center. Any opinions,
24 * findings and conclusions or recommendations expressed in this
25 * material are those of the author(s) and do not
26 * necessarily reflect the views of the United States
27 * Department of Defense.
28 *
29 * NO WARRANTY
30 *
31 * THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE
32 * MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY
33 * MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED
34 * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
35 * FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
36 * OBTAINED FROM THE USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY
37 * DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM
38 * PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
39 *
40 * This material has been approved for public release and unlimited
41 * distribution.
42 *
43 * Carnegie Mellon®, CERT® and CERT Coordination Center® are
44 * registered marks of Carnegie Mellon University.
45 *
46 * DM-0001877
47 *
48 * Carnegie Mellon University retains
49 * copyrights in all material produced under this contract. The U.S.
50 * Government retains a non-exclusive, royalty-free license to publish or
51 * reproduce these documents, or allow others to do so, for U.S.
52 * Government purposes only pursuant to the copyright license under the
53 * contract clause at 252.227.7013.
54 *
55 * Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
56 * Mellon University, its trustees, officers, employees, and agents from
57 * all claims or demands made against them (and any related losses,
58 * expenses, or attorney's fees) arising out of, or relating to Licensee's
59 * and/or its sub licensees' negligent use or willful misuse of or
60 * negligent conduct or willful misconduct regarding the Software,
61 * facilities, or other rights or assistance granted by Carnegie Mellon
62 * University under this License, including, but not limited to, any
63 * claims of product liability, personal injury, death, damage to
64 * property, or violation of any laws or regulations.
65 *
66 * @OPENSOURCE_HEADER_END@
67 * -----------------------------------------------------------
68 */
69
70 #include "mediator_dedup.h"
71 #include <mediator/mediator_core.h>
72 #include <mediator/mediator_inf.h>
73 #include "mediator_print.h"
74
75 #define DEDUP_DEBUG 0
76 #define CERT_PEN 6871
77
78 #define FBSTLNEXT(a, b) fbSubTemplateListGetNextPtr(a, b)
79 #define FBBLNEXT(a, b) fbBasicListGetIndexedDataPtr(a, b)
80
81 #define SSL_SERIAL_IE 244
82 #define SSL_COMMON_NAME 3
83 #define SSL_ORG_UNIT 11
84
85 #define MD_APPEND_CHAR(_buf_, _ch_) \
86 *(_buf_->cp) = _ch_; \
87 ++(_buf_->cp);
88
89
90 static fbInfoElementSpec_t md_dedup_spec_add[] = {
91 /* Millisecond first seen and last seen (epoch) (native time) */
92 { "monitoringIntervalStartMilliSeconds", 8, 0 },
93 { "monitoringIntervalEndMilliSeconds", 8, 0 },
94 { "flowStartMilliseconds", 8, 0 },
95 { "observedDataTotalCount", 8, 0 },
96 { "sourceIPv6Address", 16, 0 },
97 { "sourceIPv4Address", 4, 0 },
98 { "yafFlowKeyHash", 4, 0 },
99 { "observationDomainName", FB_IE_VARLEN, 0 },
100 FB_IESPEC_NULL
101 };
102
103
md_dedup_basic_list(fbBasicList_t * bl,mdBuf_t * buf,GString * tstr,char delim,gboolean hex,gboolean escape)104 gboolean md_dedup_basic_list(
105 fbBasicList_t *bl,
106 mdBuf_t *buf,
107 GString *tstr,
108 char delim,
109 gboolean hex,
110 gboolean escape)
111 {
112
113 uint16_t k = 0;
114 fbVarfield_t *var = NULL;
115 fbVarfield_t *varlist[100];
116 int hits[100];
117 int varnum = 0;
118 int w = 0;
119 gboolean found;
120 char hexdump[65534];
121 size_t hexlen = sizeof(hexdump);
122 int ret;
123 size_t brem = (buf->buflen - (buf->cp - buf->buf));
124
125 if (bl->numElements < 2) {
126 /* not exciting - just add hit count and done */
127 g_string_append_printf(tstr, "1%c", delim);
128 mdPrintBasicList(buf, tstr, bl, delim, hex, escape);
129 return TRUE;
130 }
131
132 for (k = 0; k < 100; k++) {
133 hits[k] = 0;
134 }
135
136 varlist[varnum] = (fbVarfield_t *)FBBLNEXT(bl, 0);
137 hits[varnum] = 1;
138 varnum++;
139
140 for (k = 1; (var = (fbVarfield_t *)FBBLNEXT(bl, k)); k++) {
141 found = FALSE;
142
143 if (var->len == 0) {
144 continue;
145 }
146
147 for (w = 0; w < varnum; w++) {
148 if (var->len != varlist[w]->len) {
149 continue;
150 } else {
151 if (memcmp(var->buf, varlist[w]->buf, var->len) == 0) {
152 hits[w]++;
153 found = TRUE;
154 break;
155 }
156 }
157 }
158 if (!found) {
159 varlist[varnum] = var;
160 hits[varnum] = 1;
161 varnum++;
162 }
163 }
164
165 for (k = 0; k < varnum; k++) {
166 md_util_append_gstr(buf, &brem, tstr);
167 if (!mdPrintDecimal(buf, &brem, delim, hits[k])) {
168 return FALSE;
169 }
170 if (hex) {
171 ret = md_util_hexdump_append(hexdump, &hexlen, varlist[k]->buf,
172 varlist[k]->len);
173 if (!ret) return FALSE;
174 if (!md_util_append_buffer(buf, &brem, (uint8_t*)hexdump, ret)) {
175 return FALSE;
176 }
177 } else {
178 if (escape) {
179 if (!mdPrintEscapeChars(buf, &brem, varlist[k]->buf,
180 varlist[k]->len, delim))
181 {
182 return FALSE;
183 }
184 } else {
185 if (!md_util_append_buffer(buf, &brem, varlist[k]->buf,
186 varlist[k]->len))
187 {
188 return FALSE;
189 }
190 }
191 MD_APPEND_CHAR(buf, '\n');
192 }
193 }
194 return TRUE;
195 }
196
md_dedup_basic_list_no_count(fbBasicList_t * bl,char delim,gboolean quote,gboolean hex,gboolean escape)197 GString *md_dedup_basic_list_no_count(
198 fbBasicList_t *bl,
199 char delim,
200 gboolean quote,
201 gboolean hex,
202 gboolean escape)
203 {
204
205 uint16_t k = 1;
206 fbVarfield_t *var = NULL;
207 fbVarfield_t *varlist[100];
208 int varnum = 0;
209 int w = 0;
210 gboolean found;
211 char hexdump[65534];
212 size_t hexlen = sizeof(hexdump);
213 GString *str = NULL;
214
215 var = (fbVarfield_t *)FBBLNEXT(bl, 0);
216 if (var) {
217 varlist[varnum] = var;
218 varnum++;
219 } else {
220 return NULL;
221 }
222
223 var = NULL;
224 str = g_string_new("");
225
226 for (k = 1; (var = (fbVarfield_t *)FBBLNEXT(bl, k)); k++) {
227 found = FALSE;
228
229 if (var->len == 0) {
230 continue;
231 }
232
233 for (w = 0; w < varnum; w++) {
234 if (var->len != varlist[w]->len) {
235 continue;
236 } else {
237 if (memcmp(var->buf, varlist[w]->buf, var->len) == 0) {
238 found = TRUE;
239 break;
240 }
241 }
242 }
243 if (!found) {
244 varlist[varnum] = var;
245 varnum++;
246 }
247 }
248
249 for (k = 0; k < varnum; k++) {
250 if (quote) {
251 g_string_append_printf(str, "\"");
252 }
253 if (hex) {
254 w = md_util_hexdump_append(hexdump, &hexlen, varlist[k]->buf,
255 varlist[k]->len);
256 if (!w) return FALSE;
257 g_string_append_len(str, (gchar *)hexdump, w);
258 } else {
259 if (escape) {
260 if (quote) {
261 mdPrintEscapeStrChars(str, varlist[k]->buf,
262 varlist[k]->len, '"');
263 } else {
264 mdPrintEscapeStrChars(str, varlist[k]->buf, varlist[k]->len,
265 delim);
266 }
267 } else {
268 g_string_append_len(str, (gchar *)varlist[k]->buf,
269 varlist[k]->len);
270 }
271 }
272 if (quote) {
273 g_string_append_printf(str, "\"%c", delim);
274 } else {
275 g_string_append_printf(str, "%c", delim);
276 }
277 }
278 if (str->len) {
279 /* remove last delimiter */
280 g_string_truncate(str, str->len-1);
281 }
282
283 return str;
284
285 }
286
md_dedup_print_stats(md_dedup_state_t * state,char * exp_name)287 void md_dedup_print_stats(
288 md_dedup_state_t *state,
289 char *exp_name)
290 {
291 if (state->stats.recvd == 0) {
292 g_message("Exporter %s: %"PRIu64" Records, %"PRIu64" flushed",
293 exp_name, state->stats.recvd, state->stats.flushed);
294 return;
295 }
296
297 g_message("Exporter %s: %"PRIu64" Records, %"PRIu64" flushed"
298 "(%2.2f%% compression)", exp_name, state->stats.recvd,
299 state->stats.flushed, (1-(((double)state->stats.flushed)/
300 ((double)state->stats.recvd)))*100);
301 }
302
md_dedup_ssl_decrement_cert(md_dedup_state_t * state,md_dedup_ssl_str_node_t * node)303 static void md_dedup_ssl_decrement_cert(
304 md_dedup_state_t *state,
305 md_dedup_ssl_str_node_t *node)
306 {
307 char temp[4092];
308 smVarHashKey_t lookup;
309
310 --(node->cert1->count);
311
312 if (node->cert1->count == 0) {
313 if ((node->cert1->issuer_len + node->cert1->serial_len) < 4092) {
314 memcpy(temp, node->cert1->serial, node->cert1->serial_len);
315 memcpy(temp + node->cert1->serial_len, node->cert1->issuer,
316 node->cert1->issuer_len);
317 lookup.val = (uint8_t*)temp;
318 lookup.len = node->cert1->serial_len + node->cert1->issuer_len;
319 g_hash_table_remove(state->cert_table, &lookup);
320 g_slice_free1(node->cert1->issuer_len, node->cert1->issuer);
321 g_slice_free1(node->cert1->serial_len, node->cert1->serial);
322 g_slice_free(md_dedup_ssl_node_t, node->cert1);
323 }
324 }
325
326 if (node->cert2) {
327 --(node->cert2->count);
328 if (node->cert2->count == 0) {
329 if ((node->cert2->issuer_len + node->cert2->serial_len) < 4092) {
330 memcpy(temp, node->cert2->serial, node->cert2->serial_len);
331 memcpy(temp + node->cert2->serial_len, node->cert2->issuer,
332 node->cert2->issuer_len);
333 lookup.val = (uint8_t *)temp;
334 lookup.len = node->cert2->serial_len + node->cert2->issuer_len;
335 g_hash_table_remove(state->cert_table, &lookup);
336 g_slice_free1(node->cert2->issuer_len, node->cert2->issuer);
337 g_slice_free1(node->cert2->serial_len, node->cert2->serial);
338 g_slice_free(md_dedup_ssl_node_t, node->cert2);
339 }
340 }
341 }
342
343
344 }
345
346
347
md_dedup_flush_queue(md_export_node_t * exp,mdConfig_t * cfg,GError ** err)348 gboolean md_dedup_flush_queue(
349 md_export_node_t *exp,
350 mdConfig_t *cfg,
351 GError **err)
352 {
353 md_dedup_node_t *node = NULL;
354 md_dedup_state_t *state = exp->dedup;
355 md_dedup_cqueue_t *cq = state->cq;
356
357 if (cq == NULL) {
358 return TRUE;
359 }
360
361 while ((node = detachFromEndOfDLL((mdDLL_t **)&(cq->head),
362 (mdDLL_t **)&(cq->tail))))
363 {
364 if (!mdExporterDedupFileOpen(cfg, exp->exp, &(node->ietab->out_file),
365 &(node->ietab->last_file),
366 node->ietab->file_prefix,
367 &(node->ietab->last_rotate_ms)))
368 {
369 return FALSE;
370 }
371 #if DEDUP_DEBUG
372 g_debug("flushing queue node: %p, node->ietab %p, node->strnode %p",
373 node, node->ietab, node->strnode);
374 g_debug("file->prefix %s, intid %02x, extid %02x", node->ietab->file_prefix,
375 node->ietab->tmpl->intid, node->ietab->tmpl->extid);
376 #endif
377 if (state->add_export) {
378 if (node->exnode.mapname.len == 0) {
379 node->exnode.mapname.buf = (uint8_t*)mdExporterGetName(exp->exp);
380 node->exnode.mapname.len = strlen(mdExporterGetName(exp->exp));
381 }
382 }
383
384 if (!mdExporterWriteDedupRecord(cfg, exp, node->ietab->out_file,
385 &(node->exnode), node->ietab->file_prefix,
386 node->ietab->tmpl->intid, node->ietab->tmpl->extid, err))
387 {
388 return FALSE;
389 }
390
391 if (!node->ietab->ssl) {
392 g_slice_free1(node->strnode->caplen, node->strnode->data);
393 g_slice_free(md_dedup_str_node_t, node->strnode);
394 } else {
395 md_dedup_ssl_decrement_cert(state, (md_dedup_ssl_str_node_t*)node->strnode);
396 g_slice_free(md_dedup_ssl_str_node_t,
397 (md_dedup_ssl_str_node_t *)node->strnode);
398 /* check count on certs... free certs if necessary */
399 }
400 g_slice_free(md_dedup_node_t, node);
401
402 state->stats.flushed++;
403 }
404
405 return TRUE;
406 }
407
408
409
md_dedup_new_dedup_state()410 md_dedup_state_t *md_dedup_new_dedup_state(
411 )
412 {
413 md_dedup_state_t *state = g_slice_new0(md_dedup_state_t);
414
415 state->ie_table = g_hash_table_new((GHashFunc)g_direct_hash,
416 (GEqualFunc)g_direct_equal);
417
418 state->cq = g_slice_new0(md_dedup_cqueue_t);
419
420 /* set defaults */
421 state->max_hit_count = 5000;
422 state->flush_timeout = 300 * 1000;
423
424 #if DEDUP_DEBUG
425 fprintf(stderr, "created new dedup state %p\n", state->ie_table);
426 #endif
427
428 return state;
429
430 }
431
md_dedup_add_templates(md_dedup_state_t * state,fBuf_t * fbuf,GError ** err)432 gboolean md_dedup_add_templates(
433 md_dedup_state_t *state,
434 fBuf_t *fbuf,
435 GError **err)
436 {
437 md_dedup_tmpl_t *tnode = NULL;
438 fbSession_t *session = fBufGetSession(fbuf);
439 md_dedup_ie_t *tn = NULL, *cn = NULL;
440 fbInfoModel_t *sm_model = mdInfoModel();
441 const fbInfoElement_t *ie_to_add = NULL;
442 fbInfoElementSpec_t md_dedup_ie_spec;
443
444 for (tn = state->head; tn; tn = cn) {
445 cn = tn->next;
446 tnode = tn->tmpl;
447
448 /* really should have configureable PEN */
449 ie_to_add = fbInfoModelGetElementByID(sm_model, tnode->ie, CERT_PEN);
450
451 if (tn->ssl == FALSE) {
452 tnode->tmpl = fbTemplateAlloc(sm_model);
453
454 if (!fbTemplateAppendSpecArray(tnode->tmpl, md_dedup_spec_add,
455 0xffffffff, err))
456 {
457 return FALSE;
458 }
459
460 md_dedup_ie_spec.name = (char *)ie_to_add->ref.name;
461 md_dedup_ie_spec.len_override = 0;
462 md_dedup_ie_spec.flags = 0;
463
464 if (!fbTemplateAppendSpec(tnode->tmpl, &md_dedup_ie_spec, 0, err))
465 {
466 return FALSE;
467 }
468
469 tnode->intid = fbSessionAddTemplate(session, TRUE, tnode->intid,
470 tnode->tmpl, err);
471 if (tnode->intid == 0) {
472 return FALSE;
473 }
474
475 tnode->extid = fbSessionAddTemplate(session, FALSE, tnode->extid,
476 tnode->tmpl, err);
477 if (tnode->extid == 0) {
478 return FALSE;
479 }
480 } else {
481 tnode->intid = MD_DEDUP_FULL;
482 tnode->extid = MD_DEDUP_FULL;
483 }
484 }
485
486 /* if (!fbSessionExportTemplates(session, err)) {
487 return FALSE;
488 }*/
489
490 return TRUE;
491 }
492
md_dedup_reset(md_export_node_t * exp,uint64_t ctime)493 static void md_dedup_reset(
494 md_export_node_t *exp,
495 uint64_t ctime)
496 {
497 g_warning("Potentially out of memory for deduplication."
498 " Resetting all tables.");
499 md_dedup_flush_alltab(exp, ctime, TRUE);
500 }
501
md_dedup_configure_state(md_dedup_state_t * state,int max_hit_count,int flush_timeout,gboolean merge_truncated,gboolean add_export)502 void md_dedup_configure_state(
503 md_dedup_state_t *state,
504 int max_hit_count,
505 int flush_timeout,
506 gboolean merge_truncated,
507 gboolean add_export)
508 {
509 if (max_hit_count) {
510 state->max_hit_count = max_hit_count;
511 }
512
513 if (flush_timeout) {
514 state->flush_timeout = flush_timeout * 1000;
515 }
516
517 if (merge_truncated) {
518 state->merge = merge_truncated;
519 }
520
521 if (add_export) {
522 state->add_export = add_export;
523 }
524 }
525
md_dedup_new_str_node(uint8_t * data,size_t caplen,uint64_t time,uint32_t hash,uint64_t stime)526 static md_dedup_str_node_t *md_dedup_new_str_node(
527 uint8_t *data,
528 size_t caplen,
529 uint64_t time,
530 uint32_t hash,
531 uint64_t stime)
532 {
533 md_dedup_str_node_t *stn;
534
535 stn = g_slice_new0(md_dedup_str_node_t);
536 if (stn == NULL) {
537 return NULL;
538 }
539
540 stn->ftime = time;
541 stn->ltime = time;
542 stn->hitcount = 1;
543 stn->hash = hash;
544 stn->stime = stime;
545
546 stn->data = g_slice_alloc0(caplen);
547 if (stn->data == NULL) {
548 return NULL;
549 }
550 memcpy(stn->data, data, caplen);
551 stn->caplen = caplen;
552
553 return stn;
554 }
555
md_dedup_ie_lookup(md_dedup_state_t * state,uint16_t ie)556 static md_dedup_ie_t *md_dedup_ie_lookup(
557 md_dedup_state_t *state,
558 uint16_t ie)
559 {
560
561 md_dedup_ie_t *ret = NULL;
562
563 ret = g_hash_table_lookup(state->ie_table, GUINT_TO_POINTER((unsigned int)ie));
564
565 return ret;
566 }
567
md_dedup_add_ie(md_dedup_state_t * state,md_dedup_ie_t * ie_tab,uint16_t ie)568 void md_dedup_add_ie(
569 md_dedup_state_t *state,
570 md_dedup_ie_t *ie_tab,
571 uint16_t ie)
572 {
573 #if DEDUP_DEBUG
574 fprintf(stderr, "add ie %d to ietab %p\n", ie, ie_tab);
575 #endif
576 g_hash_table_insert(state->ie_table, GUINT_TO_POINTER((unsigned int)ie), ie_tab);
577 }
578
579
md_dedup_add_ie_table(md_dedup_state_t * state,char * prefix,smFieldMap_t * map,uint16_t ie,int sip)580 md_dedup_ie_t *md_dedup_add_ie_table(
581 md_dedup_state_t *state,
582 char *prefix,
583 smFieldMap_t *map,
584 uint16_t ie,
585 int sip)
586 {
587
588 md_dedup_ie_t *ie_tab = NULL;
589 md_dedup_tmpl_t *dedup_tmpl = NULL;
590
591 #if DEDUP_DEBUG
592 fprintf(stderr, "state->ie_table is %p\n", state->ie_table);
593 #endif
594
595 if ((ie_tab = g_hash_table_lookup(state->ie_table,
596 GUINT_TO_POINTER((unsigned int)ie))))
597 {
598 /* already exists */
599 return NULL;
600 }
601
602 ie_tab = g_slice_new0(md_dedup_ie_t);
603
604 if (map) {
605 ie_tab->ip_table = smCreateHashTable(sizeof(uint32_t) + sizeof(uint32_t), NULL, NULL);
606 ie_tab->ip6_table = smCreateHashTable(sizeof(uint32_t) + sizeof(uint8_t[16]), NULL, NULL);
607 /* ie_tab->ip_table = g_hash_table_new((GHashFunc)g_direct_hash,
608 (GEqualFunc)g_direct_equal);
609
610 ie_tab->ip6_table = g_hash_table_new_full((GHashFunc)sm_octet_array_hash,
611 (GEqualFunc)sm_octet_array_equal,
612 sm_octet_array_key_destroy,
613 NULL);*/
614 } else {
615 ie_tab->ip_table = smCreateHashTable(sizeof(uint32_t), NULL, NULL);
616 ie_tab->ip6_table = smCreateHashTable(sizeof(uint8_t[16]), NULL, NULL);
617 }
618
619 ie_tab->file_prefix = g_strdup(prefix);
620
621 ie_tab->sip = sip;
622
623 ie_tab->map = map;
624
625 g_hash_table_insert(state->ie_table, GUINT_TO_POINTER((unsigned int)ie),
626 ie_tab);
627
628 attachHeadToDLL((mdDLL_t **)&(state->head),
629 (mdDLL_t **)&(state->tail),
630 (mdDLL_t *)ie_tab);
631
632 /* if ie == serial #, then set up SSL state */
633 if (ie == SSL_SERIAL_IE) {
634 state->cert_table = g_hash_table_new_full((GHashFunc)sm_octet_array_hash,
635 (GEqualFunc)sm_octet_array_equal,
636 sm_octet_array_key_destroy, NULL);
637 ie_tab->ssl = TRUE;
638 }
639
640 dedup_tmpl = g_slice_new0(md_dedup_tmpl_t);
641
642 dedup_tmpl->ie = ie;
643
644 ie_tab->tmpl = dedup_tmpl;
645
646 return ie_tab;
647 }
648
md_dedup_ip_node_close(md_dedup_ie_t * ietab,md_dedup_ip_node_t * ipnode)649 static void md_dedup_ip_node_close(
650 md_dedup_ie_t *ietab,
651 md_dedup_ip_node_t *ipnode)
652 {
653
654 if (ipnode->sip6_key) {
655 /*g_hash_table_remove(ietab->ip6_table, ipnode->sip6_key);*/
656 smHashTableRemove(ietab->ip6_table, (uint8_t*)(ipnode->sip6_key));
657 g_slice_free(mdMapKey6_t, ipnode->sip6_key);
658 } else {
659 smHashTableRemove(ietab->ip_table, (uint8_t*)(ipnode->sip_key));
660 g_slice_free(mdMapKey4_t, ipnode->sip_key);
661 /*g_hash_table_remove(ietab->ip_table, GUINT_TO_POINTER(ipnode->sip_key));*/
662 }
663
664 detachThisEntryOfDLL((mdDLL_t **)&(ietab->head),
665 (mdDLL_t **)&(ietab->tail),
666 (mdDLL_t *)ipnode);
667 #if DEDUP_DEBUG
668 g_debug("REMOVE IPNODE %u", ipnode->sip_key);
669 #endif
670
671 if (ietab->ssl) {
672 g_slice_free(md_dedup_ssl_ip_node_t, (md_dedup_ssl_ip_node_t*)ipnode);
673 } else {
674 g_slice_free(md_dedup_ip_node_t, ipnode);
675 }
676 --(ietab->count);
677 }
678
md_dedup_str_node_close(md_export_node_t * exp,md_dedup_ie_t * ietab,md_dedup_ip_node_t * ipnode,md_dedup_str_node_t * strnode)679 static void md_dedup_str_node_close(
680 md_export_node_t *exp,
681 md_dedup_ie_t *ietab,
682 md_dedup_ip_node_t *ipnode,
683 md_dedup_str_node_t *strnode)
684 {
685 md_dedup_cqueue_t *cq = exp->dedup->cq;
686 md_dedup_node_t *cn = g_slice_new0(md_dedup_node_t);
687
688 if (cn == NULL) {
689 g_warning("Error allocating md_dedup_node.");
690 return;
691 }
692 #if DEDUP_DEBUG
693 fprintf(stderr, "CLOSING STRNODE %p\n", strnode);
694 #endif
695 cn->strnode = strnode;
696 cn->ietab = ietab;
697 cn->exnode.monitoringIntervalStartMilliSeconds = strnode->ftime;
698 cn->exnode.monitoringIntervalEndMilliSeconds = strnode->ltime;
699 cn->exnode.observedDataTotalCount = strnode->hitcount;
700 cn->exnode.yafFlowKeyHash = strnode->hash;
701 cn->exnode.flowStartMilliseconds = strnode->stime;
702 if (ipnode->sip_key) {
703 cn->exnode.sourceIPv4Address = ipnode->sip_key->ip;
704 }
705 if (ipnode->sip6_key) {
706 /* memcpy(cn->exnode.sip6, ipnode->sip6_key->val, 16); */
707 memcpy(cn->exnode.sourceIPv6Address, ipnode->sip6_key, 16);
708 }
709
710 if (ietab->map) {
711 int mapindex = 0;
712 if (ipnode->sip_key) {
713 mapindex = ipnode->sip_key->map;
714 } else {
715 /*mapindex = ((mdMapKey6_t*)(ipnode->sip6_key->val))->map;*/
716 mapindex = ((mdMapKey6_t*)(ipnode->sip6_key))->map;
717 }
718 #if DEDUP_DEBUG
719 g_debug("maps on %s", ietab->map->labels[mapindex]);
720 #endif
721 cn->exnode.mapname.buf = (uint8_t*)(ietab->map->labels[mapindex]);
722 cn->exnode.mapname.len = strlen(ietab->map->labels[mapindex]);
723 }
724
725 if (!ietab->ssl) {
726 cn->exnode.observedData.buf = strnode->data;
727 cn->exnode.observedData.len = strnode->caplen;
728 } else {
729 md_dedup_ssl_str_node_t *ssl = (md_dedup_ssl_str_node_t *)strnode;
730 cn->exnode.sslCertSerialNumber1.buf = ssl->cert1->serial;
731 cn->exnode.sslCertSerialNumber1.len = ssl->cert1->serial_len;
732 cn->exnode.sslCertIssuerCommonName1.buf = ssl->cert1->issuer;
733 cn->exnode.sslCertIssuerCommonName1.len = ssl->cert1->issuer_len;
734 if (ssl->cert2) {
735 cn->exnode.sslCertSerialNumber2.buf = ssl->cert2->serial;
736 cn->exnode.sslCertSerialNumber2.len= ssl->cert2->serial_len;
737 cn->exnode.sslCertIssuerCommonName2.buf= ssl->cert2->issuer;
738 cn->exnode.sslCertIssuerCommonName2.len= ssl->cert2->issuer_len;
739 }
740 }
741
742 attachHeadToDLL((mdDLL_t **)&(cq->head),
743 (mdDLL_t **)&(cq->tail),
744 (mdDLL_t *)cn);
745
746 detachThisEntryOfDLL((mdDLL_t**)&(ipnode->head),
747 (mdDLL_t**)&(ipnode->tail),
748 (mdDLL_t*)strnode);
749
750 if (!ipnode->head) {
751 md_dedup_ip_node_close(ietab, ipnode);
752 }
753
754 }
755
756
md_dedup_str_node_tick(md_export_node_t * exp,md_dedup_ie_t * ietab,md_dedup_ip_node_t * ipnode,md_dedup_str_node_t * strnode)757 static void md_dedup_str_node_tick(
758 md_export_node_t *exp,
759 md_dedup_ie_t *ietab,
760 md_dedup_ip_node_t *ipnode,
761 md_dedup_str_node_t *strnode)
762 {
763 if (ipnode->head != strnode) {
764 if (strnode->prev != NULL) {
765 detachThisEntryOfDLL((mdDLL_t**)&(ipnode->head),
766 (mdDLL_t**)&(ipnode->tail),
767 (mdDLL_t*)strnode);
768 }
769 attachHeadToDLL((mdDLL_t **)&(ipnode->head),
770 (mdDLL_t **)&(ipnode->tail),
771 (mdDLL_t *)strnode);
772 }
773
774 while (ipnode->tail && ((strnode->ltime - ipnode->tail->ltime) > exp->dedup->flush_timeout)) {
775 md_dedup_str_node_close(exp, ietab, ipnode, ipnode->tail);
776 }
777 }
778
md_dedup_ip_node_tick(md_dedup_ie_t * ietab,md_dedup_ip_node_t * ipnode)779 static void md_dedup_ip_node_tick(
780 md_dedup_ie_t *ietab,
781 md_dedup_ip_node_t *ipnode)
782 {
783 if (ietab->head != ipnode) {
784 if (ipnode->prev != NULL) {
785 detachThisEntryOfDLL((mdDLL_t **)&(ietab->head),
786 (mdDLL_t **)&(ietab->tail), (mdDLL_t *)ipnode);
787 }
788
789 attachHeadToDLL((mdDLL_t **)&(ietab->head),
790 (mdDLL_t **)&(ietab->tail),
791 (mdDLL_t *)ipnode);
792 }
793 }
794
md_dedup_add_node(mdContext_t * ctx,md_export_node_t * exp,md_dedup_ie_t * ietab,uint8_t * data,size_t datalen,uint16_t ie,mdFullFlow_t * flow,gboolean rev)795 static void md_dedup_add_node(
796 mdContext_t *ctx,
797 md_export_node_t *exp,
798 md_dedup_ie_t *ietab,
799 uint8_t *data,
800 size_t datalen,
801 uint16_t ie,
802 mdFullFlow_t *flow,
803 gboolean rev)
804 {
805
806 md_dedup_state_t *state = exp->dedup;
807 md_dedup_ip_node_t *ipnode = NULL;
808 md_dedup_str_node_t *strnode = NULL, *cn = NULL, *tn = NULL;
809 size_t cmpsize = datalen;
810 int rc;
811 uint32_t sip = 0;
812 uint32_t hash = md_util_flow_key_hash(flow->rec);
813 mdMapKey4_t mapkey4;
814 mdMapKey6_t mapkey6;
815 gboolean v6 = FALSE;
816
817 mapkey4.map = 0;
818 mapkey6.map = 0;
819
820 if (datalen == 0 || !data) {
821 /* no data to add */
822 return;
823 }
824
825 if (ietab->sip == 2) {
826 mapkey4.ip = hash;
827 sip = hash;
828 } else {
829
830 if (flow->rec->sourceIPv4Address || flow->rec->destinationIPv4Address) {
831 if (rev || (!rev && !ietab->sip)) {
832 mapkey4.ip = flow->rec->destinationIPv4Address;
833 sip = flow->rec->destinationIPv4Address;
834 } else {
835 mapkey4.ip = flow->rec->sourceIPv4Address;
836 sip = flow->rec->sourceIPv4Address;
837 }
838 } else {
839 if (rev || (!rev && !ietab->sip)) {
840 memcpy(mapkey6.ip, flow->rec->destinationIPv6Address, 16);
841 } else {
842 memcpy(mapkey6.ip, flow->rec->sourceIPv6Address, 16);
843
844 }
845 v6 = TRUE;
846 }
847 }
848
849 if (ietab->map) {
850 mapkey4.map = smFieldMapTranslate(ietab->map, flow);
851 if (ietab->map->discard && (mapkey4.map == 0)) {
852 return;
853 }
854 mapkey6.map = mapkey4.map;
855 }
856
857 if (v6) {
858 ipnode = smHashLookup(ietab->ip6_table, (uint8_t*)&mapkey6);
859 } else {
860 ipnode = smHashLookup(ietab->ip_table, (uint8_t*)&mapkey4);
861 #if DEDUP_DEBUG
862 g_debug("looking up sip %08x %p %04x - returned %p", mapkey4, ietab->ip_table, mapkey4.ip, ipnode);
863 #endif
864 }
865
866
867 if (ipnode) {
868 for (tn = ipnode->head; tn; tn = cn) {
869 cn = tn->next;
870 if (ie != tn->ie) {
871 continue;
872 }
873 if (!state->merge) {
874 /* not merging truncated fields - so if lengths don't match,
875 continue */
876 if (datalen != tn->caplen) {
877 continue;
878 }
879 } else {
880 cmpsize = datalen < tn->caplen ? datalen : tn->caplen;
881 }
882 rc = memcmp(tn->data, data, cmpsize);
883 if (!rc) {
884 state->stats.recvd++;
885 ++(tn->hitcount);
886 tn->hash = hash;
887 tn->stime = flow->rec->flowStartMilliseconds;
888 tn->ltime = ctx->cfg->ctime;
889 if (tn->hitcount == state->max_hit_count) {
890 md_dedup_str_node_close(exp, ietab, ipnode, tn);
891 } else {
892 md_dedup_str_node_tick(exp, ietab, ipnode, tn);
893 md_dedup_ip_node_tick(ietab, ipnode);
894 }
895 return;
896 }
897 }
898 } else {
899 /* IP address not found for this IE */
900 ipnode = g_slice_new0(md_dedup_ip_node_t);
901 if (ipnode == NULL) {
902 md_dedup_reset(exp, ctx->cfg->ctime);
903 return;
904 }
905 if (v6) {
906 ipnode->sip6_key = g_slice_new0(mdMapKey6_t);
907 memcpy(ipnode->sip6_key, &mapkey6, sizeof(mdMapKey6_t));
908 smHashTableInsert(ietab->ip6_table, (uint8_t*)ipnode->sip6_key,
909 (uint8_t*)ipnode);
910 } else {
911 ipnode->sip_key = g_slice_new0(mdMapKey4_t);
912 ipnode->sip_key->ip = sip;
913 ipnode->sip_key->map = mapkey4.map;
914 smHashTableInsert(ietab->ip_table, (uint8_t*)ipnode->sip_key,
915 (uint8_t*)ipnode);
916 /*g_hash_table_insert(ietab->ip_table,
917 GUINT_TO_POINTER((unsigned int)sip), ipnode);*/
918 }
919 ++(ietab->count);
920 }
921
922 strnode = md_dedup_new_str_node(data, datalen, ctx->cfg->ctime, hash,
923 flow->rec->flowStartMilliseconds);
924
925 if (strnode == NULL) {
926 md_dedup_reset(exp, ctx->cfg->ctime);
927 return;
928 }
929
930 strnode->ie = ie;
931
932 /* add to stats recvd count */
933 state->stats.recvd++;
934
935 md_dedup_str_node_tick(exp, ietab, ipnode, strnode);
936 md_dedup_ip_node_tick(ietab, ipnode);
937 }
938
md_dedup_free_ietab(md_export_node_t * exp,md_dedup_ie_t * ietab)939 static void md_dedup_free_ietab(
940 md_export_node_t *exp,
941 md_dedup_ie_t *ietab)
942 {
943 /*g_hash_table_destroy(ietab->ip_table);
944 g_hash_table_destroy(ietab->ip6_table);*/
945 smHashTableFree(ietab->ip_table);
946 smHashTableFree(ietab->ip6_table);
947
948 if (ietab->out_file) {
949 mdExporterDedupFileClose(exp->exp, ietab->out_file, ietab->last_file);
950 }
951
952 if (ietab->file_prefix) {
953 g_free(ietab->file_prefix);
954 }
955
956 if (ietab->tmpl) {
957 /* ietab->tmpl->tmpl should get freed when session is freed */
958 g_slice_free(md_dedup_tmpl_t, ietab->tmpl);
959 }
960
961 g_slice_free(md_dedup_ie_t, ietab);
962 }
963
md_dedup_flush_ietab(md_export_node_t * exp,md_dedup_ie_t * ietab,uint64_t ctime,gboolean flush_all)964 static void md_dedup_flush_ietab(
965 md_export_node_t *exp,
966 md_dedup_ie_t *ietab,
967 uint64_t ctime,
968 gboolean flush_all)
969 {
970 if (ietab == NULL) {
971 return;
972 }
973
974 ietab->last_flush = ctime;
975
976 while (flush_all && ietab->tail) {
977 md_dedup_str_node_close(exp, ietab, ietab->tail, ietab->tail->tail);
978 }
979
980 while (ietab->tail && (ietab->last_flush - ietab->tail->tail->ltime >
981 exp->dedup->flush_timeout))
982 {
983 md_dedup_str_node_close(exp, ietab, ietab->tail, ietab->tail->tail);
984 }
985 }
986
md_dedup_flush_alltab(md_export_node_t * exp,uint64_t ctime,gboolean flush_all)987 void md_dedup_flush_alltab(
988 md_export_node_t *exp,
989 uint64_t ctime,
990 gboolean flush_all)
991 {
992 md_dedup_state_t *state = exp->dedup;
993 md_dedup_ie_t *tn = NULL, *cn = NULL;
994
995 for (tn = state->head; tn; tn = cn) {
996 cn = tn->next;
997 md_dedup_flush_ietab(exp, tn, ctime, flush_all);
998 }
999 }
1000
md_dedup_free_state(mdConfig_t * cfg,md_export_node_t * exp,GError ** err)1001 gboolean md_dedup_free_state(
1002 mdConfig_t *cfg,
1003 md_export_node_t *exp,
1004 GError **err)
1005 {
1006 md_dedup_state_t *state = exp->dedup;
1007 md_dedup_ie_t *tn = NULL, *cn = NULL;
1008
1009 md_dedup_flush_alltab(exp, cfg->ctime, TRUE);
1010
1011 if (!md_dedup_flush_queue(exp, cfg, err)) {
1012 return FALSE;
1013 }
1014
1015 for (tn = state->head; tn; tn = cn) {
1016 cn = tn->next;
1017 md_dedup_free_ietab(exp, tn);
1018 }
1019
1020 g_hash_table_destroy(state->ie_table);
1021
1022 if (state->cert_table) {
1023 g_hash_table_destroy(state->cert_table);
1024 }
1025
1026 g_slice_free1(sizeof(md_dedup_cqueue_t), state->cq);
1027
1028 return TRUE;
1029 }
1030
1031
md_dedup_add_bl(mdContext_t * ctx,md_export_node_t * exp,md_dedup_ie_t * ietab,fbBasicList_t * bl,mdFullFlow_t * flow)1032 static void md_dedup_add_bl(
1033 mdContext_t *ctx,
1034 md_export_node_t *exp,
1035 md_dedup_ie_t *ietab,
1036 fbBasicList_t *bl,
1037 mdFullFlow_t *flow)
1038 {
1039 uint16_t w = 0;
1040 fbVarfield_t *var = NULL;
1041
1042 for (w = 0;
1043 (var = (fbVarfield_t *)fbBasicListGetIndexedDataPtr(bl, w));
1044 w++) {
1045
1046 if (var->len == 0) {
1047 continue;
1048 }
1049
1050 md_dedup_add_node(ctx, exp, ietab, var->buf, var->len,
1051 bl->infoElement->num, flow, FALSE);
1052 }
1053 }
1054
md_dedup_new_ssl_node(uint8_t * serial,size_t serial_len,uint8_t * issuer,size_t issuer_len)1055 static md_dedup_ssl_node_t *md_dedup_new_ssl_node(
1056 uint8_t *serial,
1057 size_t serial_len,
1058 uint8_t *issuer,
1059 size_t issuer_len)
1060 {
1061
1062 md_dedup_ssl_node_t *node = g_slice_new0(md_dedup_ssl_node_t);
1063
1064 node->serial = g_slice_alloc0(serial_len);
1065 memcpy(node->serial, serial, serial_len);
1066 node->serial_len = serial_len;
1067
1068 node->issuer = g_slice_alloc0(issuer_len);
1069 memcpy(node->issuer, issuer, issuer_len);
1070 node->issuer_len = issuer_len;
1071
1072 return node;
1073 }
1074
1075
1076
md_dedup_ssl_add_node(mdContext_t * ctx,md_export_node_t * exp,md_dedup_ie_t * ietab,yaf_newssl_t * ssl,mdFullFlow_t * flow)1077 static void md_dedup_ssl_add_node(
1078 mdContext_t *ctx,
1079 md_export_node_t *exp,
1080 md_dedup_ie_t *ietab,
1081 yaf_newssl_t *ssl,
1082 mdFullFlow_t *flow)
1083 {
1084 yaf_newssl_cert_t *cert = NULL;
1085 yaf_subssl_t *obj = NULL;
1086 yaf_subssl_t *ou = NULL;
1087 md_dedup_state_t *state = exp->dedup;
1088 md_dedup_ssl_node_t *cert1 = NULL, *cert2 = NULL;
1089 md_dedup_ssl_str_node_t *cn = NULL, *tn = NULL;
1090 md_dedup_ssl_str_node_t *strnode = NULL;
1091 md_dedup_ssl_ip_node_t *ipnode = NULL;
1092 smVarHashKey_t lookup;
1093 smVarHashKey_t *newkey;
1094 uint32_t hash = md_util_flow_key_hash(flow->rec);
1095 uint32_t sip;
1096 mdMapKey4_t mapkey4;
1097 mdMapKey6_t mapkey6;
1098 gboolean found;
1099 gboolean v6 = FALSE;
1100 uint8_t temp[4092];
1101 int cert_no = 0;
1102
1103 if (ietab->sip == 2) {
1104 mapkey4.ip = hash;
1105 sip = hash;
1106 } else {
1107
1108 if (flow->rec->sourceIPv4Address || flow->rec->destinationIPv4Address) {
1109 if (ietab->sip == 0) {
1110 mapkey4.ip = flow->rec->destinationIPv4Address;
1111 sip = flow->rec->destinationIPv4Address;
1112 } else {
1113 mapkey4.ip = flow->rec->sourceIPv4Address;
1114 sip = flow->rec->sourceIPv4Address;
1115 }
1116 } else {
1117 if (ietab->sip == 0) {
1118 memcpy(mapkey6.ip, flow->rec->destinationIPv6Address, 16);
1119 } else {
1120 memcpy(mapkey6.ip, flow->rec->sourceIPv6Address, 16);
1121 }
1122 v6 = TRUE;
1123 }
1124 }
1125
1126 if (ietab->map) {
1127 mapkey4.map = smFieldMapTranslate(ietab->map, flow);
1128 mapkey6.map = mapkey4.map;
1129 }
1130
1131 while ((cert = (yaf_newssl_cert_t *)FBSTLNEXT(&(ssl->sslCertList), cert)))
1132 {
1133
1134 obj = NULL;
1135 ou = NULL;
1136 if (cert->sslCertSerialNumber.len == 0) {
1137 /* no serial number */
1138 if (cert_no == 0) {
1139 return;
1140 } else {
1141 break;
1142 }
1143 }
1144
1145 found = FALSE;
1146 while ((obj = (yaf_subssl_t *)FBSTLNEXT(&(cert->issuer), obj)))
1147 {
1148
1149 if (obj->sslObjectType != SSL_COMMON_NAME) {
1150 if (obj->sslObjectType == SSL_ORG_UNIT) {
1151 /* save just in case */
1152 ou = obj;
1153 }
1154 continue;
1155 }
1156
1157 if (obj->sslObjectValue.len == 0) {
1158 continue;
1159 }
1160
1161 found = TRUE;
1162 break;
1163 }
1164
1165 if (!found) {
1166 if (ou) {
1167 obj = ou;
1168 } else return;
1169 }
1170
1171 if (cert->sslCertSerialNumber.len + obj->sslObjectValue.len < 4092) {
1172 memcpy(temp, cert->sslCertSerialNumber.buf, cert->sslCertSerialNumber.len);
1173 memcpy(temp + cert->sslCertSerialNumber.len, obj->sslObjectValue.buf,
1174 obj->sslObjectValue.len);
1175 } else {
1176 /* cut this off somehow */
1177 g_debug("COMBO serial + issuer name over 4092");
1178 return;
1179 }
1180
1181 lookup.val = temp;
1182 lookup.len = cert->sslCertSerialNumber.len + obj->sslObjectValue.len;
1183
1184 if (!cert1) {
1185 cert1 = g_hash_table_lookup(state->cert_table, &lookup);
1186
1187 if (!cert1) {
1188 /* add this cert */
1189 cert1 = md_dedup_new_ssl_node(cert->sslCertSerialNumber.buf,
1190 cert->sslCertSerialNumber.len,
1191 obj->sslObjectValue.buf,
1192 obj->sslObjectValue.len);
1193 newkey = sm_new_hash_key(lookup.val, lookup.len);
1194 g_hash_table_insert(state->cert_table, newkey, cert1);
1195
1196 }
1197 } else if (!cert2) {
1198 cert2 = g_hash_table_lookup(state->cert_table, &lookup);
1199
1200 if (!cert2) {
1201 /* add this cert */
1202 cert2 = md_dedup_new_ssl_node(cert->sslCertSerialNumber.buf,
1203 cert->sslCertSerialNumber.len,
1204 obj->sslObjectValue.buf,
1205 obj->sslObjectValue.len);
1206 newkey = sm_new_hash_key(lookup.val, lookup.len);
1207 g_hash_table_insert(state->cert_table, newkey, cert2);
1208 }
1209 }
1210
1211 if (cert1 && cert2) {
1212 break;
1213 }
1214 }
1215
1216 if (!cert1) {
1217 /* must have 1 valid cert! */
1218 return;
1219 }
1220
1221 if (v6) {
1222 ipnode = smHashLookup(ietab->ip6_table, (uint8_t*)&mapkey6);
1223 /*ipnode = g_hash_table_lookup(ietab->ip6_table, &iplookup);*/
1224 } else {
1225 ipnode = smHashLookup(ietab->ip_table, (uint8_t*)&mapkey4);
1226 #if DEDUP_DEBUG
1227 g_debug("looking up sip %u - returned %p", sip, ipnode);
1228 #endif
1229 /*ipnode = g_hash_table_lookup(ietab->ip_table,
1230 GUINT_TO_POINTER((unsigned int)sip));*/
1231 }
1232
1233 if (ipnode) {
1234 for (tn = ipnode->head; tn; tn = cn) {
1235 cn = tn->next;
1236
1237 if (cert1 != tn->cert1) {
1238 continue;
1239 }
1240 if (cert2 != tn->cert2) {
1241 continue;
1242 }
1243 /* found a match */
1244 state->stats.recvd++;
1245 ++(tn->hitcount);
1246 tn->ltime = ctx->cfg->ctime;
1247 tn->hash = hash;
1248 tn->stime = flow->rec->flowStartMilliseconds;
1249 if (tn->hitcount == state->max_hit_count) {
1250 md_dedup_str_node_close(exp, ietab, (md_dedup_ip_node_t*)ipnode,
1251 (md_dedup_str_node_t *)tn);
1252 } else {
1253 md_dedup_str_node_tick(exp, ietab, (md_dedup_ip_node_t*)ipnode,
1254 (md_dedup_str_node_t *)tn);
1255 md_dedup_ip_node_tick(ietab, (md_dedup_ip_node_t*)ipnode);
1256 }
1257 return;
1258 }
1259 } else {
1260 /* IP address not found in this table */
1261 ipnode = g_slice_new0(md_dedup_ssl_ip_node_t);
1262 if (ipnode == NULL) {
1263 md_dedup_reset(exp, ctx->cfg->ctime);
1264 return;
1265 }
1266
1267 if (v6) {
1268 ipnode->sip6_key = g_slice_new0(mdMapKey6_t);
1269 memcpy(ipnode->sip6_key, &mapkey6, sizeof(mdMapKey6_t));
1270 /*ipnode->sip6_key = sm_new_hash_key(iplookup.val, iplookup.len);*/
1271 /*g_hash_table_insert(ietab->ip6_table, ipnode->sip6_key, ipnode);*/
1272 smHashTableInsert(ietab->ip6_table, (uint8_t*)ipnode->sip6_key,
1273 (uint8_t*)ipnode);
1274 } else {
1275 ipnode->sip_key = g_slice_new0(mdMapKey4_t);
1276 ipnode->sip_key->ip = sip;
1277 ipnode->sip_key->map = mapkey4.map;
1278 smHashTableInsert(ietab->ip_table, (uint8_t*)ipnode->sip_key,
1279 (uint8_t*)ipnode);
1280 /*g_hash_table_insert(ietab->ip_table,
1281 GUINT_TO_POINTER((unsigned int)sip), ipnode);*/
1282 }
1283 ++(ietab->count);
1284 }
1285
1286 strnode = g_slice_new0(md_dedup_ssl_str_node_t);
1287 if (strnode == NULL) {
1288 md_dedup_reset(exp, ctx->cfg->ctime);
1289 return;
1290 }
1291
1292 strnode->ftime = ctx->cfg->ctime;
1293 strnode->ltime = ctx->cfg->ctime;
1294 strnode->hitcount = 1;
1295 strnode->hash = hash;
1296 strnode->stime = flow->rec->flowStartMilliseconds;
1297 strnode->cert1 = cert1;
1298 strnode->cert2 = cert2;
1299 ++(cert1->count);
1300 if (cert2) {
1301 ++(cert2->count);
1302 }
1303
1304 state->stats.recvd++;
1305
1306 md_dedup_str_node_tick(exp, ietab, (md_dedup_ip_node_t*)ipnode,
1307 (md_dedup_str_node_t*)strnode);
1308 md_dedup_ip_node_tick(ietab, (md_dedup_ip_node_t*)ipnode);
1309 }
1310
md_dedup_write_dedup(mdContext_t * ctx,md_export_node_t * exp,md_dedup_t * dedup,uint16_t ie,GError ** err)1311 gboolean md_dedup_write_dedup(
1312 mdContext_t *ctx,
1313 md_export_node_t *exp,
1314 md_dedup_t *dedup,
1315 uint16_t ie,
1316 GError **err)
1317 {
1318
1319 md_dedup_state_t *state = exp->dedup;
1320 md_dedup_ie_t *ietab = NULL;
1321
1322 ietab = md_dedup_ie_lookup(state, ie);
1323
1324 if (!ietab) {
1325 g_message("Ignoring incoming record: No IE dedup table for ie %d", ie);
1326 return TRUE;
1327 }
1328
1329 if (!mdExporterDedupFileOpen(ctx->cfg, exp->exp, &(ietab->out_file),
1330 &(ietab->last_file),
1331 ietab->file_prefix,
1332 &(ietab->last_rotate_ms)))
1333 {
1334 return FALSE;
1335 }
1336
1337 if (!mdExporterWriteDedupRecord(ctx->cfg, exp, ietab->out_file,
1338 dedup, ietab->file_prefix,
1339 ietab->tmpl->intid, ietab->tmpl->extid,
1340 err))
1341 {
1342 return FALSE;
1343 }
1344
1345 state->stats.flushed++;
1346
1347 return TRUE;
1348
1349 }
1350
1351
md_dedup_lookup_node(mdContext_t * ctx,md_export_node_t * exp,mdFullFlow_t * flow,GError ** err)1352 void md_dedup_lookup_node(
1353 mdContext_t *ctx,
1354 md_export_node_t *exp,
1355 mdFullFlow_t *flow,
1356 GError **err)
1357 {
1358 md_dedup_state_t *state = exp->dedup;
1359 fbBasicList_t *bl = NULL;
1360 md_dedup_ie_t *ietab = NULL;
1361 gboolean rev = FALSE;
1362 int loop;
1363
1364 if (flow->rec->reversePacketTotalCount) {
1365 rev = TRUE;
1366 }
1367
1368 if (flow->p0f) {
1369 ietab = md_dedup_ie_lookup(state, 36);
1370 if (ietab) {
1371 md_dedup_add_node(ctx, exp, ietab, flow->p0f->osName.buf,
1372 flow->p0f->osName.len, 36, flow, FALSE);
1373 if (rev) {
1374 md_dedup_add_node(ctx, exp,ietab, flow->p0f->reverseOsName.buf,
1375 flow->p0f->reverseOsName.len, 36, flow, TRUE);
1376 }
1377 }
1378 ietab = md_dedup_ie_lookup(state, 37);
1379 if (ietab) {
1380 md_dedup_add_node(ctx, exp,ietab, flow->p0f->osVersion.buf,
1381 flow->p0f->osVersion.len, 37, flow, FALSE);
1382 if (rev) {
1383 md_dedup_add_node(ctx, exp,ietab, flow->p0f->reverseOsVersion.buf,
1384 flow->p0f->reverseOsVersion.len, 37, flow, TRUE);
1385 }
1386 }
1387 ietab = md_dedup_ie_lookup(state, 107);
1388 if (ietab) {
1389 md_dedup_add_node(ctx, exp,ietab, flow->p0f->osFingerPrint.buf,
1390 flow->p0f->osFingerPrint.len, 107, flow, FALSE);
1391 if (rev) {
1392 md_dedup_add_node(ctx, exp,ietab,
1393 flow->p0f->reverseOsFingerPrint.buf,
1394 flow->p0f->reverseOsFingerPrint.len, 107,
1395 flow, TRUE);
1396 }
1397 }
1398 }
1399
1400 if (flow->dhcpfp) {
1401 if ((flow->dhcpfp->tmplID & YTF_BIF) == YAF_DHCP_FLOW_TID) {
1402 yaf_dhcp_fp_t *dhcp = NULL;
1403 ietab = md_dedup_ie_lookup(state, 242);
1404 if (ietab) {
1405 dhcp = (yaf_dhcp_fp_t*)FBSTMLNEXT(flow->dhcpfp, dhcp);
1406 md_dedup_add_node(ctx, exp,ietab, dhcp->dhcpFingerPrint.buf,
1407 dhcp->dhcpFingerPrint.len, 242, flow, FALSE);
1408 if (flow->dhcpfp->tmplID & YTF_REV) {
1409 md_dedup_add_node(ctx, exp,ietab, dhcp->reverseDhcpFingerPrint.buf,
1410 dhcp->reverseDhcpFingerPrint.len, 242,
1411 flow, TRUE);
1412 }
1413 }
1414 ietab = md_dedup_ie_lookup(state, 243);
1415 if (ietab) {
1416 md_dedup_add_node(ctx, exp,ietab, dhcp->dhcpVendorCode.buf,
1417 dhcp->dhcpVendorCode.len, 243, flow, FALSE);
1418 if (flow->dhcpfp->tmplID & YTF_REV) {
1419 md_dedup_add_node(ctx, exp,ietab, dhcp->reverseDhcpVendorCode.buf,
1420 dhcp->reverseDhcpVendorCode.len, 243,
1421 flow, TRUE);
1422 }
1423 }
1424 } else if ((flow->dhcpfp->tmplID & YTF_BIF) == YAF_DHCP_OP_TID) {
1425 yaf_dhcp_options_t *dhcp = NULL;
1426 ietab = md_dedup_ie_lookup(state, 243);
1427 if (ietab) {
1428 dhcp = (yaf_dhcp_options_t*)FBSTMLNEXT(flow->dhcpfp, dhcp);
1429 md_dedup_add_node(ctx, exp,ietab, dhcp->dhcpVendorCode.buf,
1430 dhcp->dhcpVendorCode.len, 243, flow, FALSE);
1431 if (flow->dhcpfp->tmplID & YTF_REV) {
1432 md_dedup_add_node(ctx, exp,ietab, dhcp->reverseDhcpVendorCode.buf,
1433 dhcp->reverseDhcpVendorCode.len, 243,
1434 flow, TRUE);
1435 }
1436 }
1437 }
1438 }
1439
1440
1441 switch(flow->app_tid & YTF_BIF) {
1442 case YAF_HTTP_FLOW_TID:
1443 case YAF_POP3_FLOW_TID:
1444 case YAF_IRC_FLOW_TID:
1445 case YAF_FTP_FLOW_TID:
1446 case YAF_IMAP_FLOW_TID:
1447 case YAF_SIP_FLOW_TID:
1448 case YAF_RTSP_FLOW_TID:
1449 case YAF_SSH_FLOW_TID:
1450 case YAF_SMTP_FLOW_TID:
1451 case YAF_NNTP_FLOW_TID:
1452 case YAF_MODBUS_FLOW_TID:
1453 case YAF_ENIP_FLOW_TID:
1454 bl = (fbBasicList_t *)flow->app;
1455 for (loop = 0; loop < flow->app_elements; loop++) {
1456 if (bl->infoElement) {
1457 ietab = md_dedup_ie_lookup(state, bl->infoElement->num);
1458 if (ietab) {
1459 md_dedup_add_bl(ctx, exp, ietab, bl, flow);
1460 }
1461 }
1462 bl++;
1463 }
1464 break;
1465 case YAF_TFTP_FLOW_TID:
1466 {
1467 yaf_tftp_t *tftp = (yaf_tftp_t *)flow->app;
1468 ietab = md_dedup_ie_lookup(state, 127);
1469 if (ietab) {
1470 md_dedup_add_node(ctx, exp, ietab, tftp->tftpMode.buf,
1471 tftp->tftpMode.len, 127, flow, FALSE);
1472 }
1473 ietab = md_dedup_ie_lookup(state, 126);
1474 if (ietab) {
1475 md_dedup_add_node(ctx, exp, ietab, tftp->tftpFilename.buf,
1476 tftp->tftpFilename.len, 126, flow, FALSE);
1477 }
1478 break;
1479 }
1480 case YAF_SLP_FLOW_TID:
1481 {
1482 yaf_slp_t *slp = (yaf_slp_t *)flow->app;
1483 char slp_buffer[20];
1484
1485 ietab = md_dedup_ie_lookup(state, 128);
1486 if (ietab) {
1487 snprintf(slp_buffer,sizeof(slp_buffer), "%d", slp->slpVersion);
1488 md_dedup_add_node(ctx, exp, ietab, (uint8_t*)slp_buffer,
1489 strlen(slp_buffer), 128, flow, FALSE);
1490 }
1491 ietab = md_dedup_ie_lookup(state, 129);
1492 if (ietab) {
1493 snprintf(slp_buffer, sizeof(slp_buffer), "%d",
1494 slp->slpMessageType);
1495 md_dedup_add_node(ctx, exp, ietab, (uint8_t*)slp_buffer,
1496 strlen(slp_buffer), 129, flow, FALSE);
1497 }
1498 bl = (fbBasicList_t *)flow->app;
1499 if (bl->infoElement) {
1500 ietab = md_dedup_ie_lookup(state, bl->infoElement->num);
1501 if (ietab) {
1502 md_dedup_add_bl(ctx, exp, ietab, bl, flow);
1503 }
1504 }
1505 break;
1506 }
1507 case YAF_NEW_SSL_FLOW_TID:
1508 case SM_INTSSL_FLOW_TID:
1509 {
1510 yaf_newssl_t *sslflow = (yaf_newssl_t *)flow->app;
1511
1512 ietab = md_dedup_ie_lookup(state, SSL_SERIAL_IE);
1513 if (!ietab) {
1514 break;
1515 }
1516
1517 md_dedup_ssl_add_node(ctx, exp, ietab, sslflow, flow);
1518
1519 }
1520 break;
1521 case YAF_MYSQL_FLOW_TID:
1522 {
1523 yaf_mysql_t *mflow = (yaf_mysql_t *)flow->app;
1524 yaf_mysql_txt_t *mtxt = NULL;
1525 ietab = md_dedup_ie_lookup(state, 223);
1526 if (ietab) {
1527 md_dedup_add_node(ctx, exp, ietab, mflow->mysqlUsername.buf,
1528 mflow->mysqlUsername.len, 223, flow, FALSE);
1529 }
1530 ietab = md_dedup_ie_lookup(state, 225);
1531 if (ietab) {
1532 while((mtxt =(yaf_mysql_txt_t *)FBSTLNEXT(&(mflow->mysqlList),
1533 mtxt)))
1534 {
1535 md_dedup_add_node(ctx, exp, ietab,
1536 mtxt->mysqlCommandText.buf,
1537 mtxt->mysqlCommandText.len, 225, flow,
1538 FALSE);
1539 }
1540 }
1541 break;
1542 }
1543 case YAF_DNS_FLOW_TID:
1544 {
1545 yaf_dns_t *dnsflow = (yaf_dns_t *)flow->app;
1546 yaf_dnsQR_t *dnsqrflow = NULL;
1547
1548 /* dns query name */
1549 ietab = md_dedup_ie_lookup(state, 179);
1550 if (ietab) {
1551 while (( dnsqrflow = (yaf_dnsQR_t *)FBSTLNEXT(&(dnsflow->dnsQRList),
1552 dnsqrflow)))
1553 {
1554 /* just queries */
1555 if (dnsqrflow->dnsQueryResponse == 0) {
1556 md_dedup_add_node(ctx, exp, ietab,
1557 dnsqrflow->dnsQName.buf,
1558 dnsqrflow->dnsQName.len, 179, flow,
1559 FALSE);
1560 }
1561 }
1562 }
1563 }
1564 break;
1565 case YAF_RTP_FLOW_TID:
1566 {
1567 yaf_rtp_t *rtp = (yaf_rtp_t *)flow->app;
1568 char rtp_buffer[20];
1569
1570 ietab = md_dedup_ie_lookup(state, 287);
1571 if (ietab && rtp) {
1572 snprintf(rtp_buffer, sizeof(rtp_buffer), "%d",
1573 rtp->rtpPayloadType);
1574 md_dedup_add_node(ctx, exp, ietab, (uint8_t *)rtp_buffer,
1575 strlen(rtp_buffer), 287, flow, FALSE);
1576 snprintf(rtp_buffer, sizeof(rtp_buffer), "%d",
1577 rtp->reverseRtpPayloadType);
1578 /* for reverse fields, swap sip & dip */
1579 md_dedup_add_node(ctx, exp, ietab, (uint8_t *)rtp_buffer,
1580 strlen(rtp_buffer), 287, flow, TRUE);
1581 }
1582 }
1583 break;
1584 case YAF_DNP3_FLOW_TID:
1585 {
1586 yaf_dnp_t *dnp = (yaf_dnp_t *)flow->app;
1587 yaf_dnp_rec_t *rec = NULL;
1588 char dnp_buffer[65535];
1589 size_t bufsz = sizeof(dnp_buffer);
1590 size_t buflen;
1591 int i;
1592
1593 ietab = md_dedup_ie_lookup(state, 284);
1594 if (ietab && dnp) {
1595 while ((rec = (yaf_dnp_rec_t *)FBSTLNEXT(&(dnp->dnp_list), rec)))
1596 {
1597 buflen = rec->dnp3ObjectData.len;
1598 if (buflen > bufsz) {
1599 buflen = bufsz;
1600 }
1601 i = md_util_hexdump_append(dnp_buffer, &bufsz,
1602 rec->dnp3ObjectData.buf, buflen);
1603 md_dedup_add_node(ctx, exp, ietab, (uint8_t*)dnp_buffer,
1604 i, 284, flow, FALSE);
1605 }
1606
1607 }
1608 }
1609 break;
1610 default:
1611 break;
1612 }
1613
1614 /* attempt to flush all tables */
1615 md_dedup_flush_alltab(exp, ctx->cfg->ctime, FALSE);
1616
1617 }
1618