1 /**
2 * @file mediator_dns.c
3 *
4 * Yaf mediator for filtering, DNS deduplication, and other mediator-like
5 * things
6 *
7 ** ------------------------------------------------------------------------
8 ** Copyright (C) 2006-2017 Carnegie Mellon University. All Rights Reserved.
9 * -----------------------------------------------------------
10 * Authors: Emily Sarneso <netsa-help@cert.org>
11 * -----------------------------------------------------------
12 * @OPENSOURCE_HEADER_START@
13 * Use of this (and related) source code is subject to the terms
14 * of the following licenses:
15 *
16 * GNU Public License (GPL) Rights pursuant to Version 2, June 1991
17 * Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013
18 *
19 *
20 * This material is based upon work funded and supported by
21 * the Department of Defense under Contract FA8721-05-C-0003 with
22 * Carnegie Mellon University for the operation of the Software Engineering
23 * Institue, a federally funded research and development center. Any opinions,
24 * findings and conclusions or recommendations expressed in this
25 * material are those of the author(s) and do not
26 * necessarily reflect the views of the United States
27 * Department of Defense.
28 *
29 * NO WARRANTY
30 *
31 * THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE
32 * MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY
33 * MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED
34 * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
35 * FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
36 * OBTAINED FROM THE USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY
37 * DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM
38 * PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
39 *
40 * This material has been approved for public release and unlimited
41 * distribution.
42 *
43 * Carnegie Mellon®, CERT® and CERT Coordination Center® are
44 * registered marks of Carnegie Mellon University.
45 *
46 * DM-0001877
47 *
48 * Carnegie Mellon University retains
49 * copyrights in all material produced under this contract. The U.S.
50 * Government retains a non-exclusive, royalty-free license to publish or
51 * reproduce these documents, or allow others to do so, for U.S.
52 * Government purposes only pursuant to the copyright license under the
53 * contract clause at 252.227.7013.
54 *
55 * Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
56 * Mellon University, its trustees, officers, employees, and agents from
57 * all claims or demands made against them (and any related losses,
58 * expenses, or attorney's fees) arising out of, or relating to Licensee's
59 * and/or its sub licensees' negligent use or willful misuse of or
60 * negligent conduct or willful misconduct regarding the Software,
61 * facilities, or other rights or assistance granted by Carnegie Mellon
62 * University under this License, including, but not limited to, any
63 * claims of product liability, personal injury, death, damage to
64 * property, or violation of any laws or regulations.
65 *
66 * @OPENSOURCE_HEADER_END@
67 * -----------------------------------------------------------
68 */
69
70 #include "mediator_dns.h"
71 #include <mediator/mediator_inf.h>
72 #include <mediator/mediator_util.h>
73
74 #define DNS_DEBUG 0
75
76 #define FBSTLNEXT(a, b) fbSubTemplateListGetNextPtr(a, b)
77 #define A_REC_TID(_id_) (_id_ |= MD_DNS_AREC)
78 #define OTHER_REC_TID(_id_) (_id_ |= MD_DNS_OREC)
79
80 typedef struct md_dns_dedup_stats_st {
81 uint64_t dns_recvd;
82 uint64_t dns_filtered;
83 uint64_t dns_flushed;
84 } md_dns_dedup_stats_t;
85
86
87 struct md_dns_dedup_state_st {
88 md_dns_dedup_stats_t stats;
89 md_type_hashtab_t *a_table;
90 md_type_hashtab_t *ns_table;
91 md_type_hashtab_t *cname_table;
92 md_type_hashtab_t *soa_table;
93 md_type_hashtab_t *ptr_table;
94 md_type_hashtab_t *mx_table;
95 md_type_hashtab_t *txt_table;
96 md_type_hashtab_t *aaaa_table;
97 md_type_hashtab_t *srv_table;
98 md_type_hashtab_t *nx_table;
99 md_dns_cqueue_t *cq;
100 smFieldMap_t *map;
101 int *dedup_type_list;
102 uint64_t dedup_flush_to;
103 uint32_t dedup_max_hit_count;
104 gboolean print_lastseen;
105 gboolean export_name;
106 };
107
108 struct md_dns_node_st {
109 struct md_dns_node_st *next;
110 struct md_dns_node_st *prev;
111 md_dns_t dns_node;
112 };
113
114
115 /**
116 * allocTypeTab
117 *
118 *
119 */
allocTypeTab(uint64_t cur_time)120 static md_type_hashtab_t *allocTypeTab (
121 uint64_t cur_time)
122 {
123
124 md_type_hashtab_t *md_type_tab;
125
126 md_type_tab = g_slice_new0(md_type_hashtab_t);
127 /*md_type_tab->table = g_hash_table_new((GHashFunc)g_str_hash,
128 (GEqualFunc)g_str_equal);*/
129 md_type_tab->table = smCreateHashTable(0xFF,
130 sm_octet_array_key_destroy, NULL);
131 if (md_type_tab->table == NULL) {
132 return NULL;
133 }
134
135 md_type_tab->last_flush = cur_time;
136
137 return md_type_tab;
138
139 }
140
141 /**
142 * md_dns_dedup_print_stats
143 *
144 * Prints stats to the log.
145 *
146 *
147 */
md_dns_dedup_print_stats(md_dns_dedup_state_t * state,char * exp_name)148 void md_dns_dedup_print_stats(
149 md_dns_dedup_state_t *state,
150 char *exp_name)
151 {
152 if (state->stats.dns_recvd == 0) {
153 return;
154 }
155
156 g_message("Exporter %s: %"PRIu64" DNS records, %"PRIu64" filtered"
157 ", %"PRIu64" flushed (%2.2f%% compression)", exp_name,
158 state->stats.dns_recvd, state->stats.dns_filtered,
159 state->stats.dns_flushed, (1 -(((double)state->stats.dns_flushed)/
160 ((double)state->stats.dns_recvd))) *
161 100);
162 }
163
164 /**
165 * md_dns_reset_dedup
166 *
167 * Flushes all Hash Tables.
168 *
169 */
170
md_dns_reset_dedup(md_dns_dedup_state_t * state,uint64_t cur_time)171 void md_dns_reset_dedup(
172 md_dns_dedup_state_t *state,
173 uint64_t cur_time)
174 {
175 g_warning("Out of Memory Error. Resetting all Hash Tables");
176 md_dns_flush_all_tab(state, cur_time, TRUE);
177 }
178
md_dns_attempt_flush_tab(md_type_hashtab_t * md_type_tab,md_dns_dedup_state_t * state,uint64_t ctime)179 static void md_dns_attempt_flush_tab(
180 md_type_hashtab_t *md_type_tab,
181 md_dns_dedup_state_t *state,
182 uint64_t ctime)
183 {
184
185 if (md_type_tab && ((ctime - md_type_tab->last_flush) >
186 state->dedup_flush_to))
187 {
188 md_dns_flush_tab(md_type_tab, state, ctime, FALSE);
189 }
190 }
191
md_dns_attempt_all_flush(md_dns_dedup_state_t * state,uint64_t cur_time)192 static void md_dns_attempt_all_flush(
193 md_dns_dedup_state_t *state,
194 uint64_t cur_time)
195 {
196 md_dns_attempt_flush_tab(state->a_table, state, cur_time);
197 md_dns_attempt_flush_tab(state->ns_table, state, cur_time);
198 md_dns_attempt_flush_tab(state->cname_table, state, cur_time);
199 md_dns_attempt_flush_tab(state->soa_table, state, cur_time);
200 md_dns_attempt_flush_tab(state->ptr_table, state, cur_time);
201 md_dns_attempt_flush_tab(state->mx_table, state, cur_time);
202 md_dns_attempt_flush_tab(state->txt_table, state, cur_time);
203 md_dns_attempt_flush_tab(state->aaaa_table, state, cur_time);
204 md_dns_attempt_flush_tab(state->srv_table, state, cur_time);
205 md_dns_attempt_flush_tab(state->nx_table, state, cur_time);
206 }
207
208
209
210
211 /**
212 * md_dns_destroy_tab
213 *
214 * destroys all hash tables
215 *
216 */
md_dns_destroy_tab(md_dns_dedup_state_t * state)217 void md_dns_destroy_tab(
218 md_dns_dedup_state_t *state)
219 {
220
221 if (state->a_table && state->a_table->table) {
222 smHashTableFree(state->a_table->table);
223 }
224 if (state->ns_table && state->ns_table->table) {
225 smHashTableFree(state->ns_table->table);
226 }
227 if (state->cname_table && state->cname_table->table) {
228 smHashTableFree(state->cname_table->table);
229 }
230 if (state->soa_table && state->soa_table->table) {
231 smHashTableFree(state->soa_table->table);
232 }
233 if (state->ptr_table && state->ptr_table->table) {
234 smHashTableFree(state->ptr_table->table);
235 }
236 if (state->mx_table && state->mx_table->table) {
237 smHashTableFree(state->mx_table->table);
238 }
239 if (state->txt_table && state->txt_table->table) {
240 smHashTableFree(state->txt_table->table);
241 }
242 if (state->aaaa_table && state->aaaa_table->table) {
243 smHashTableFree(state->aaaa_table->table);
244 }
245 if (state->nx_table && state->nx_table->table) {
246 smHashTableFree(state->nx_table->table);
247 }
248 if (state->srv_table && state->srv_table->table) {
249 smHashTableFree(state->srv_table->table);
250 }
251 }
252
md_dns_dedup_free_state(mdConfig_t * cfg,md_export_node_t * exp,GError ** err)253 gboolean md_dns_dedup_free_state(
254 mdConfig_t *cfg,
255 md_export_node_t *exp,
256 GError **err)
257
258 {
259 md_dns_dedup_state_t *state = exp->dns_dedup;
260
261 md_dns_flush_all_tab(state, cfg->ctime, TRUE);
262
263 if (!md_dns_flush_queue(exp, cfg, err)) {
264 return FALSE;
265 }
266
267 md_dns_destroy_tab(state);
268 if (state->dedup_type_list) {
269 g_free(state->dedup_type_list);
270 }
271
272 g_slice_free1(sizeof(md_dns_cqueue_t), state->cq);
273 return TRUE;
274 }
275
276 /**
277 * md_debug_table
278 *
279 *
280 */
281 #if DNS_DEBUG == 1
md_debug_table(md_type_hashtab_t * nodeTab)282 static void md_debug_table(
283 md_type_hashtab_t *nodeTab)
284 {
285
286 md_cache_node_t *cq;
287 md_hashtab_node_t *hn;
288
289 for (hn = nodeTab->head; hn; hn = hn->next) {
290 for (cq = hn->head; cq; cq = cq->next) {
291 g_debug("%d %p rrname %s", cq->rrtype, cq,
292 hn->rrname);
293 g_debug("cq->next is %p", cq->next);
294 }
295 }
296 }
297 #endif
298 /**
299 * md_new_dns_queue
300 *
301 * creates a new close queue for dns-dedup
302 */
md_new_dns_queue(void)303 md_dns_cqueue_t *md_new_dns_queue(
304 void)
305 {
306
307 md_dns_cqueue_t *cq = g_slice_new0(md_dns_cqueue_t);
308
309 cq->head = NULL;
310 cq->tail = NULL;
311
312 return cq;
313 }
314
md_new_dns_dedup_state(void)315 md_dns_dedup_state_t *md_new_dns_dedup_state(
316 void)
317 {
318 md_dns_dedup_state_t *state = g_slice_new0(md_dns_dedup_state_t);
319
320 state->cq = md_new_dns_queue();
321
322 /* set defaults */
323 state->dedup_max_hit_count = 5000;
324 state->dedup_flush_to = 300 * 1000;
325 state->print_lastseen = FALSE;
326
327 return state;
328 }
329
md_dns_dedup_configure_state(md_dns_dedup_state_t * state,int * dedup_list,int max_hit,int flush_timeout,gboolean lastseen,smFieldMap_t * map,gboolean export_name)330 void md_dns_dedup_configure_state(
331 md_dns_dedup_state_t *state,
332 int *dedup_list,
333 int max_hit,
334 int flush_timeout,
335 gboolean lastseen,
336 smFieldMap_t *map,
337 gboolean export_name)
338 {
339 if (!state) {
340 return;
341 }
342
343 state->dedup_type_list = dedup_list;
344 state->print_lastseen = lastseen;
345 state->export_name = export_name;
346 if (max_hit) {
347 state->dedup_max_hit_count = max_hit;
348 }
349 if (flush_timeout) {
350 state->dedup_flush_to = flush_timeout * 1000;
351 }
352 if (map) {
353 state->map = map;
354 }
355 }
356
357 /**
358 * md_dns_flush_queue
359 *
360 * Flushes all records in the close queue.
361 *
362 */
md_dns_flush_queue(md_export_node_t * exp,mdConfig_t * cfg,GError ** err)363 gboolean md_dns_flush_queue(
364 md_export_node_t *exp,
365 mdConfig_t *cfg,
366 GError **err)
367 {
368 md_dns_node_t *node;
369 md_dns_dedup_state_t *state = exp->dns_dedup;
370 md_dns_cqueue_t *cq = exp->dns_dedup->cq;
371 uint16_t tid = MD_DNS_OUT;
372 uint16_t wtid;
373
374 if (cq == NULL) {
375 return TRUE;
376 }
377
378 if (state->print_lastseen) {
379 tid |= MD_LAST_SEEN;
380 }
381
382 wtid = tid;
383
384 while ((node = detachFromEndOfDLL((mdDLL_t **)&(cq->head),
385 (mdDLL_t **)&(cq->tail))))
386 { wtid = tid;
387 if (node->dns_node.rrtype == 1) {
388 A_REC_TID(wtid);
389 } else {
390 OTHER_REC_TID(wtid);
391 }
392
393 if (state->export_name && (node->dns_node.mapname.len == 0)) {
394 node->dns_node.mapname.buf = (uint8_t*)mdExporterGetName(exp->exp);
395 node->dns_node.mapname.len = strlen(mdExporterGetName(exp->exp));
396 }
397
398 if (!mdExporterWriteRecord(cfg, exp->exp, wtid,
399 (uint8_t *)&(node->dns_node),
400 sizeof(md_dns_t), err))
401 {
402 return FALSE;
403 }
404
405 state->stats.dns_flushed++;
406 g_slice_free1(node->dns_node.rrdata.len, node->dns_node.rrdata.buf);
407 g_slice_free1(node->dns_node.rrname.len, node->dns_node.rrname.buf);
408 g_slice_free(md_dns_node_t, node);
409
410 }
411
412 /* free the node we just sent out */
413
414 return TRUE;
415 }
416
417
418
419 /**
420 * nodeClose
421 *
422 * closes the HASHnode, this means that there is no more
423 * cache nodes that belong to this "hash node." Basically
424 * this means that we flushed all information associated
425 * with this query name.
426 *
427 * @param struct that contains node hash table
428 * @param pointer to the node entry that we want to close
429 *
430 */
nodeClose(md_type_hashtab_t * nodeTab,md_hashtab_node_t * hnode)431 static void nodeClose (
432 md_type_hashtab_t *nodeTab,
433 md_hashtab_node_t *hnode)
434 {
435 /*Remove it from list*/
436
437 /* g_hash_table_remove(nodeTab->table, hnode->rrname);*/
438 smHashTableRemove(nodeTab->table, (uint8_t*)hnode->rkey);
439
440 detachThisEntryOfDLL((mdDLL_t**)&(nodeTab->head),
441 (mdDLL_t**)&(nodeTab->tail), (mdDLL_t*)hnode);
442
443 /* free the rrname */
444
445 /* g_slice_free1(hnode->rrname_len, hnode->rrname); */
446 g_slice_free(md_hashtab_node_t, hnode);
447
448 --(nodeTab->count);
449 }
450
451 /**
452 * newCacheNode
453 *
454 * creates a new cache node which will go into
455 * a linked list by hash node. Basically this
456 * has the same query name, but a different type
457 * or rrdata
458 */
newCacheNode(uint64_t start_time,uint32_t ip,uint16_t rrtype,uint8_t * capt,unsigned int caplen)459 static md_cache_node_t *newCacheNode (
460 uint64_t start_time,
461 uint32_t ip,
462 uint16_t rrtype,
463 uint8_t *capt,
464 unsigned int caplen)
465 {
466 md_cache_node_t *cn;
467
468 cn = g_slice_new0(md_cache_node_t);
469 if (cn == NULL) {
470 return NULL;
471 }
472 cn->hitcount = 1;
473 cn->ftime = start_time;
474 cn->ltime = start_time;
475 cn->ip = ip;
476 cn->rrtype = rrtype;
477
478 if (caplen) {
479 cn->rrdata = g_slice_alloc0(caplen);
480 if (cn->rrdata == NULL) {
481 return NULL;
482 }
483 memcpy(cn->rrdata, capt, caplen);
484 cn->caplen = caplen;
485 }
486
487 return cn;
488 }
489
490 /**
491 * hashTick
492 *
493 * advances a node to the head of the
494 * queue - bottom of queue gets examined
495 * for flush timeouts
496 *
497 * @param pointer to table
498 * @param pointer to node
499 *
500 */
hashTick(md_type_hashtab_t * nodeTab,md_hashtab_node_t * entry)501 static void hashTick (
502 md_type_hashtab_t *nodeTab,
503 md_hashtab_node_t *entry)
504 {
505
506 if (nodeTab->head != entry) {
507 if (entry->prev != NULL) {
508 detachThisEntryOfDLL((mdDLL_t**)&(nodeTab->head),
509 (mdDLL_t**)&(nodeTab->tail), (mdDLL_t*)entry);
510 }
511 attachHeadToDLL((mdDLL_t**)&(nodeTab->head),
512 (mdDLL_t**)&(nodeTab->tail),
513 (mdDLL_t*)entry);
514 }
515
516 /* md_debug_table(nodeTab);*/
517 }
518
519 /**
520 * cacheNodeClose
521 *
522 * creates a new md_dns_node_t for output,
523 * attaches it to the close queue, and frees the
524 * cache node associated with the domain name.
525 *
526 *
527 * @param hashNode
528 * @param CacheNode to close
529 * @param filepointers
530 */
cacheNodeClose(md_type_hashtab_t * nodeTab,md_hashtab_node_t * hn,md_cache_node_t * cn,md_dns_dedup_state_t * state)531 static void cacheNodeClose (
532 md_type_hashtab_t *nodeTab,
533 md_hashtab_node_t *hn,
534 md_cache_node_t *cn,
535 md_dns_dedup_state_t *state)
536 {
537
538 md_dns_cqueue_t *cq = state->cq;
539
540 if (state->print_lastseen) {
541 md_dns_node_t *node = g_slice_new0(md_dns_node_t);
542 node->dns_node.flowStartMilliseconds = cn->ftime;
543 node->dns_node.flowEndMilliseconds = cn->ltime;
544 node->dns_node.sourceIPv4Address = cn->ip;
545 node->dns_node.rrtype = cn->rrtype;
546 node->dns_node.dnsHitCount = cn->hitcount;
547 node->dns_node.dnsTTL = cn->ttl;
548 node->dns_node.rrdata.buf = g_slice_alloc0(cn->caplen);
549 memcpy(node->dns_node.rrdata.buf, cn->rrdata, cn->caplen);
550 node->dns_node.rrdata.len = cn->caplen;
551 /*node->dns_node.rrname.buf = g_slice_alloc0(hn->rrname_len);*/
552 /*memcpy(node->dns_node.rrname.buf, hn->rrname, hn->rrname_len);
553 node->dns_node.rrname.len = hn->rrname_len;*/
554 if (hn->mapindex < 0) {
555 node->dns_node.rrname.buf = g_slice_alloc0(hn->rkey->len);
556 memcpy(node->dns_node.rrname.buf, hn->rkey->val, hn->rkey->len);
557 node->dns_node.rrname.len = hn->rkey->len;
558 node->dns_node.mapname.len = 0;
559 } else {
560 node->dns_node.rrname.buf = g_slice_alloc0(hn->rkey->len - sizeof(uint32_t));
561 memcpy(node->dns_node.rrname.buf, hn->rkey->val+sizeof(uint32_t), hn->rkey->len-sizeof(uint32_t));
562 node->dns_node.rrname.len = hn->rkey->len - sizeof(uint32_t);
563 node->dns_node.mapname.buf = (uint8_t*)(state->map->labels[hn->mapindex]);
564 node->dns_node.mapname.len = strlen(state->map->labels[hn->mapindex]);
565 }
566
567
568 attachHeadToDLL((mdDLL_t **)&(cq->head),
569 (mdDLL_t **)&(cq->tail),
570 (mdDLL_t *)node);
571 }
572 detachThisEntryOfDLL((mdDLL_t**)&(hn->head),
573 (mdDLL_t**)&(hn->tail),
574 (mdDLL_t*)cn);
575
576 g_slice_free1(cn->caplen, cn->rrdata);
577 g_slice_free(md_cache_node_t, cn);
578
579 if (!hn->head) {
580 /*last cacheNode in hashTabNode - remove from hashtable*/
581 nodeClose(nodeTab, hn);
582 }
583 }
584
585 /**
586 * md_dns_emit_record
587 *
588 * Adds the record to the close queue without removing
589 * the node.
590 *
591 * @param cq - the close queue to add it to
592 * @param cn - the node to add
593 *
594 */
595
md_dns_emit_record(md_dns_dedup_state_t * state,md_dns_cqueue_t * cq,md_hashtab_node_t * hn,md_cache_node_t * cn)596 static void md_dns_emit_record(
597 md_dns_dedup_state_t *state,
598 md_dns_cqueue_t *cq,
599 md_hashtab_node_t *hn,
600 md_cache_node_t *cn)
601 {
602
603 md_dns_node_t *node = g_slice_new0(md_dns_node_t);
604
605 if (node == NULL) {
606 g_debug("Potentially out of memory.");
607 return;
608 }
609
610 node->dns_node.flowStartMilliseconds = cn->ftime;
611 node->dns_node.flowEndMilliseconds = cn->ltime;
612 node->dns_node.sourceIPv4Address = cn->ip;
613 node->dns_node.rrtype = cn->rrtype;
614 node->dns_node.dnsHitCount = cn->hitcount;
615 node->dns_node.dnsTTL = cn->ttl;
616 if (hn->mapindex < 0) {
617 node->dns_node.rrname.buf = g_slice_alloc0(hn->rkey->len);
618 memcpy(node->dns_node.rrname.buf, hn->rkey->val, hn->rkey->len);
619 node->dns_node.rrname.len = hn->rkey->len;
620 node->dns_node.mapname.len = 0;
621 } else {
622 node->dns_node.rrname.buf = g_slice_alloc0(hn->rkey->len - sizeof(uint32_t));
623 memcpy(node->dns_node.rrname.buf, hn->rkey->val+sizeof(uint32_t),
624 hn->rkey->len-sizeof(uint32_t));
625 node->dns_node.rrname.len = hn->rkey->len - sizeof(uint32_t);
626 node->dns_node.mapname.buf = (uint8_t*)(state->map->labels[hn->mapindex]);
627 node->dns_node.mapname.len = strlen(state->map->labels[hn->mapindex]);
628 }
629
630 node->dns_node.rrdata.buf = g_slice_alloc0(cn->caplen);
631 memcpy(node->dns_node.rrdata.buf, cn->rrdata, cn->caplen);
632 node->dns_node.rrdata.len = cn->caplen;
633
634 /*node->dns_node.rrname.buf = g_slice_alloc0(hn->rrname_len);
635 memcpy(node->dns_node.rrname.buf, hn->rrname, hn->rrname_len);
636 node->dns_node.rrname.len = hn->rrname_len;*/
637
638 attachHeadToDLL((mdDLL_t **)&(cq->head),
639 (mdDLL_t **)&(cq->tail),
640 (mdDLL_t *)node);
641 }
642
643
644 /**
645 * hashCacheTick
646 *
647 * advances a node to the head of the cache queue
648 * bottom gets examined for flush timeouts
649 *
650 * @param pointer to head of table
651 * @param pointer to node
652 *
653 */
hashCacheTick(md_dns_dedup_state_t * state,md_type_hashtab_t * nodeTab,md_hashtab_node_t * hn,md_cache_node_t * cn)654 static void hashCacheTick (
655 md_dns_dedup_state_t *state,
656 md_type_hashtab_t *nodeTab,
657 md_hashtab_node_t *hn,
658 md_cache_node_t *cn)
659 {
660
661 if (hn->head != cn) {
662 if (cn->prev != NULL) {
663 detachThisEntryOfDLL((mdDLL_t**)&(hn->head),
664 (mdDLL_t**)&(hn->tail),
665 (mdDLL_t*)cn);
666 }
667 attachHeadToDLL((mdDLL_t**)&(hn->head),
668 (mdDLL_t**)&(hn->tail),
669 (mdDLL_t*)cn);
670 }
671
672 while (hn->tail &&((cn->ltime - hn->tail->ltime) > state->dedup_flush_to))
673 {
674 cacheNodeClose(nodeTab, hn, hn->tail, state);
675 }
676
677 }
678
679
680 /**
681 * md_dns_flush_tab
682 *
683 * Checks entries in the hash table to see if they are past the
684 * flush limit. If so, it outputs to the appropriate file and deallocates
685 * the memory
686 *
687 * @param the struct that contains the hash table and linked list
688 * @param cq - the close queue.
689 * @param cur_time to keep track of how often we're flushing
690 * @param flush_all (if TRUE -> close all)
691 *
692 */
md_dns_flush_tab(md_type_hashtab_t * nodeTab,md_dns_dedup_state_t * state,uint64_t cur_time,gboolean flush_all)693 void md_dns_flush_tab (
694 md_type_hashtab_t *nodeTab,
695 md_dns_dedup_state_t *state,
696 uint64_t cur_time,
697 gboolean flush_all)
698 {
699
700 if (nodeTab == NULL) {
701 return;
702 }
703
704 nodeTab->last_flush = cur_time;
705
706 while (flush_all && nodeTab->tail) {
707 cacheNodeClose(nodeTab, nodeTab->tail, nodeTab->tail->tail, state);
708 }
709
710 while (nodeTab->tail && (nodeTab->last_flush - nodeTab->tail->tail->ltime >
711 state->dedup_flush_to))
712 {
713 cacheNodeClose(nodeTab, nodeTab->tail, nodeTab->tail->tail, state);
714 }
715
716 }
717
718
719 /**
720 * md_dns_flush_all_tab
721 *
722 * Flushes all entries from all hash tables
723 *
724 * @param cq
725 *
726 */
md_dns_flush_all_tab(md_dns_dedup_state_t * state,uint64_t cur_time,gboolean flush_all)727 void md_dns_flush_all_tab(
728 md_dns_dedup_state_t *state,
729 uint64_t cur_time,
730 gboolean flush_all)
731 {
732 md_dns_flush_tab(state->a_table, state, cur_time, flush_all);
733 md_dns_flush_tab(state->ns_table, state, cur_time, flush_all);
734 md_dns_flush_tab(state->cname_table, state, cur_time, flush_all);
735 md_dns_flush_tab(state->soa_table, state, cur_time, flush_all);
736 md_dns_flush_tab(state->ptr_table, state, cur_time, flush_all);
737 md_dns_flush_tab(state->mx_table, state, cur_time, flush_all);
738 md_dns_flush_tab(state->txt_table, state, cur_time, flush_all);
739 md_dns_flush_tab(state->aaaa_table, state, cur_time, flush_all);
740 md_dns_flush_tab(state->srv_table, state, cur_time, flush_all);
741 md_dns_flush_tab(state->nx_table, state, cur_time, flush_all);
742
743
744 }
745
746 /**
747 * md_add_dns_node
748 *
749 * add the dns node to the appropriate hash table
750 * this is the main part of deduplication.
751 *
752 * @param ctx
753 * @param mdflow
754 *
755 */
756
md_add_dns_node(mdContext_t * ctx,md_export_node_t * exp,mdFullFlow_t * flow)757 void md_add_dns_node(
758 mdContext_t *ctx,
759 md_export_node_t *exp,
760 mdFullFlow_t *flow)
761 {
762
763 md_dns_dedup_state_t *state = exp->dns_dedup;
764 yaf_dns_t *dnsflow = (yaf_dns_t *)flow->app;
765 yaf_dnsQR_t *dnsqrflow = NULL;
766 md_cache_node_t *cn = NULL, *tn = NULL;
767 int *type_list = state->dedup_type_list;
768 md_cache_node_t find;
769 md_hashtab_node_t *hn = NULL;
770 md_type_hashtab_t *md_type_tab = NULL;
771 uint8_t namebuf[1024];
772 uint16_t name_offset=0;
773 size_t namelen = 0;
774 gboolean found = FALSE;
775 int nx = 0;
776 smVarHashKey_t key;
777 uint32_t mapkey = 0;
778
779 while (( dnsqrflow = (yaf_dnsQR_t *)FBSTLNEXT(&(dnsflow->dnsQRList),
780 dnsqrflow)))
781 {
782 find.ip = 0;
783 find.caplen = 0;
784 find.rrdata = NULL;
785 namelen = 0;
786 name_offset = 0;
787 found = FALSE;
788 nx = 0;
789 find.rrtype = dnsqrflow->dnsQRType;
790 find.ttl = dnsqrflow->dnsTTL;
791
792 if (dnsqrflow->dnsNXDomain == 3 && dnsqrflow->dnsRRSection == 0) {
793 find.rrtype = 0;
794 nx = 1;
795 }
796
797 if (!nx && !dnsqrflow->dnsQueryResponse) {
798 /* don't do queries */
799 continue;
800 }
801
802 if (find.rrtype > 34) {
803 /* not a valid DNS type for super_mediator dedup */
804 state->stats.dns_filtered++;
805 continue;
806 }
807
808 if (type_list) {
809 if (type_list[find.rrtype] == 0) {
810 /* filtered out*/
811 state->stats.dns_filtered++;
812 continue;
813 }
814 }
815
816 if (nx == 1) {
817 /* NXDomain */
818 if (dnsqrflow->dnsQName.buf) {
819 if (state->nx_table == NULL) {
820 state->nx_table = allocTypeTab(ctx->cfg->ctime);
821 }
822 md_type_tab = state->nx_table;
823 } else {
824 state->stats.dns_filtered++;
825 continue;
826 }
827 } else if (dnsqrflow->dnsQueryResponse) {
828
829 if (dnsqrflow->dnsQName.len == 0) {
830 state->stats.dns_filtered++;
831 continue;
832 }
833
834 if (dnsqrflow->dnsQRType == 1) {
835 yaf_dnsA_t *aflow = NULL;
836 if (state->a_table == NULL) {
837 state->a_table = allocTypeTab(ctx->cfg->ctime);
838 }
839 while (( aflow = (yaf_dnsA_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), aflow))) {
840
841 md_type_tab = state->a_table;
842 find.ip = aflow->sourceIPv4Address;
843 }
844 } else if (dnsqrflow->dnsQRType == 2) {
845 yaf_dnsNS_t *nsflow = NULL;
846 if (state->ns_table == NULL) {
847 state->ns_table = allocTypeTab(ctx->cfg->ctime);
848 }
849 while ((nsflow = (yaf_dnsNS_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), nsflow))) {
850 md_type_tab = state->ns_table;
851 find.caplen = nsflow->dnsNSDName.len;
852 find.rrdata = nsflow->dnsNSDName.buf;
853 }
854
855 } else if (dnsqrflow->dnsQRType == 5) {
856 yaf_dns_CNAME_t *cflow = NULL;
857 if (state->cname_table == NULL) {
858 state->cname_table = allocTypeTab(ctx->cfg->ctime);
859 }
860 while ((cflow = (yaf_dns_CNAME_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), cflow))) {
861 md_type_tab = state->cname_table;
862 find.caplen = cflow->dnsCName.len;
863 find.rrdata = cflow->dnsCName.buf;
864 }
865 } else if (dnsqrflow->dnsQRType == 12) {
866 yaf_dnsPTR_t *ptrflow = NULL;
867 if (state->ptr_table == NULL) {
868 state->ptr_table = allocTypeTab(ctx->cfg->ctime);
869 }
870 while ((ptrflow = (yaf_dnsPTR_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), ptrflow)))
871 {
872 md_type_tab = state->ptr_table;
873 find.caplen = ptrflow->dnsPTRDName.len;
874 find.rrdata = ptrflow->dnsPTRDName.buf;
875 }
876 } else if (dnsqrflow->dnsQRType == 15) {
877 yaf_dnsMX_t *mx = NULL;
878 if (state->mx_table == NULL) {
879 state->mx_table = allocTypeTab(ctx->cfg->ctime);
880 }
881 while (( mx = (yaf_dnsMX_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), mx)))
882 {
883 md_type_tab = state->mx_table;
884 find.caplen = mx->dnsMXExchange.len;
885 find.rrdata = mx->dnsMXExchange.buf;
886 }
887 } else if (dnsqrflow->dnsQRType == 28) {
888 yaf_dnsAAAA_t *aa = NULL;
889 if (state->aaaa_table == NULL) {
890 state->aaaa_table = allocTypeTab(ctx->cfg->ctime);
891 }
892 while ((aa = (yaf_dnsAAAA_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), aa)))
893 {
894 md_type_tab = state->aaaa_table;
895 find.rrdata = (aa->sourceIPv6Address);
896 find.caplen = 16;
897 }
898 } else if (dnsqrflow->dnsQRType == 16) {
899 yaf_dnsTXT_t *txt = NULL;
900 if (state->txt_table == NULL) {
901 state->txt_table = allocTypeTab(ctx->cfg->ctime);
902 }
903 while ((txt = (yaf_dnsTXT_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), txt)))
904 {
905 md_type_tab = state->txt_table;
906 find.caplen = txt->dnsTXTData.len;
907 find.rrdata = txt->dnsTXTData.buf;
908 }
909 } else if (dnsqrflow->dnsQRType == 33) {
910 yaf_dnsSRV_t *srv = NULL;
911 if (state->srv_table == NULL) {
912 state->srv_table = allocTypeTab(ctx->cfg->ctime);
913 }
914 while ((srv = (yaf_dnsSRV_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), srv)))
915 {
916 md_type_tab = state->srv_table;
917 find.rrdata = srv->dnsSRVTarget.buf;
918 find.caplen = srv->dnsSRVTarget.len;
919 }
920 } else if (dnsqrflow->dnsQRType == 6) {
921 yaf_dnsSOA_t *soa = NULL;
922 if (state->soa_table == NULL) {
923 state->soa_table = allocTypeTab(ctx->cfg->ctime);
924 }
925 while ((soa = (yaf_dnsSOA_t *)FBSTLNEXT(&(dnsqrflow->dnsRRList), soa))) {
926 md_type_tab = state->soa_table;
927 find.rrdata = soa->dnsSOAMName.buf;
928 find.caplen = soa->dnsSOAMName.len;
929 }
930 } else {
931 /* we don't do this one */
932 state->stats.dns_filtered++;
933 continue;
934 }
935 }
936
937 if (find.caplen == 0 && find.ip == 0) {
938 if (nx == 0) {
939 /* got nothing */
940 state->stats.dns_filtered++;
941 continue;
942 }
943 }
944
945 /* update stats */
946 state->stats.dns_recvd++;
947
948 if (state->map) {
949 mapkey = smFieldMapTranslate(state->map, flow);
950 if (state->map->discard && mapkey == 0) {
951 return;
952 }
953 memcpy(namebuf, &mapkey, sizeof(uint32_t));
954 name_offset += sizeof(uint32_t);
955 namelen += sizeof(uint32_t);
956 }
957
958 memcpy(namebuf + name_offset, dnsqrflow->dnsQName.buf,
959 dnsqrflow->dnsQName.len);
960 /* get rid of trailing "." */
961 namelen += dnsqrflow->dnsQName.len;
962 key.val = namebuf;
963 key.len = namelen;
964 /* namebuf[dnsqrflow->dnsQName.len] = '\0'; */
965 if (( hn = smHashLookup(md_type_tab->table, (uint8_t *)&key))) {
966 /*if (( hn = g_hash_table_lookup(md_type_tab->table,
967 namebuf)))
968 {*/
969 for (tn = hn->head; tn; tn = cn) {
970 cn = tn->next;
971 if (find.rrtype != tn->rrtype) {
972 continue;
973 }
974 if (find.ip && (dnsqrflow->dnsQRType == 1)) {
975 if (find.ip == tn->ip) {
976 ++(tn->hitcount);
977 tn->ltime = ctx->cfg->ctime;
978 if (find.ttl > tn->ttl) tn->ttl = find.ttl;
979 if (tn->hitcount == state->dedup_max_hit_count) {
980 cacheNodeClose(md_type_tab, hn, tn, state);
981 } else {
982 hashCacheTick(state, md_type_tab, hn, tn);
983 hashTick(state->a_table, hn);
984 }
985 found = TRUE;
986 break;
987 }
988 } else if (find.caplen == tn->caplen) {
989 if (memcmp(find.rrdata, tn->rrdata, find.caplen) == 0) {
990 /* match */
991 ++(tn->hitcount);
992 tn->ltime = ctx->cfg->ctime;
993 if (find.ttl > tn->ttl) tn->ttl = find.ttl;
994 if (tn->hitcount == state->dedup_max_hit_count) {
995 cacheNodeClose(md_type_tab, hn, tn, state);
996 } else {
997 hashCacheTick(state, md_type_tab, hn, tn);
998 hashTick(md_type_tab, hn);
999 }
1000 found = TRUE;
1001 break;
1002 }
1003 }
1004 }
1005 } else {
1006 hn = g_slice_new0(md_hashtab_node_t);
1007 if (hn == NULL) {
1008 md_dns_reset_dedup(state, ctx->cfg->ctime);
1009 hn = g_slice_new0(md_hashtab_node_t);
1010 }
1011
1012 /* copy key over */
1013 /*hn->rrname = g_slice_alloc0(dnsqrflow->dnsQName.len + 1);*/
1014 hn->rkey = sm_new_hash_key(key.val, key.len);
1015 /* if (hn->rrname == NULL) {*/
1016 if (hn->rkey == NULL) {
1017 md_dns_reset_dedup(state, ctx->cfg->ctime);
1018 /*hn->rrname = g_slice_alloc0(dnsqrflow->dnsQName.len + 1);*/
1019 hn->rkey = sm_new_hash_key(key.val, key.len);
1020 }
1021
1022 if (state->map) {
1023 hn->mapindex = mapkey;
1024 } else {
1025 hn->mapindex = -1;
1026 }
1027
1028
1029 /*memcpy(hn->rrname, dnsqrflow->dnsQName.buf,
1030 dnsqrflow->dnsQName.len);
1031 hn->rrname_len = dnsqrflow->dnsQName.len;
1032 *(hn->rrname + hn->rrname_len) = '\0';
1033 hn->rrname_len += 1;*/
1034 /* Insert into hash table */
1035 smHashTableInsert(md_type_tab->table, (uint8_t*)hn->rkey,
1036 (uint8_t*)hn);
1037 /*g_hash_table_insert(md_type_tab->table, hn->rrname, hn);*/
1038 ++(md_type_tab->count);
1039 }
1040
1041 if (!found) {
1042 cn = newCacheNode(ctx->cfg->ctime, find.ip, find.rrtype,
1043 find.rrdata, find.caplen);
1044 if (cn == NULL) {
1045 md_dns_reset_dedup(state, ctx->cfg->ctime);
1046 cn = newCacheNode(ctx->cfg->ctime, find.ip, find.rrtype,
1047 find.rrdata, find.caplen);
1048 }
1049 cn->ttl = find.ttl;
1050 if (!state->print_lastseen) {
1051 md_dns_emit_record(state, state->cq, hn, cn);
1052 }
1053 hashCacheTick(state, md_type_tab, hn, cn);
1054 if (hn) hashTick(md_type_tab, hn);
1055 }
1056 }
1057
1058 /* attempt a flush on all tables */
1059 md_dns_attempt_all_flush(state, ctx->cfg->ctime);
1060 }
1061