1 /**
2  * @file mediator_inf.h
3  *
4  * Yaf mediator for filtering, DNS deduplication, and other mediator-like
5  * things
6  ** ------------------------------------------------------------------------
7  ** Copyright (C) 2012-2017 Carnegie Mellon University. All Rights Reserved.
8  ** ------------------------------------------------------------------------
9  * Authors: Emily Sarneso
10  * -------------------------------------------------------------------------
11  * @OPENSOURCE_HEADER_START@
12  * Use of this (and related) source code is subject to the terms
13  * of the following licenses:
14  *
15  * GNU Public License (GPL) Rights pursuant to Version 2, June 1991
16  * Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013
17  *
18  *
19  * This material is based upon work funded and supported by
20  * the Department of Defense under Contract FA8721-05-C-0003 with
21  * Carnegie Mellon University for the operation of the Software Engineering
22  * Institue, a federally funded research and development center. Any opinions,
23  * findings and conclusions or recommendations expressed in this
24  * material are those of the author(s) and do not
25  * necessarily reflect the views of the United States
26  * Department of Defense.
27  *
28  * NO WARRANTY
29  *
30  * THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE
31  * MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY
32  * MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED
33  * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
34  * FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
35  * OBTAINED FROM THE USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY
36  * DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM
37  * PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
38  *
39  * This material has been approved for public release and unlimited
40  * distribution.
41  *
42  * Carnegie Mellon®, CERT® and CERT Coordination Center® are
43  * registered marks of Carnegie Mellon University.
44  *
45  * DM-0001877
46  *
47  * Carnegie Mellon University retains
48  * copyrights in all material produced under this contract. The U.S.
49  * Government retains a non-exclusive, royalty-free license to publish or
50  * reproduce these documents, or allow others to do so, for U.S.
51  * Government purposes only pursuant to the copyright license under the
52  * contract clause at 252.227.7013.
53  * Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
54  * Mellon University, its trustees, officers, employees, and agents from
55  * all claims or demands made against them (and any related losses,
56  * expenses, or attorney's fees) arising out of, or relating to Licensee's
57  * and/or its sub licensees' negligent use or willful misuse of or
58  * negligent conduct or willful misconduct regarding the Software,
59  * facilities, or other rights or assistance granted by Carnegie Mellon
60  * University under this License, including, but not limited to, any
61  * claims of product liability, personal injury, death, damage to
62  * property, or violation of any laws or regulations.
63  *
64  * @OPENSOURCE_HEADER_END@
65  * -----------------------------------------------------------
66  */
67 
68 
69 #include "templates.h"
70 #include "mediator_ctx.h"
71 #include <pthread.h>
72 #if ENABLE_SKIPSET
73 #include SKIPSET_HEADER_NAME
74 #ifdef HAVE_SILK_UTILS_H
75 #include <silk/utils.h>
76 #endif
77 #endif
78 
79 #define FTP_DEFAULT   "ftp"
80 #define SSH_DEFAULT   "ssh"
81 #define SMTP_DEFAULT  "smtp"
82 #define DNS_DEFAULT   "dns"
83 #define TFTP_DEFAULT  "tftp"
84 #define HTTP_DEFAULT  "http"
85 #define IMAP_DEFAULT  "imap"
86 #define IRC_DEFAULT   "irc"
87 #define SIP_DEFAULT   "sip"
88 #define MYSQL_DEFAULT "mysql"
89 #define SLP_DEFAULT   "slp"
90 #define POP3_DEFAULT  "pop3"
91 #define RTSP_DEFAULT  "rtsp"
92 #define NNTP_DEFAULT  "nntp"
93 #define SSL_DEFAULT   "tls"
94 #define DHCP_DEFAULT  "dhcp"
95 #define P0F_DEFAULT   "p0f"
96 #define INDEX_DEFAULT "flow"
97 #define DNS_DEDUP_DEFAULT "dns"
98 #define FLOW_STATS_DEFAULT "flowstats"
99 #define YAF_STATS_DEFAULT "yaf_stats"
100 #define DNP_DEFAULT   "dnp"
101 #define RTP_DEFAULT   "rtp"
102 #define MODBUS_DEFAULT "modbus"
103 #define ENIP_DEFAULT   "enip"
104 
105 mdFlowExporter_t *mdNewFlowExporter(
106     mdTransportType_t type);
107 
108 mdFlowCollector_t *mdNewFlowCollector(
109     mdTransportType_t    mode,
110     char                 *name);
111 
112 gboolean mdCollectorsInit(
113     mdConfig_t            *md,
114     md_collect_node_t     *collector,
115     GError                **err);
116 
117 void mdInterruptListeners(
118     mdConfig_t        *cfg);
119 
120 void mdCollectorSetInSpec(
121     mdFlowCollector_t      *collector,
122     char                   *inspec);
123 
124 void mdCollectorSetDeleteFiles(
125     mdFlowCollector_t        *collector,
126     gboolean                 delete);
127 
128 void mdCollectorSetPollTime(
129     mdFlowCollector_t      *collector,
130     char                   *poll_time);
131 
132 void mdCollectorSetDecompressDir(
133     mdFlowCollector_t *collector,
134     char            *path);
135 
136 void mdCollectorSetMoveDir(
137     mdFlowCollector_t      *collector,
138     char                   *move_dir);
139 
140 void mdCollectorSetLockMode(
141     mdFlowCollector_t      *collector,
142     gboolean               lockmode);
143 
144 md_collect_node_t *mdCollectorFindListener(
145     md_collect_node_t *collector,
146     fbListener_t      *listener);
147 
148 void mdCollectorSetPort(
149     mdFlowCollector_t      *collector,
150     char               *port);
151 
152 void mdCollectorAddSpreadGroup(
153     mdFlowCollector_t          *collector,
154     char                     *group,
155     int                      group_no);
156 
157 char *mdCollectorGetName(
158     md_collect_node_t *node);
159 
160 uint8_t mdCollectorGetID(
161     md_collect_node_t *node);
162 
163 gboolean mdCollectorVerifySetup(
164     mdFlowCollector_t       *collector,
165     GError              **err);
166 
167 void *mdNewTable(
168     char    *table);
169 
170 void *mdGetTable(
171     int id);
172 
173 void mdBuildDefaultTableHash(void);
174 
175 gboolean mdInsertTableItem(
176     void    *table_name,
177     int     val);
178 
179 void mdInsertDPIFieldItem(
180     mdFlowExporter_t      *exporter,
181     int                   ie);
182 
183 mdFieldList_t *mdNewFieldList(void);
184 
185 void mdExporterSetPort(
186     mdFlowExporter_t *exporter,
187     char             *port);
188 
189 void mdExporterSetHost(
190     mdFlowExporter_t *exporter,
191     char             *host);
192 
193 void mdExporterSetRotate(
194     mdFlowExporter_t *exporter,
195     uint32_t         rotate);
196 
197 void mdExporterSetDelim(
198     mdFlowExporter_t *exporter,
199     char             *delim);
200 
201 void mdExporterSetDPIDelim(
202     mdFlowExporter_t *exporter,
203     char             *delim);
204 
205 void mdExporterSetFileSpec(
206     mdFlowExporter_t *exporter,
207     char             *spec);
208 
209 void mdExporterFree(
210     mdFlowExporter_t *exporter);
211 
212 void mdExporterSetLock(
213     mdFlowExporter_t *exporter);
214 
215 void  mdExporterDedupPerFlow(
216     mdFlowExporter_t *exporter);
217 
218 void mdExporterSetRemoveEmpty(
219     mdFlowExporter_t *exporter);
220 
221 gboolean mdExporterVerifySetup(
222     mdFlowExporter_t *exporter);
223 
224 void mdExporterSetName(
225     mdFlowExporter_t *exporter,
226     char             *name);
227 
228 void mdExporterGZIPFiles(
229     mdFlowExporter_t *exporter);
230 
231 gboolean mdExporterSetDPIOnly(
232     mdFlowExporter_t *exporter);
233 
234 gboolean mdExporterSetFlowOnly(
235     mdFlowExporter_t *exporter);
236 
237 void mdExporterSetDNSDeDup(
238     mdFlowExporter_t *exporter);
239 
240 void mdExporterSetDeDupConfig(
241     mdFlowExporter_t *exporter);
242 
243 void mdExporterSetSSLDeDupConfig(
244     mdFlowExporter_t *exporter);
245 
246 gboolean mdExporterSetSSLDeDupOnly(
247     mdFlowExporter_t *exporter,
248     gboolean          dedup_only);
249 
250 gboolean mdExporterGetDNSDedupStatus(
251     mdFlowExporter_t *exporter);
252 
253 void mdExporterSetDNSRespOnly(
254     mdFlowExporter_t *exporter);
255 
256 gboolean mdExporterSetDNSDeDupOnly(
257     mdFlowExporter_t *exporter);
258 
259 void mdExporterSetStats(
260     mdFlowExporter_t *exporter,
261     uint8_t          mode);
262 
263 void mdExporterSetNoFlowStats(
264     mdFlowExporter_t *exporter);
265 
266 void mdExporterSetJson(
267     mdFlowExporter_t *exporter);
268 
269 void mdExporterSetNoIndex(
270     mdFlowExporter_t *exporter,
271     gboolean         val);
272 
273 void mdExporterSetPrintHeader(
274     mdFlowExporter_t *exporter);
275 
276 void mdExporterSetEscapeChars(
277     mdFlowExporter_t *exporter);
278 
279 gboolean mdExportMultiFiles(
280     mdFlowExporter_t  *exporter);
281 
282 int mdExporterGetType(
283     mdFlowExporter_t *exporter);
284 
285 void mdExporterSetTimestampFiles(
286     mdFlowExporter_t *exporter);
287 
288 void mdExporterSetRemoveUploaded(
289     mdFlowExporter_t *exporter);
290 
291 void mdExportCustomList(
292     mdFlowExporter_t *exporter,
293     mdFieldList_t    *list);
294 
295 void mdExporterCustomListDPI(
296     mdFlowExporter_t *exporter);
297 
298 void mdExporterSetId(
299     mdFlowExporter_t *exporter,
300     uint8_t          id);
301 
302 gboolean mdExporterCompareNames(
303     mdFlowExporter_t *exporter,
304     char             *name);
305 
306 void mdExporterSetSSLConfig(
307     mdFlowExporter_t  *exporter,
308     int               *list,
309     int                type);
310 
311 gboolean mdExporterSetDNSRROnly(
312     mdFlowExporter_t *exporter,
313     int               mode);
314 
315 gboolean mdExporterAddMySQLInfo(
316     mdFlowExporter_t *exporter,
317     char             *user,
318     char             *password,
319     char             *db_name,
320     char             *db_host,
321     char             *table);
322 
323 void mdInterruptFlowSource(
324     mdConfig_t *md);
325 
326 
327 int mdExporterWriteFlow(
328     mdConfig_t          *cfg,
329     mdFlowExporter_t    *exporter,
330     mdFullFlow_t        *flow,
331     GError              **err);
332 
333 gboolean mdExporterWriteOptions(
334     mdConfig_t         *cfg,
335     mdFlowExporter_t   *exporter,
336     uint8_t           *rec,
337     size_t             rec_length,
338     uint16_t           tid,
339     GError             **err);
340 
341 gboolean mdExporterWriteRecord(
342     mdConfig_t        *cfg,
343     mdFlowExporter_t  *exporter,
344     uint16_t          tid,
345     uint8_t           *rec,
346     size_t             rec_length,
347     GError            **err);
348 
349 
350 gboolean mdExporterWriteDNSRRRecord(
351     mdConfig_t          *cfg,
352     mdFlowExporter_t    *exporter,
353     uint16_t            tid,
354     uint8_t             *rec,
355     size_t              rec_length,
356     GError              **err);
357 
358 gboolean mdCollectorWait(
359     mdContext_t *ctx,
360     GError      **err);
361 
362 gboolean mdCollectorRestartListener(
363     mdConfig_t         *md,
364     md_collect_node_t  *collector,
365     GError             **err);
366 
367 gboolean mdCollectorStartListeners(
368     mdConfig_t         *md,
369     md_collect_node_t  *collector,
370     GError             **err);
371 
372 gboolean mdExportersInit(
373     mdConfig_t       *cfg,
374     md_export_node_t *node,
375     GError            **err);
376 
377 gboolean mdExporterRestart(
378     mdConfig_t          *cfg,
379     mdFlowExporter_t    *exp,
380     GError              **err);
381 
382 void mdExporterUpdateStats(
383     mdConfig_t       *cfg,
384     gboolean         dedup);
385 
386 gboolean mdExporterDestroy(
387     mdConfig_t        *cfg,
388     GError            **err);
389 
390 void mdCollectorDestroy(
391     mdConfig_t    *cfg,
392     gboolean      active);
393 
394 
395 int mdExporterDPIFlowPrint(
396     mdFlowExporter_t   *exporter,
397     mdFullFlow_t       *flow,
398     char               *index_str,
399     size_t             index_len,
400     GError             **err);
401 
402 gboolean mdExporterTextDNSPrint(
403     mdFlowExporter_t   *exporter,
404     yaf_dnsQR_t      *dns);
405 
406 GString *mdExporterJsonDNSPrint(
407     mdFlowExporter_t   *exporter,
408     yaf_dnsQR_t      *dnsqrflow);
409 
410 gboolean mdExporterDPIGetIndexStr(
411     mdFlowExporter_t *exporter,
412     mdFullFlow_t     *flow);
413 
414 fBuf_t *mdCollectorSpread(
415     mdConfig_t        *md,
416     GError            **err);
417 
418 gboolean mdExporterConnectionReset(
419     mdConfig_t       *md_config,
420     GError           **err);
421 
422 gboolean mdExportDNSRR(
423     mdConfig_t       *cfg,
424     mdFlowExporter_t *exporter,
425     mdFullFlow_t     *flow,
426     uint16_t         tid,
427     GError           **err);
428 
429 void mdExporterSetMetadataExport(
430     mdFlowExporter_t *exporter);
431 
432 /** print functions */
433 int mdCustomFlowPrint(
434     mdFieldList_t      *list,
435     mdFullFlow_t       *fflow,
436     mdFlowExporter_t   *exporter,
437     GError             **err);
438 
439 md_collect_node_t *mdCollectorGetNode(
440     fBuf_t         *fbuf);
441 
442 void mdCollectorUpdateStats(
443     mdConfig_t        *cfg);
444 
445 /* various types of printing functions for basic lists, varfields */
446 
447 gboolean mdExportBLMultiFiles(
448     mdFlowExporter_t *exporter,
449     fbBasicList_t    *bl,
450     char             *index_str,
451     size_t           index_len,
452     char             *label,
453     gboolean         hex);
454 gboolean mdExportBL(
455     mdFlowExporter_t *exporter,
456     fbBasicList_t    *bl,
457     char             *index_str,
458     size_t           index_len,
459     char             *label,
460     gboolean         hex);
461 
462 gboolean mdExportBLCustomList(
463     mdFlowExporter_t *exporter,
464     fbBasicList_t    *bl,
465     char             *index_str,
466     size_t           index_len,
467     char             *label,
468     gboolean         hex);
469 
470 gboolean mdJsonizeBLElement(
471     mdFlowExporter_t *exporter,
472     fbBasicList_t    *bl,
473     char             *index_str,
474     size_t           index_len,
475     char             *label,
476     gboolean         hex);
477 
478 gboolean mdJsonizeVLElement(
479     mdFlowExporter_t    *exporter,
480     uint8_t             *buf,
481     char                *label,
482     char             *index_str,
483     size_t           index_len,
484     uint16_t            id,
485     size_t              buflen,
486     gboolean            hex);
487 
488 gboolean mdAppendDPIStr(
489     mdFlowExporter_t  *exporter,
490     uint8_t           *buf,
491     char              *label,
492     char             *index_str,
493     size_t           index_len,
494     uint16_t          id,
495     size_t            buflen,
496     gboolean          hex);
497 
498 gboolean mdAppendDPIStrMultiFiles(
499     mdFlowExporter_t  *exporter,
500     uint8_t           *buf,
501     char              *label,
502     char             *index_str,
503     size_t           index_len,
504     uint16_t          id,
505     size_t            buflen,
506     gboolean          hex);
507 
508 gboolean mdJsonifyNewSSLRecord(
509     mdFlowExporter_t    *exporter,
510     yaf_newssl_t      *sslflow,
511     gboolean            hex,
512     gboolean            escape);
513 
514 gboolean mdExporterTextNewSSLPrint(
515     mdFlowExporter_t    *exporter,
516     yaf_newssl_t      *sslflow,
517     char                *index_str,
518     size_t              index_len);
519 
520 gboolean mdExporterDedupFileOpen(
521     mdConfig_t          *cfg,
522     mdFlowExporter_t    *exporter,
523     FILE                **file,
524     char                **last_file,
525     char                *prefix,
526     uint64_t            *rotate);
527 
528 
529 void mdExporterDedupFileClose(
530     mdFlowExporter_t *exporter,
531     FILE             *fp,
532     char             *last_file);
533 
534 gboolean mdExporterSSLCertRecord(
535     mdConfig_t           *cfg,
536     mdFlowExporter_t     *exporter,
537     FILE                 *cert_file,
538     yaf_newssl_cert_t   *ssl,
539     yfSSLFullCert_t      *fullcert,
540     uint8_t              *issuer,
541     size_t               issuer_len,
542     uint8_t              cert_no,
543     GError               **err);
544 
545 gboolean mdExporterWriteSSLDedupRecord(
546     mdConfig_t        *cfg,
547     mdFlowExporter_t  *exporter,
548     uint16_t          tid,
549     uint8_t           *rec,
550     size_t             rec_length,
551     GError            **err);
552 
553 gboolean mdExporterWriteDedupRecord(
554     mdConfig_t           *cfg,
555     md_export_node_t     *enode,
556     FILE                 *fp,
557     md_dedup_t           *rec,
558     char                 *prefix,
559     uint16_t             int_tid,
560     uint16_t             ext_tid,
561     GError               **err);
562 
563 gboolean mdExporterSetSSLSHA1Hash(
564     mdFlowExporter_t *exporter);
565 
566 gboolean mdExporterSetSSLMD5Hash(
567     mdFlowExporter_t *exporter);
568 
569 char *mdExporterGetName(
570     mdFlowExporter_t *exporter);
571 
572 void mdExporterSetMovePath(
573     mdFlowExporter_t *exporter,
574     char           *path);
575 
576 void mdExporterSetNoFlow(
577     mdFlowExporter_t  *exporter);
578 
579 gboolean mdExporterDedupOnly(
580     mdFlowExporter_t *exporter);
581 
582 gboolean mdExporterGetJson(
583     mdFlowExporter_t *exporter);
584