1 /**
2  * @file mediator_ctx.h
3  *
4  * Yaf mediator for filtering, DNS deduplication, and other mediator-like
5  * things
6  *
7  ** ------------------------------------------------------------------------
8  ** Copyright (C) 2012-2018 Carnegie Mellon University. All Rights Reserved.
9  * -------------------------------------------------------------------------
10  * Authors: Emily Sarneso
11  * -------------------------------------------------------------------------
12  * @OPENSOURCE_HEADER_START@
13  * Use of this (and related) source code is subject to the terms
14  * of the following licenses:
15  *
16  * GNU Public License (GPL) Rights pursuant to Version 2, June 1991
17  * Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013
18  *
19  * This material is based upon work funded and supported by
20  * the Department of Defense under Contract FA8721-05-C-0003 with
21  * Carnegie Mellon University for the operation of the Software Engineering
22  * Institue, a federally funded research and development center. Any opinions,
23  * findings and conclusions or recommendations expressed in this
24  * material are those of the author(s) and do not
25  * necessarily reflect the views of the United States
26  * Department of Defense.
27  *
28  * NO WARRANTY
29  *
30  * THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE
31  * MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY
32  * MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED
33  * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
34  * FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
35  * OBTAINED FROM THE USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY
36  * DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM
37  * PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
38  *
39  * This material has been approved for public release and unlimited
40  * distribution.
41  *
42  * Carnegie Mellon®, CERT® and CERT Coordination Center® are
43  * registered marks of Carnegie Mellon University.
44  *
45  * DM-0001877
46  *
47  * Carnegie Mellon University retains
48  * copyrights in all material produced under this contract. The U.S.
49  * Government retains a non-exclusive, royalty-free license to publish or
50  * reproduce these documents, or allow others to do so, for U.S.
51  * Government purposes only pursuant to the copyright license under the
52  * contract clause at 252.227.7013.
53  *
54  * Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
55  * Mellon University, its trustees, officers, employees, and agents from
56  * all claims or demands made against them (and any related losses,
57  * expenses, or attorney's fees) arising out of, or relating to Licensee's
58  * and/or its sub licensees' negligent use or willful misuse of or
59  * negligent conduct or willful misconduct regarding the Software,
60  * facilities, or other rights or assistance granted by Carnegie Mellon
61  * University under this License, including, but not limited to, any
62  * claims of product liability, personal injury, death, damage to
63  * property, or violation of any laws or regulations.
64  *
65  * @OPENSOURCE_HEADER_END@
66  * -----------------------------------------------------------
67  */
68 
69 #ifndef MD_CTX
70 #define MD_CTX
71 
72 
73 #include <stdlib.h>
74 #include <stdio.h>
75 #include <stdint.h>
76 #include <signal.h>
77 #include <stdarg.h>
78 #include <time.h>
79 #include <libgen.h>
80 #include <unistd.h>
81 #include <errno.h>
82 #include <glib.h>
83 #include <sys/types.h>
84 #include <sys/socket.h>
85 #include <sys/fcntl.h>
86 #include <arpa/inet.h>
87 #include <netinet/in.h>
88 #include <assert.h>
89 #include <fixbuf/public.h>
90 #include "mediator_config.h"
91 #include "config.h"
92 
93 #if ENABLE_SKIPSET
94 #include SKIPSET_HEADER_NAME
95 #endif
96 
97 #define CERT_PEN 6871
98 
99 #define MAX_LIST 10
100 /* 30 sec */
101 #define MD_RESTART_MS 30000
102 #define PRINT_TIME_FMT "%04u-%02u-%02u %02u:%02u:%02u"
103 #define MD_MSGLEN_STD 65535
104 #define MAX_MAPS 100
105 typedef enum mdTransportType_en {
106     NONE,
107     TCP,
108     UDP,
109     SPREAD,
110     FILEHANDLER,
111     TEXT,
112     DIRECTORY
113 } mdTransportType_t;
114 
115 typedef enum fieldOperator_en {
116     OPER_UNTOUCHED,
117     IN_LIST,
118     NOT_IN_LIST,
119     EQUAL,
120     NOT_EQUAL,
121     LESS_THAN,
122     LESS_THAN_OR_EQUAL,
123     GREATER_THAN,
124     GREATER_THAN_OR_EQUAL
125 } fieldOperator;
126 
127 typedef enum mdAcceptFilterField_en {
128     SIP_ANY,
129     DIP_ANY,
130     SIP_V4,
131     DIP_V4,
132     SPORT,
133     DPORT,
134     PROTOCOL,
135     APPLICATION,
136     SIP_V6,
137     DIP_V6,
138     ANY_IP6,
139     ANY_IP,
140     ANY_PORT,
141     OBDOMAIN,
142     IPVERSION,
143     VLAN,
144     FLOWKEYHASH,
145     DURATION,
146     STIME,
147     ENDTIME,
148     STIMEMS,
149     ETIMEMS,
150     SIP_INT,
151     DIP_INT,
152     RTT,
153     PKTS,
154     RPKTS,
155     BYTES,
156     RBYTES,
157     IFLAGS,
158     RIFLAGS,
159     UFLAGS,
160     RUFLAGS,
161     ATTRIBUTES,
162     RATTRIBUTES,
163     MAC,
164     DSTMAC,
165     TCPSEQ,
166     RTCPSEQ,
167     ENTROPY,
168     RENTROPY,
169     END,
170     OSNAME,
171     OSVERSION,
172     ROSNAME,
173     ROSVERSION,
174     FINGERPRINT,
175     RFINGERPRINT,
176     DHCPFP,
177     DHCPVC,
178     RDHCPFP,
179     RDHCPVC,
180     INGRESS,
181     EGRESS,
182     DATABYTES,
183     RDATABYTES,
184     ITIME,
185     RITIME,
186     STDITIME,
187     RSTDITIME,
188     TCPURG,
189     RTCPURG,
190     SMALLPKTS,
191     RSMALLPKTS,
192     LARGEPKTS,
193     RLARGEPKTS,
194     NONEMPTYPKTS,
195     RNONEMPTYPKTS,
196     MAXSIZE,
197     RMAXSIZE,
198     STDPAYLEN,
199     RSTDPAYLEN,
200     FIRSTEIGHT,
201     DPI,
202     VLANINT,
203     TOS,
204     RTOS,
205     MPLS1,
206     MPLS2,
207     MPLS3,
208     COLLECTOR,
209     FIRSTNONEMPTY,
210     RFIRSTNONEMPTY,
211     MPTCPSEQ,
212     MPTCPTOKEN,
213     MPTCPMSS,
214     MPTCPID,
215     MPTCPFLAGS,
216     PAYLOAD,
217     RPAYLOAD,
218     DHCPOPTIONS,
219     RDHCPOPTIONS,
220     NDPI_MASTER,
221     NDPI_SUB,
222     NONE_FIELD
223 } mdAcceptFilterField_t;
224 
225 typedef enum mdLogLevel_en {
226     MD_DEBUG,
227     MESSAGE,
228     WARNING,
229     ERROR,
230     QUIET
231 } mdLogLevel_t;
232 
233 typedef struct mdConfig_st mdConfig_t;
234 
235 /* configuration options */
236 extern int             myVersion;
237 extern int             md_stats_timeout;
238 extern mdConfig_t      md_config;
239 #if HAVE_SPREAD
240 char                   **md_out_groups;
241 extern int             num_out_groups;
242 #endif
243 extern char            *md_logfile;
244 extern char            *md_logdest;
245 extern char            *md_pidfile;
246 extern mdLogLevel_t    md_log_level;
247 extern uint16_t        dns_max_hit_count;
248 extern uint16_t        dns_flush_timeout;
249 extern gboolean        multi_file_mode;
250 extern fbInfoElement_t  *user_elements;
251 
252 struct mdFlowCollector_st;
253 typedef struct mdFlowCollector_st mdFlowCollector_t;
254 
255 struct mdFlowExporter_st;
256 typedef struct mdFlowExporter_st mdFlowExporter_t;
257 
258 typedef struct mdDLL_st mdDLL_t;
259 
260 struct mdDLL_st {
261     mdDLL_t *next;
262     mdDLL_t *prev;
263 };
264 
265 typedef struct mdSLL_st mdSLL_t;
266 
267 struct mdSLL_st {
268     mdSLL_t *next;
269 };
270 
271 typedef struct mdQueue_st {
272     mdDLL_t *head;
273     mdDLL_t *tail;
274 } mdQueue_t;
275 
276 typedef struct smHashTable_st {
277     size_t     len;
278     GHashTable *table;
279 } smHashTable_t;
280 
281 typedef struct smFieldMap_st smFieldMap_t;
282 
283 struct smFieldMap_st {
284     smFieldMap_t            *next;
285     mdAcceptFilterField_t   field;
286     smHashTable_t           *table;
287     char                    *name;
288     char                   **labels;
289     size_t                  count;
290     gboolean                discard;
291 };
292 
293 typedef struct smFieldMapKV_st {
294     uint32_t              val;
295 } smFieldMapKV_t;
296 
297 typedef struct md_dns_node_st md_dns_node_t;
298 
299 /* dns close queue */
300 typedef struct md_dns_cqueue_st {
301     md_dns_node_t *head;
302     md_dns_node_t *tail;
303 } md_dns_cqueue_t;
304 
305 typedef struct md_dns_dedup_state_st md_dns_dedup_state_t;
306 
307 typedef struct md_dedup_state_st md_dedup_state_t;
308 typedef struct md_dedup_str_node_st md_dedup_str_node_t;
309 
310 typedef struct md_ssl_dedup_state_st md_ssl_dedup_state_t;
311 
312 typedef struct md_filter_st md_filter_t;
313 
314 struct md_filter_st {
315     md_filter_t           *next;
316     fieldOperator         oper;
317     mdAcceptFilterField_t field;
318 #if ENABLE_SKIPSET
319     skipset_t             *ipset;
320 #endif
321     uint8_t               num_in_list;
322     uint32_t              val[MAX_LIST];
323 };
324 
325 typedef struct md_spread_filter_st md_spread_filter_t;
326 
327 struct md_spread_filter_st {
328     md_spread_filter_t     *next;
329     char                   *group;
330     md_filter_t            *filterList;
331 };
332 
333 typedef struct md_export_node_st md_export_node_t;
334 
335 struct md_export_node_st {
336     md_export_node_t       *next;
337     mdFlowExporter_t       *exp;
338     md_filter_t            *filter;
339     md_dns_dedup_state_t   *dns_dedup;
340     md_dedup_state_t       *dedup;
341     md_ssl_dedup_state_t   *ssl_dedup;
342     gboolean               and_filter;
343     gboolean               md5_hash;
344     gboolean               sha1_hash;
345 };
346 
347 typedef struct md_stats_st {
348     uint64_t              recvd_flows;
349     uint64_t              dns;
350     uint64_t              recvd_filtered;
351     uint64_t              recvd_stats;
352     uint64_t              nonstd_flows;
353     uint64_t              uniflows;
354     uint32_t              files;
355     uint16_t              restarts;
356 } md_stats_t;
357 
358 typedef struct md_collect_node_st md_collect_node_t;
359 
360 struct md_collect_node_st {
361     md_collect_node_t      *next;
362     mdFlowCollector_t      *coll;
363     md_filter_t            *filter;
364     fBuf_t                 *fbuf;
365     md_stats_t             *stats;
366     pthread_cond_t         cond;
367     pthread_mutex_t        mutex;
368     gboolean               and_filter;
369     gboolean               active;
370 };
371 
372 typedef struct mdBuf_st {
373     char   *cp;
374     char   *buf;
375     size_t buflen;
376 } mdBuf_t;
377 
378 struct mdConfig_st {
379     md_collect_node_t       *flowsrc;
380     md_export_node_t        *flowexit;
381     smFieldMap_t            *maps;
382     FILE                    *log;
383     md_spread_filter_t      *mdspread;
384     char                    *collector_name;
385     pthread_cond_t          log_cond;
386     pthread_mutex_t         log_mutex;
387     gboolean                no_stats;
388     gboolean                ipfixSpreadTrans;
389     gboolean                lockmode;
390     gboolean                dns_base64_encode;
391     gboolean                dns_print_lastseen;
392     gboolean                shared_filter;
393     gboolean                gen_tombstone;
394     uint16_t                tombstone_configured_id;
395     uint16_t                tombstone_unique_id;
396     uint64_t                udp_template_timeout;
397     uint64_t                ctime;
398     uint32_t                current_domain;
399     unsigned int            usec_sleep;
400     uint8_t                 num_listeners;
401     uint8_t                 collector_id;
402 #ifdef HAVE_SPREAD
403     fbSpreadParams_t        out_spread;
404 #endif
405 };
406 
407 #ifdef HAVE_SPREAD
408 #define MD_CONFIG_INIT { NULL, NULL, NULL, NULL, NULL, NULL, PTHREAD_COND_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, 0, 0, 600, 0, 0, 0, 0, 0, FB_SPREADPARAMS_INIT}
409 #else
410 #define MD_CONFIG_INIT { NULL, NULL, NULL, NULL, NULL, NULL, PTHREAD_COND_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, 0, 0, 600, 0, 0, 0, 0, 0}
411 #endif
412 
413 typedef struct mdContext_st {
414     mdConfig_t        *cfg;
415     md_stats_t        *stats;
416     GError            *err;
417 } mdContext_t;
418 
419 #define MD_CTX_INIT { NULL, NULL, NULL }
420 
421 #endif
422