1 /** 2 * @file mediator_ctx.h 3 * 4 * Yaf mediator for filtering, DNS deduplication, and other mediator-like 5 * things 6 * 7 ** ------------------------------------------------------------------------ 8 ** Copyright (C) 2012-2018 Carnegie Mellon University. All Rights Reserved. 9 * ------------------------------------------------------------------------- 10 * Authors: Emily Sarneso 11 * ------------------------------------------------------------------------- 12 * @OPENSOURCE_HEADER_START@ 13 * Use of this (and related) source code is subject to the terms 14 * of the following licenses: 15 * 16 * GNU Public License (GPL) Rights pursuant to Version 2, June 1991 17 * Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013 18 * 19 * This material is based upon work funded and supported by 20 * the Department of Defense under Contract FA8721-05-C-0003 with 21 * Carnegie Mellon University for the operation of the Software Engineering 22 * Institue, a federally funded research and development center. Any opinions, 23 * findings and conclusions or recommendations expressed in this 24 * material are those of the author(s) and do not 25 * necessarily reflect the views of the United States 26 * Department of Defense. 27 * 28 * NO WARRANTY 29 * 30 * THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE 31 * MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY 32 * MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED 33 * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF 34 * FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS 35 * OBTAINED FROM THE USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY 36 * DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM 37 * PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT. 38 * 39 * This material has been approved for public release and unlimited 40 * distribution. 41 * 42 * Carnegie Mellon®, CERT® and CERT Coordination Center® are 43 * registered marks of Carnegie Mellon University. 44 * 45 * DM-0001877 46 * 47 * Carnegie Mellon University retains 48 * copyrights in all material produced under this contract. The U.S. 49 * Government retains a non-exclusive, royalty-free license to publish or 50 * reproduce these documents, or allow others to do so, for U.S. 51 * Government purposes only pursuant to the copyright license under the 52 * contract clause at 252.227.7013. 53 * 54 * Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie 55 * Mellon University, its trustees, officers, employees, and agents from 56 * all claims or demands made against them (and any related losses, 57 * expenses, or attorney's fees) arising out of, or relating to Licensee's 58 * and/or its sub licensees' negligent use or willful misuse of or 59 * negligent conduct or willful misconduct regarding the Software, 60 * facilities, or other rights or assistance granted by Carnegie Mellon 61 * University under this License, including, but not limited to, any 62 * claims of product liability, personal injury, death, damage to 63 * property, or violation of any laws or regulations. 64 * 65 * @OPENSOURCE_HEADER_END@ 66 * ----------------------------------------------------------- 67 */ 68 69 #ifndef MD_CTX 70 #define MD_CTX 71 72 73 #include <stdlib.h> 74 #include <stdio.h> 75 #include <stdint.h> 76 #include <signal.h> 77 #include <stdarg.h> 78 #include <time.h> 79 #include <libgen.h> 80 #include <unistd.h> 81 #include <errno.h> 82 #include <glib.h> 83 #include <sys/types.h> 84 #include <sys/socket.h> 85 #include <sys/fcntl.h> 86 #include <arpa/inet.h> 87 #include <netinet/in.h> 88 #include <assert.h> 89 #include <fixbuf/public.h> 90 #include "mediator_config.h" 91 #include "config.h" 92 93 #if ENABLE_SKIPSET 94 #include SKIPSET_HEADER_NAME 95 #endif 96 97 #define CERT_PEN 6871 98 99 #define MAX_LIST 10 100 /* 30 sec */ 101 #define MD_RESTART_MS 30000 102 #define PRINT_TIME_FMT "%04u-%02u-%02u %02u:%02u:%02u" 103 #define MD_MSGLEN_STD 65535 104 #define MAX_MAPS 100 105 typedef enum mdTransportType_en { 106 NONE, 107 TCP, 108 UDP, 109 SPREAD, 110 FILEHANDLER, 111 TEXT, 112 DIRECTORY 113 } mdTransportType_t; 114 115 typedef enum fieldOperator_en { 116 OPER_UNTOUCHED, 117 IN_LIST, 118 NOT_IN_LIST, 119 EQUAL, 120 NOT_EQUAL, 121 LESS_THAN, 122 LESS_THAN_OR_EQUAL, 123 GREATER_THAN, 124 GREATER_THAN_OR_EQUAL 125 } fieldOperator; 126 127 typedef enum mdAcceptFilterField_en { 128 SIP_ANY, 129 DIP_ANY, 130 SIP_V4, 131 DIP_V4, 132 SPORT, 133 DPORT, 134 PROTOCOL, 135 APPLICATION, 136 SIP_V6, 137 DIP_V6, 138 ANY_IP6, 139 ANY_IP, 140 ANY_PORT, 141 OBDOMAIN, 142 IPVERSION, 143 VLAN, 144 FLOWKEYHASH, 145 DURATION, 146 STIME, 147 ENDTIME, 148 STIMEMS, 149 ETIMEMS, 150 SIP_INT, 151 DIP_INT, 152 RTT, 153 PKTS, 154 RPKTS, 155 BYTES, 156 RBYTES, 157 IFLAGS, 158 RIFLAGS, 159 UFLAGS, 160 RUFLAGS, 161 ATTRIBUTES, 162 RATTRIBUTES, 163 MAC, 164 DSTMAC, 165 TCPSEQ, 166 RTCPSEQ, 167 ENTROPY, 168 RENTROPY, 169 END, 170 OSNAME, 171 OSVERSION, 172 ROSNAME, 173 ROSVERSION, 174 FINGERPRINT, 175 RFINGERPRINT, 176 DHCPFP, 177 DHCPVC, 178 RDHCPFP, 179 RDHCPVC, 180 INGRESS, 181 EGRESS, 182 DATABYTES, 183 RDATABYTES, 184 ITIME, 185 RITIME, 186 STDITIME, 187 RSTDITIME, 188 TCPURG, 189 RTCPURG, 190 SMALLPKTS, 191 RSMALLPKTS, 192 LARGEPKTS, 193 RLARGEPKTS, 194 NONEMPTYPKTS, 195 RNONEMPTYPKTS, 196 MAXSIZE, 197 RMAXSIZE, 198 STDPAYLEN, 199 RSTDPAYLEN, 200 FIRSTEIGHT, 201 DPI, 202 VLANINT, 203 TOS, 204 RTOS, 205 MPLS1, 206 MPLS2, 207 MPLS3, 208 COLLECTOR, 209 FIRSTNONEMPTY, 210 RFIRSTNONEMPTY, 211 MPTCPSEQ, 212 MPTCPTOKEN, 213 MPTCPMSS, 214 MPTCPID, 215 MPTCPFLAGS, 216 PAYLOAD, 217 RPAYLOAD, 218 DHCPOPTIONS, 219 RDHCPOPTIONS, 220 NDPI_MASTER, 221 NDPI_SUB, 222 NONE_FIELD 223 } mdAcceptFilterField_t; 224 225 typedef enum mdLogLevel_en { 226 MD_DEBUG, 227 MESSAGE, 228 WARNING, 229 ERROR, 230 QUIET 231 } mdLogLevel_t; 232 233 typedef struct mdConfig_st mdConfig_t; 234 235 /* configuration options */ 236 extern int myVersion; 237 extern int md_stats_timeout; 238 extern mdConfig_t md_config; 239 #if HAVE_SPREAD 240 char **md_out_groups; 241 extern int num_out_groups; 242 #endif 243 extern char *md_logfile; 244 extern char *md_logdest; 245 extern char *md_pidfile; 246 extern mdLogLevel_t md_log_level; 247 extern uint16_t dns_max_hit_count; 248 extern uint16_t dns_flush_timeout; 249 extern gboolean multi_file_mode; 250 extern fbInfoElement_t *user_elements; 251 252 struct mdFlowCollector_st; 253 typedef struct mdFlowCollector_st mdFlowCollector_t; 254 255 struct mdFlowExporter_st; 256 typedef struct mdFlowExporter_st mdFlowExporter_t; 257 258 typedef struct mdDLL_st mdDLL_t; 259 260 struct mdDLL_st { 261 mdDLL_t *next; 262 mdDLL_t *prev; 263 }; 264 265 typedef struct mdSLL_st mdSLL_t; 266 267 struct mdSLL_st { 268 mdSLL_t *next; 269 }; 270 271 typedef struct mdQueue_st { 272 mdDLL_t *head; 273 mdDLL_t *tail; 274 } mdQueue_t; 275 276 typedef struct smHashTable_st { 277 size_t len; 278 GHashTable *table; 279 } smHashTable_t; 280 281 typedef struct smFieldMap_st smFieldMap_t; 282 283 struct smFieldMap_st { 284 smFieldMap_t *next; 285 mdAcceptFilterField_t field; 286 smHashTable_t *table; 287 char *name; 288 char **labels; 289 size_t count; 290 gboolean discard; 291 }; 292 293 typedef struct smFieldMapKV_st { 294 uint32_t val; 295 } smFieldMapKV_t; 296 297 typedef struct md_dns_node_st md_dns_node_t; 298 299 /* dns close queue */ 300 typedef struct md_dns_cqueue_st { 301 md_dns_node_t *head; 302 md_dns_node_t *tail; 303 } md_dns_cqueue_t; 304 305 typedef struct md_dns_dedup_state_st md_dns_dedup_state_t; 306 307 typedef struct md_dedup_state_st md_dedup_state_t; 308 typedef struct md_dedup_str_node_st md_dedup_str_node_t; 309 310 typedef struct md_ssl_dedup_state_st md_ssl_dedup_state_t; 311 312 typedef struct md_filter_st md_filter_t; 313 314 struct md_filter_st { 315 md_filter_t *next; 316 fieldOperator oper; 317 mdAcceptFilterField_t field; 318 #if ENABLE_SKIPSET 319 skipset_t *ipset; 320 #endif 321 uint8_t num_in_list; 322 uint32_t val[MAX_LIST]; 323 }; 324 325 typedef struct md_spread_filter_st md_spread_filter_t; 326 327 struct md_spread_filter_st { 328 md_spread_filter_t *next; 329 char *group; 330 md_filter_t *filterList; 331 }; 332 333 typedef struct md_export_node_st md_export_node_t; 334 335 struct md_export_node_st { 336 md_export_node_t *next; 337 mdFlowExporter_t *exp; 338 md_filter_t *filter; 339 md_dns_dedup_state_t *dns_dedup; 340 md_dedup_state_t *dedup; 341 md_ssl_dedup_state_t *ssl_dedup; 342 gboolean and_filter; 343 gboolean md5_hash; 344 gboolean sha1_hash; 345 }; 346 347 typedef struct md_stats_st { 348 uint64_t recvd_flows; 349 uint64_t dns; 350 uint64_t recvd_filtered; 351 uint64_t recvd_stats; 352 uint64_t nonstd_flows; 353 uint64_t uniflows; 354 uint32_t files; 355 uint16_t restarts; 356 } md_stats_t; 357 358 typedef struct md_collect_node_st md_collect_node_t; 359 360 struct md_collect_node_st { 361 md_collect_node_t *next; 362 mdFlowCollector_t *coll; 363 md_filter_t *filter; 364 fBuf_t *fbuf; 365 md_stats_t *stats; 366 pthread_cond_t cond; 367 pthread_mutex_t mutex; 368 gboolean and_filter; 369 gboolean active; 370 }; 371 372 typedef struct mdBuf_st { 373 char *cp; 374 char *buf; 375 size_t buflen; 376 } mdBuf_t; 377 378 struct mdConfig_st { 379 md_collect_node_t *flowsrc; 380 md_export_node_t *flowexit; 381 smFieldMap_t *maps; 382 FILE *log; 383 md_spread_filter_t *mdspread; 384 char *collector_name; 385 pthread_cond_t log_cond; 386 pthread_mutex_t log_mutex; 387 gboolean no_stats; 388 gboolean ipfixSpreadTrans; 389 gboolean lockmode; 390 gboolean dns_base64_encode; 391 gboolean dns_print_lastseen; 392 gboolean shared_filter; 393 gboolean gen_tombstone; 394 uint16_t tombstone_configured_id; 395 uint16_t tombstone_unique_id; 396 uint64_t udp_template_timeout; 397 uint64_t ctime; 398 uint32_t current_domain; 399 unsigned int usec_sleep; 400 uint8_t num_listeners; 401 uint8_t collector_id; 402 #ifdef HAVE_SPREAD 403 fbSpreadParams_t out_spread; 404 #endif 405 }; 406 407 #ifdef HAVE_SPREAD 408 #define MD_CONFIG_INIT { NULL, NULL, NULL, NULL, NULL, NULL, PTHREAD_COND_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, 0, 0, 600, 0, 0, 0, 0, 0, FB_SPREADPARAMS_INIT} 409 #else 410 #define MD_CONFIG_INIT { NULL, NULL, NULL, NULL, NULL, NULL, PTHREAD_COND_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, 0, 0, 600, 0, 0, 0, 0, 0} 411 #endif 412 413 typedef struct mdContext_st { 414 mdConfig_t *cfg; 415 md_stats_t *stats; 416 GError *err; 417 } mdContext_t; 418 419 #define MD_CTX_INIT { NULL, NULL, NULL } 420 421 #endif 422