1 /** 2 * @file mediator_inf.h 3 * 4 * Yaf mediator for filtering, DNS deduplication, and other mediator-like 5 * things 6 ** ------------------------------------------------------------------------ 7 ** Copyright (C) 2012-2017 Carnegie Mellon University. All Rights Reserved. 8 ** ------------------------------------------------------------------------ 9 * Authors: Emily Sarneso 10 * ------------------------------------------------------------------------- 11 * @OPENSOURCE_HEADER_START@ 12 * Use of this (and related) source code is subject to the terms 13 * of the following licenses: 14 * 15 * GNU Public License (GPL) Rights pursuant to Version 2, June 1991 16 * Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013 17 * 18 * 19 * This material is based upon work funded and supported by 20 * the Department of Defense under Contract FA8721-05-C-0003 with 21 * Carnegie Mellon University for the operation of the Software Engineering 22 * Institue, a federally funded research and development center. Any opinions, 23 * findings and conclusions or recommendations expressed in this 24 * material are those of the author(s) and do not 25 * necessarily reflect the views of the United States 26 * Department of Defense. 27 * 28 * NO WARRANTY 29 * 30 * THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE 31 * MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY 32 * MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED 33 * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF 34 * FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS 35 * OBTAINED FROM THE USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY 36 * DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM 37 * PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT. 38 * 39 * This material has been approved for public release and unlimited 40 * distribution. 41 * 42 * Carnegie Mellon®, CERT® and CERT Coordination Center® are 43 * registered marks of Carnegie Mellon University. 44 * 45 * DM-0001877 46 * 47 * Carnegie Mellon University retains 48 * copyrights in all material produced under this contract. The U.S. 49 * Government retains a non-exclusive, royalty-free license to publish or 50 * reproduce these documents, or allow others to do so, for U.S. 51 * Government purposes only pursuant to the copyright license under the 52 * contract clause at 252.227.7013. 53 * Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie 54 * Mellon University, its trustees, officers, employees, and agents from 55 * all claims or demands made against them (and any related losses, 56 * expenses, or attorney's fees) arising out of, or relating to Licensee's 57 * and/or its sub licensees' negligent use or willful misuse of or 58 * negligent conduct or willful misconduct regarding the Software, 59 * facilities, or other rights or assistance granted by Carnegie Mellon 60 * University under this License, including, but not limited to, any 61 * claims of product liability, personal injury, death, damage to 62 * property, or violation of any laws or regulations. 63 * 64 * @OPENSOURCE_HEADER_END@ 65 * ----------------------------------------------------------- 66 */ 67 68 69 #include "templates.h" 70 #include "mediator_ctx.h" 71 #include <pthread.h> 72 #if ENABLE_SKIPSET 73 #include SKIPSET_HEADER_NAME 74 #ifdef HAVE_SILK_UTILS_H 75 #include <silk/utils.h> 76 #endif 77 #endif 78 79 #define FTP_DEFAULT "ftp" 80 #define SSH_DEFAULT "ssh" 81 #define SMTP_DEFAULT "smtp" 82 #define DNS_DEFAULT "dns" 83 #define TFTP_DEFAULT "tftp" 84 #define HTTP_DEFAULT "http" 85 #define IMAP_DEFAULT "imap" 86 #define IRC_DEFAULT "irc" 87 #define SIP_DEFAULT "sip" 88 #define MYSQL_DEFAULT "mysql" 89 #define SLP_DEFAULT "slp" 90 #define POP3_DEFAULT "pop3" 91 #define RTSP_DEFAULT "rtsp" 92 #define NNTP_DEFAULT "nntp" 93 #define SSL_DEFAULT "tls" 94 #define DHCP_DEFAULT "dhcp" 95 #define P0F_DEFAULT "p0f" 96 #define INDEX_DEFAULT "flow" 97 #define DNS_DEDUP_DEFAULT "dns" 98 #define FLOW_STATS_DEFAULT "flowstats" 99 #define YAF_STATS_DEFAULT "yaf_stats" 100 #define DNP_DEFAULT "dnp" 101 #define RTP_DEFAULT "rtp" 102 #define MODBUS_DEFAULT "modbus" 103 #define ENIP_DEFAULT "enip" 104 105 mdFlowExporter_t *mdNewFlowExporter( 106 mdTransportType_t type); 107 108 mdFlowCollector_t *mdNewFlowCollector( 109 mdTransportType_t mode, 110 char *name); 111 112 gboolean mdCollectorsInit( 113 mdConfig_t *md, 114 md_collect_node_t *collector, 115 GError **err); 116 117 void mdInterruptListeners( 118 mdConfig_t *cfg); 119 120 void mdCollectorSetInSpec( 121 mdFlowCollector_t *collector, 122 char *inspec); 123 124 void mdCollectorSetDeleteFiles( 125 mdFlowCollector_t *collector, 126 gboolean delete); 127 128 void mdCollectorSetPollTime( 129 mdFlowCollector_t *collector, 130 char *poll_time); 131 132 void mdCollectorSetDecompressDir( 133 mdFlowCollector_t *collector, 134 char *path); 135 136 void mdCollectorSetMoveDir( 137 mdFlowCollector_t *collector, 138 char *move_dir); 139 140 void mdCollectorSetLockMode( 141 mdFlowCollector_t *collector, 142 gboolean lockmode); 143 144 md_collect_node_t *mdCollectorFindListener( 145 md_collect_node_t *collector, 146 fbListener_t *listener); 147 148 void mdCollectorSetPort( 149 mdFlowCollector_t *collector, 150 char *port); 151 152 void mdCollectorAddSpreadGroup( 153 mdFlowCollector_t *collector, 154 char *group, 155 int group_no); 156 157 char *mdCollectorGetName( 158 md_collect_node_t *node); 159 160 uint8_t mdCollectorGetID( 161 md_collect_node_t *node); 162 163 gboolean mdCollectorVerifySetup( 164 mdFlowCollector_t *collector, 165 GError **err); 166 167 void *mdNewTable( 168 char *table); 169 170 void *mdGetTable( 171 int id); 172 173 void mdBuildDefaultTableHash(void); 174 175 gboolean mdInsertTableItem( 176 void *table_name, 177 int val); 178 179 void mdInsertDPIFieldItem( 180 mdFlowExporter_t *exporter, 181 int ie); 182 183 mdFieldList_t *mdNewFieldList(void); 184 185 void mdExporterSetPort( 186 mdFlowExporter_t *exporter, 187 char *port); 188 189 void mdExporterSetHost( 190 mdFlowExporter_t *exporter, 191 char *host); 192 193 void mdExporterSetRotate( 194 mdFlowExporter_t *exporter, 195 uint32_t rotate); 196 197 void mdExporterSetDelim( 198 mdFlowExporter_t *exporter, 199 char *delim); 200 201 void mdExporterSetDPIDelim( 202 mdFlowExporter_t *exporter, 203 char *delim); 204 205 void mdExporterSetFileSpec( 206 mdFlowExporter_t *exporter, 207 char *spec); 208 209 void mdExporterFree( 210 mdFlowExporter_t *exporter); 211 212 void mdExporterSetLock( 213 mdFlowExporter_t *exporter); 214 215 void mdExporterDedupPerFlow( 216 mdFlowExporter_t *exporter); 217 218 void mdExporterSetRemoveEmpty( 219 mdFlowExporter_t *exporter); 220 221 gboolean mdExporterVerifySetup( 222 mdFlowExporter_t *exporter); 223 224 void mdExporterSetName( 225 mdFlowExporter_t *exporter, 226 char *name); 227 228 void mdExporterGZIPFiles( 229 mdFlowExporter_t *exporter); 230 231 gboolean mdExporterSetDPIOnly( 232 mdFlowExporter_t *exporter); 233 234 gboolean mdExporterSetFlowOnly( 235 mdFlowExporter_t *exporter); 236 237 void mdExporterSetDNSDeDup( 238 mdFlowExporter_t *exporter); 239 240 void mdExporterSetDeDupConfig( 241 mdFlowExporter_t *exporter); 242 243 void mdExporterSetSSLDeDupConfig( 244 mdFlowExporter_t *exporter); 245 246 gboolean mdExporterSetSSLDeDupOnly( 247 mdFlowExporter_t *exporter, 248 gboolean dedup_only); 249 250 gboolean mdExporterGetDNSDedupStatus( 251 mdFlowExporter_t *exporter); 252 253 void mdExporterSetDNSRespOnly( 254 mdFlowExporter_t *exporter); 255 256 gboolean mdExporterSetDNSDeDupOnly( 257 mdFlowExporter_t *exporter); 258 259 void mdExporterSetStats( 260 mdFlowExporter_t *exporter, 261 uint8_t mode); 262 263 void mdExporterSetNoFlowStats( 264 mdFlowExporter_t *exporter); 265 266 void mdExporterSetJson( 267 mdFlowExporter_t *exporter); 268 269 void mdExporterSetNoIndex( 270 mdFlowExporter_t *exporter, 271 gboolean val); 272 273 void mdExporterSetPrintHeader( 274 mdFlowExporter_t *exporter); 275 276 void mdExporterSetEscapeChars( 277 mdFlowExporter_t *exporter); 278 279 gboolean mdExportMultiFiles( 280 mdFlowExporter_t *exporter); 281 282 int mdExporterGetType( 283 mdFlowExporter_t *exporter); 284 285 void mdExporterSetTimestampFiles( 286 mdFlowExporter_t *exporter); 287 288 void mdExporterSetRemoveUploaded( 289 mdFlowExporter_t *exporter); 290 291 void mdExportCustomList( 292 mdFlowExporter_t *exporter, 293 mdFieldList_t *list); 294 295 void mdExporterCustomListDPI( 296 mdFlowExporter_t *exporter); 297 298 void mdExporterSetId( 299 mdFlowExporter_t *exporter, 300 uint8_t id); 301 302 gboolean mdExporterCompareNames( 303 mdFlowExporter_t *exporter, 304 char *name); 305 306 void mdExporterSetSSLConfig( 307 mdFlowExporter_t *exporter, 308 int *list, 309 int type); 310 311 gboolean mdExporterSetDNSRROnly( 312 mdFlowExporter_t *exporter, 313 int mode); 314 315 gboolean mdExporterAddMySQLInfo( 316 mdFlowExporter_t *exporter, 317 char *user, 318 char *password, 319 char *db_name, 320 char *db_host, 321 char *table); 322 323 void mdInterruptFlowSource( 324 mdConfig_t *md); 325 326 327 int mdExporterWriteFlow( 328 mdConfig_t *cfg, 329 mdFlowExporter_t *exporter, 330 mdFullFlow_t *flow, 331 GError **err); 332 333 gboolean mdExporterWriteOptions( 334 mdConfig_t *cfg, 335 mdFlowExporter_t *exporter, 336 uint8_t *rec, 337 size_t rec_length, 338 uint16_t tid, 339 GError **err); 340 341 gboolean mdExporterWriteRecord( 342 mdConfig_t *cfg, 343 mdFlowExporter_t *exporter, 344 uint16_t tid, 345 uint8_t *rec, 346 size_t rec_length, 347 GError **err); 348 349 350 gboolean mdExporterWriteDNSRRRecord( 351 mdConfig_t *cfg, 352 mdFlowExporter_t *exporter, 353 uint16_t tid, 354 uint8_t *rec, 355 size_t rec_length, 356 GError **err); 357 358 gboolean mdCollectorWait( 359 mdContext_t *ctx, 360 GError **err); 361 362 gboolean mdCollectorRestartListener( 363 mdConfig_t *md, 364 md_collect_node_t *collector, 365 GError **err); 366 367 gboolean mdCollectorStartListeners( 368 mdConfig_t *md, 369 md_collect_node_t *collector, 370 GError **err); 371 372 gboolean mdExportersInit( 373 mdConfig_t *cfg, 374 md_export_node_t *node, 375 GError **err); 376 377 gboolean mdExporterRestart( 378 mdConfig_t *cfg, 379 mdFlowExporter_t *exp, 380 GError **err); 381 382 void mdExporterUpdateStats( 383 mdConfig_t *cfg, 384 gboolean dedup); 385 386 gboolean mdExporterDestroy( 387 mdConfig_t *cfg, 388 GError **err); 389 390 void mdCollectorDestroy( 391 mdConfig_t *cfg, 392 gboolean active); 393 394 395 int mdExporterDPIFlowPrint( 396 mdFlowExporter_t *exporter, 397 mdFullFlow_t *flow, 398 char *index_str, 399 size_t index_len, 400 GError **err); 401 402 gboolean mdExporterTextDNSPrint( 403 mdFlowExporter_t *exporter, 404 yaf_dnsQR_t *dns); 405 406 GString *mdExporterJsonDNSPrint( 407 mdFlowExporter_t *exporter, 408 yaf_dnsQR_t *dnsqrflow); 409 410 gboolean mdExporterDPIGetIndexStr( 411 mdFlowExporter_t *exporter, 412 mdFullFlow_t *flow); 413 414 fBuf_t *mdCollectorSpread( 415 mdConfig_t *md, 416 GError **err); 417 418 gboolean mdExporterConnectionReset( 419 mdConfig_t *md_config, 420 GError **err); 421 422 gboolean mdExportDNSRR( 423 mdConfig_t *cfg, 424 mdFlowExporter_t *exporter, 425 mdFullFlow_t *flow, 426 uint16_t tid, 427 GError **err); 428 429 void mdExporterSetMetadataExport( 430 mdFlowExporter_t *exporter); 431 432 /** print functions */ 433 int mdCustomFlowPrint( 434 mdFieldList_t *list, 435 mdFullFlow_t *fflow, 436 mdFlowExporter_t *exporter, 437 GError **err); 438 439 md_collect_node_t *mdCollectorGetNode( 440 fBuf_t *fbuf); 441 442 void mdCollectorUpdateStats( 443 mdConfig_t *cfg); 444 445 /* various types of printing functions for basic lists, varfields */ 446 447 gboolean mdExportBLMultiFiles( 448 mdFlowExporter_t *exporter, 449 fbBasicList_t *bl, 450 char *index_str, 451 size_t index_len, 452 char *label, 453 gboolean hex); 454 gboolean mdExportBL( 455 mdFlowExporter_t *exporter, 456 fbBasicList_t *bl, 457 char *index_str, 458 size_t index_len, 459 char *label, 460 gboolean hex); 461 462 gboolean mdExportBLCustomList( 463 mdFlowExporter_t *exporter, 464 fbBasicList_t *bl, 465 char *index_str, 466 size_t index_len, 467 char *label, 468 gboolean hex); 469 470 gboolean mdJsonizeBLElement( 471 mdFlowExporter_t *exporter, 472 fbBasicList_t *bl, 473 char *index_str, 474 size_t index_len, 475 char *label, 476 gboolean hex); 477 478 gboolean mdJsonizeVLElement( 479 mdFlowExporter_t *exporter, 480 uint8_t *buf, 481 char *label, 482 char *index_str, 483 size_t index_len, 484 uint16_t id, 485 size_t buflen, 486 gboolean hex); 487 488 gboolean mdAppendDPIStr( 489 mdFlowExporter_t *exporter, 490 uint8_t *buf, 491 char *label, 492 char *index_str, 493 size_t index_len, 494 uint16_t id, 495 size_t buflen, 496 gboolean hex); 497 498 gboolean mdAppendDPIStrMultiFiles( 499 mdFlowExporter_t *exporter, 500 uint8_t *buf, 501 char *label, 502 char *index_str, 503 size_t index_len, 504 uint16_t id, 505 size_t buflen, 506 gboolean hex); 507 508 gboolean mdJsonifyNewSSLRecord( 509 mdFlowExporter_t *exporter, 510 yaf_newssl_t *sslflow, 511 gboolean hex, 512 gboolean escape); 513 514 gboolean mdExporterTextNewSSLPrint( 515 mdFlowExporter_t *exporter, 516 yaf_newssl_t *sslflow, 517 char *index_str, 518 size_t index_len); 519 520 gboolean mdExporterDedupFileOpen( 521 mdConfig_t *cfg, 522 mdFlowExporter_t *exporter, 523 FILE **file, 524 char **last_file, 525 char *prefix, 526 uint64_t *rotate); 527 528 529 void mdExporterDedupFileClose( 530 mdFlowExporter_t *exporter, 531 FILE *fp, 532 char *last_file); 533 534 gboolean mdExporterSSLCertRecord( 535 mdConfig_t *cfg, 536 mdFlowExporter_t *exporter, 537 FILE *cert_file, 538 yaf_newssl_cert_t *ssl, 539 yfSSLFullCert_t *fullcert, 540 uint8_t *issuer, 541 size_t issuer_len, 542 uint8_t cert_no, 543 GError **err); 544 545 gboolean mdExporterWriteSSLDedupRecord( 546 mdConfig_t *cfg, 547 mdFlowExporter_t *exporter, 548 uint16_t tid, 549 uint8_t *rec, 550 size_t rec_length, 551 GError **err); 552 553 gboolean mdExporterWriteDedupRecord( 554 mdConfig_t *cfg, 555 md_export_node_t *enode, 556 FILE *fp, 557 md_dedup_t *rec, 558 char *prefix, 559 uint16_t int_tid, 560 uint16_t ext_tid, 561 GError **err); 562 563 gboolean mdExporterSetSSLSHA1Hash( 564 mdFlowExporter_t *exporter); 565 566 gboolean mdExporterSetSSLMD5Hash( 567 mdFlowExporter_t *exporter); 568 569 char *mdExporterGetName( 570 mdFlowExporter_t *exporter); 571 572 void mdExporterSetMovePath( 573 mdFlowExporter_t *exporter, 574 char *path); 575 576 void mdExporterSetNoFlow( 577 mdFlowExporter_t *exporter); 578 579 gboolean mdExporterDedupOnly( 580 mdFlowExporter_t *exporter); 581 582 gboolean mdExporterGetJson( 583 mdFlowExporter_t *exporter); 584