1 /* 2 ** yaftab.h 3 ** YAF Active Flow Table 4 ** 5 ** ------------------------------------------------------------------------ 6 ** Copyright (C) 2006-2015 Carnegie Mellon University. All Rights Reserved. 7 ** ------------------------------------------------------------------------ 8 ** Authors: Brian Trammell 9 ** ------------------------------------------------------------------------ 10 ** @OPENSOURCE_HEADER_START@ 11 ** Use of the YAF system and related source code is subject to the terms 12 ** of the following licenses: 13 ** 14 ** GNU Public License (GPL) Rights pursuant to Version 2, June 1991 15 ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013 16 ** 17 ** NO WARRANTY 18 ** 19 ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER 20 ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY 21 ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN 22 ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY 23 ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT 24 ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE, 25 ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE 26 ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT, 27 ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY 28 ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF 29 ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES. 30 ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF 31 ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON 32 ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE 33 ** DELIVERABLES UNDER THIS LICENSE. 34 ** 35 ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie 36 ** Mellon University, its trustees, officers, employees, and agents from 37 ** all claims or demands made against them (and any related losses, 38 ** expenses, or attorney's fees) arising out of, or relating to Licensee's 39 ** and/or its sub licensees' negligent use or willful misuse of or 40 ** negligent conduct or willful misconduct regarding the Software, 41 ** facilities, or other rights or assistance granted by Carnegie Mellon 42 ** University under this License, including, but not limited to, any 43 ** claims of product liability, personal injury, death, damage to 44 ** property, or violation of any laws or regulations. 45 ** 46 ** Carnegie Mellon University Software Engineering Institute authored 47 ** documents are sponsored by the U.S. Department of Defense under 48 ** Contract FA8721-05-C-0003. Carnegie Mellon University retains 49 ** copyrights in all material produced under this contract. The U.S. 50 ** Government retains a non-exclusive, royalty-free license to publish or 51 ** reproduce these documents, or allow others to do so, for U.S. 52 ** Government purposes only pursuant to the copyright license under the 53 ** contract clause at 252.227.7013. 54 ** 55 ** @OPENSOURCE_HEADER_END@ 56 ** ------------------------------------------------------------------------ 57 */ 58 59 /* 60 * This is the documentation for the _old_ yaftab.h; it is no longer current, 61 * and should not be read by anyone. 62 * 63 * Flow generation interface for YAF. This facility works by maintaining a 64 * current flow table. Packets may be added to the active flows within this 65 * table using the yfFlowPkt() call. Completed flows may be written to an 66 * IPFIX message buffer using yfFlowFlush(). 67 * 68 * The flow table is configured by a number of global variables. 69 * 70 * <tt>yaf_idle</tt> sets 71 * the idle timeout in seconds. A flow that receives no packets for the idle 72 * timeout is assumed to be complete. The idle timeout is set to 300 seconds 73 * (five minutes) by default. 74 * 75 * <tt>yaf_active</tt> sets the active timeout in seconds. 76 * The maximum duration of a flow is the active timeout; additional packets 77 * for the same flow will be counted as part of a new flow. The active timeout 78 * is set to 1800 seconds (half an hour) by default. 79 * 80 * <tt>yaf_flowlim</tt> sets the maximum size of the flow table; flows exceeding 81 * this limit will be expired in least-recent order, as if they were idle. The 82 * flow limit defaults to zero, for no limit. Use this global to limit resource 83 * usage by the flow table. 84 * 85 * <tt>yaf_paylen</tt> sets the number of bytes of payload to capture from the 86 * start of each flow. The payload length defaults to zero, which disables 87 * payload capture. 88 * 89 * <tt>yaf_uniflow</tt>, if TRUE, exports flows in uniflow mode, using the 90 * record adjacency export method described in section 3 of 91 * draft-ietf-ipfix-biflow. Defaults to FALSE. 92 * 93 * <tt>yaf_macmode</tt>, if TRUE, exports layer 2 information with each flow; 94 * presently this is limited to VLAN tags but may be expanded to include the 95 * MPLS stack and MAC addresses in the future. Defaults to FALSE. 96 * 97 * <tt>yaf_silkmode</tt>, if TRUE, enables SiLK compatibility mode. In this 98 * mode, totalOctetCount and reverseTotalOctetCount are clamped to 32 bits. 99 * Any packet that would cause either of these counters to overflow 32 bits 100 * will force an active timeout. The high-order bit of the flowEndReason IE 101 * is set on any flow created on a counter overflow, as above, or on an active 102 * timeout. Defaults to FALSE. 103 * 104 * <tt>yaf_reqtype</tt> limits the flow table to collecting IPv4 or IPv6 flows 105 * only. Set to YF_TYPE_IPv4 for IPv4 flows only, YF_TYPE_IPv6 for IPv6 flows 106 * only, or YF_TYPE_IPANY (the default) to collect both IPv4 and IPv6 flows. 107 * 108 * This facility is used by YAF to assemble packets into flows. 109 */ 110 111 /** 112 * @file 113 * 114 * Flow generation interface for YAF. [TODO - frontmatter] 115 * 116 * This facility is used by YAF to assemble packets into flows. 117 */ 118 119 #ifndef _YAF_TAB_H_ 120 #define _YAF_TAB_H_ 121 122 #include <yaf/autoinc.h> 123 #include <yaf/yafcore.h> 124 #include <yaf/decode.h> 125 126 127 struct yfFlowTab_st; 128 /** 129 * A flow table. Opaque. Create with yfFlowTabAlloc() and free with 130 * yfFlowTabFree(). 131 */ 132 typedef struct yfFlowTab_st yfFlowTab_t; 133 134 /** 135 *yfFlowTabAlloc 136 * 137 * Allocate a flow table. 138 * 139 * @param idle_ms idle timeout in milliseconds. A flow that receives no 140 * packets for the idle timeout is assumed to be complete. 141 * @param active_ms active timeout in milliseconds. The maximum duration of a 142 * flow is the active timeout; additional packets 143 * for the same flow will be counted as part of a new flow. 144 * @param max_flows maximum number of active flows. Flows exceeding this limit 145 * will be expired in least-recent order, as if they were idle. 146 * Used to limit resource usage of a flow table. A value of 0 147 * disables flow count limits. 148 * @param max_payload maximum octets of payload to capture per flow direction. 149 * Requires at least max_payload octets of payload to be 150 * available in each packet buffer passed to yfFlowPBuf(). 151 * A value of 0 disables payload capture and export. 152 * @param uniflow If TRUE, export biflows using record adjacency (two uniflows 153 * exported back-to-back. Use this for interoperability with 154 * IPFIX collectors that do not implement RFC 5103. 155 * @param silkmode If TRUE, clamp totalOctetCount and maxTotalOctetCount to 32 156 * bits and force active timeout on overflow. Set high order 157 * bit in flowEndReason for each flow created on an overflow 158 * or active timeout. Breaks IPFIX interoperability; use for 159 * direct export to SiLK rwflowpack or flowcap. 160 * 161 * @param macmode If TRUE, collect and export source and destination Mac 162 Addresses. 163 * @param applabelmode If TRUE, then the payload, (as limited by max_payload,) 164 * is sent through various plugins and code in order to 165 * determine which protocol is running on the flow by doing 166 * only payload inspection and exporting payload relevent 167 * information. 168 * 169 * @param entropymode If TRUE, then a Shannon Entropy measurement is made over 170 * the captured payload (as limited by max_payload). The 171 * entropy value is exported as two values one for forward 172 * payload and one for reverse payload. 173 * 174 * @param fingerprintmode If TRUE, then this will enable passive OS finger printing 175 * using the p0f engine based mostly on TCP negotiation 176 * 177 * 178 * @param fpExportMode If TRUE, then this will enable exporting of full 179 * packet banners of the TCP negotiations for the first 180 * three packets (including IP 181 * and transport headers) for external fingerprinting 182 * 183 * @param udp_max_payload If TRUE, then this will enable capturing up to 184 * max_payload value for udp flows 185 * (instead of just the first packet) 186 * 187 * @param udp_uniflow_port If not 0, then this will enable exporting a single 188 * UDP packet with this src/dst port as a flow. 189 * 190 * @param pcap_dir Directory to put pcap-per-flow files 191 * 192 * @param pcap_meta_file File for pcap meta output. Default is stdout 193 * 194 * @param max_pcap Maximum size [in bytes] of a pcap file before rotating. 195 * 196 * @param pcap_per_flow If TRUE, then pcap_dir will be set to the directory 197 * to place pcap-per-flow files. 198 * @param force_read_all If TRUE, then yaf will process files that are out of 199 * sequence. 200 * @param stats_mode If TRUE, then YAF will do some extra calculations 201 * on flows. 202 * @param index_pcap If TRUE, print one line per packet we export. This 203 * will give offset and length into the pcap yaf writes. 204 * @param no_vlan_in_key If TRUE, this will remove the vlan in the calculation 205 * of the flow key hash. 206 * @param ndpi If TRUE, enable nDPI application labeling with 207 * standard protocols. 208 * @param ndpi_proto_file If not NULL, and ndpi is TRUE, use the provided 209 * protocol file to expand the sub-protocols list 210 * and port-based detection methods. 211 * @param hash The flow key hash to create a PCAP for. 212 * @param stime The start time to create a PCAP for. 213 * @param hfctx The plugin hooks context variable (NULL if plugins not enabled) 214 * 215 * @return a new flow table. 216 */ 217 yfFlowTab_t *yfFlowTabAlloc( 218 uint64_t idle_ms, 219 uint64_t active_ms, 220 uint32_t max_flows, 221 uint32_t max_payload, 222 gboolean uniflow, 223 gboolean silkmode, 224 gboolean macmode, 225 gboolean applabelmode, 226 gboolean entropymode, 227 gboolean fingerprintmode, 228 gboolean fpExportMode, 229 gboolean udp_max_payload, 230 uint16_t udp_uniflow_port, 231 char *pcap_dir, 232 char *pcap_meta_file, 233 uint64_t max_pcap, 234 gboolean pcap_per_flow, 235 gboolean force_read_all, 236 gboolean stats_mode, 237 gboolean index_pcap, 238 gboolean no_vlan_in_key, 239 gboolean ndpi, 240 char *ndpi_proto_file, 241 char *hash, 242 char *stime, 243 void **hfctx); 244 245 /** 246 * Free a previously allocated flow table. Discards any outstanding active 247 * flows without closing or flushing them; use yfFlowTabFlushAll() before 248 * yfFlowFree() to do this. 249 * 250 * @param flowtab a flow table allocated by yfFlowTabAlloc() 251 */ 252 253 void yfFlowTabFree( 254 yfFlowTab_t *flowtab); 255 256 257 /** 258 * Update the Pcap Filename in the Flowtab for pcap meta data output 259 * 260 * @param flowtab pointer to flow table 261 * @param new_file_name the filename of the next pcap file to write to 262 */ 263 264 void yfUpdateRollingPcapFile( 265 yfFlowTab_t *flowtab, 266 char *new_file_name); 267 268 /** 269 * yfGetFlowTabStats 270 * Get Flow Table Stats for Export 271 * 272 * @param flowtab 273 * @param packets number of packets processed 274 * @param flows number of flows created 275 * @param rej_pkts number of packets rejected due to out of sequence 276 * @param peak maximum number of flows in the flow table at any 1 time 277 * @param flush number of flush events called on flow table 278 */ 279 void yfGetFlowTabStats( 280 yfFlowTab_t *flowtab, 281 uint64_t *packets, 282 uint64_t *flows, 283 uint64_t *rej_pkts, 284 uint32_t *peak, 285 uint32_t *flush); 286 287 /** 288 * Add a decoded packet buffer to a given flow table. Adds the packet to 289 * the flow to which it belongs, creating a new flow if necessary. Causes 290 * the flow to which it belongs to time out if it is longer than the active 291 * timeout. Closes the flow if the flow closure conditions (TCP RST, TCP FIN 292 * four-way teardown) are met. 293 * 294 * @param flowtab flow table to add the packet to 295 * @param pbuflen size of the packet buffer pbuf 296 * @param pbuf packet buffer containing decoded packet to add. 297 */ 298 299 void yfFlowPBuf( 300 yfFlowTab_t *flowtab, 301 size_t pbuflen, 302 yfPBuf_t *pbuf); 303 304 /** 305 * Flush closed flows in the given flow table to the given IPFIX Message 306 * Buffer. Causes any idle flows to time out, removing them from the active 307 * flow table; also enforces the flow table's resource limit. If close is 308 * TRUE, additionally closes all active flows and flushes as well. 309 * 310 * @param yfContext YAF thread context structure, holds pointers for the 311 * flowtable from which to flush flows and the fbuf, the 312 * destination to which the flows should be flushed 313 * @param close close all active flows before flushing 314 * @param err An error description pointer; must not be NULL. 315 * @return TRUE on success, FALSE otherwise. 316 */ 317 318 gboolean yfFlowTabFlush( 319 void *yfContext, 320 gboolean close, 321 GError **err); 322 323 /** 324 * Get the current packet clock from a flow table. 325 * 326 * @param flowtab a flow table 327 * @return current packet clock 328 */ 329 330 uint64_t yfFlowTabCurrentTime( 331 yfFlowTab_t *flowtab); 332 333 /** 334 * Print flow table statistics to the log. 335 * 336 * @param flowtab flow table to dump stats for 337 * @param timer a GTimer containing the runtime 338 * (for packet and flow rate logging). May be NULL to suppress 339 * rate logging. 340 */ 341 342 uint64_t yfFlowDumpStats( 343 yfFlowTab_t *flowtab, 344 GTimer *timer); 345 346 #endif 347