1 #ifndef NMSG_H 2 #define NMSG_H 3 4 /* 5 * Copyright (c) 2008-2015 by Farsight Security, Inc. 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 20 /*! \file nmsg.h 21 * \brief Base nmsg support header. 22 * 23 * This header ensures that needed constants, functions, result codes, 24 * vendor definitions, and opaque pointer types are defined. 25 */ 26 27 #ifdef __cplusplus 28 extern "C" { 29 #endif 30 31 #include <netinet/in.h> 32 #include <arpa/inet.h> 33 #include <sys/socket.h> 34 #include <sys/types.h> 35 #include <stdarg.h> 36 #include <stdbool.h> 37 #include <stddef.h> 38 #include <stdint.h> 39 #include <time.h> 40 41 #include <pcap.h> 42 43 #include <nmsg/res.h> 44 typedef enum nmsg_res nmsg_res; 45 46 typedef struct nmsg_container * nmsg_container_t; 47 typedef struct nmsg_fltmod * nmsg_fltmod_t; 48 typedef struct nmsg_input * nmsg_input_t; 49 typedef struct nmsg_io * nmsg_io_t; 50 typedef struct nmsg_message * nmsg_message_t; 51 typedef struct nmsg_msgmod * nmsg_msgmod_t; 52 typedef struct nmsg_output * nmsg_output_t; 53 typedef struct nmsg_pcap * nmsg_pcap_t; 54 typedef struct nmsg_pres * nmsg_pres_t; 55 typedef struct nmsg_rate * nmsg_rate_t; 56 typedef struct nmsg_random * nmsg_random_t; 57 typedef struct nmsg_strbuf * nmsg_strbuf_t; 58 typedef struct nmsg_zbuf * nmsg_zbuf_t; 59 60 /** 61 * Generic ID to name map. 62 */ 63 struct nmsg_idname { 64 unsigned id; /*%< ID number */ 65 const char *name; /*%< Human readable name */ 66 }; 67 68 /** 69 * Callback function for processing nmsg messages. 70 * 71 * \param[in] msg Valid nmsg message. 72 * 73 * \param[in] user User-provided pointer. 74 * 75 * \see nmsg_input_loop() 76 * \see nmsg_output_open_callback() 77 */ 78 typedef void (*nmsg_cb_message)(nmsg_message_t msg, void *user); 79 80 /** 81 * Callback function for generating nmsg messages. 82 * 83 * \param[out] msg Pointer to where an nmsg_message_t object may be stored. 84 * 85 * \param[in] user User-provided pointer. 86 * 87 * \see nmsg_input_open_callback() 88 * 89 * \return #nmsg_res_success 90 * \return #nmsg_res_failure 91 * \return #nmsg_res_again 92 * \return #nmsg_res_eof 93 */ 94 typedef nmsg_res (*nmsg_cb_message_read)(nmsg_message_t *msg, void *user); 95 96 #include <nmsg/alias.h> 97 #include <nmsg/asprintf.h> 98 #include <nmsg/chalias.h> 99 #include <nmsg/compat.h> 100 #include <nmsg/constants.h> 101 #include <nmsg/container.h> 102 #include <nmsg/filter.h> 103 #include <nmsg/fltmod.h> 104 #include <nmsg/input.h> 105 #include <nmsg/io.h> 106 #include <nmsg/ipdg.h> 107 #include <nmsg/message.h> 108 #include <nmsg/msgmod.h> 109 #include <nmsg/output.h> 110 #include <nmsg/pcap_input.h> 111 #include <nmsg/random.h> 112 #include <nmsg/rate.h> 113 #include <nmsg/sock.h> 114 #include <nmsg/strbuf.h> 115 #include <nmsg/timespec.h> 116 #include <nmsg/vendors.h> 117 #include <nmsg/version.h> 118 #include <nmsg/zbuf.h> 119 120 /** 121 * Initialize the libnmsg library. This function MUST be called before 122 * using any other libnmsg function. The caller MUST also check that this 123 * function returned #nmsg_res_success before using any other libnmsg function. 124 * The library should only be initialized once. 125 * 126 * \return nmsg_res_success On successful library initialization. 127 * \return nmsg_res_failure If libnmsg is not usable, or if #nmsg_init() was 128 * called more than once. 129 */ 130 #if __GNUC__ >= 4 131 __attribute__ ((warn_unused_result)) 132 #endif 133 nmsg_res nmsg_init(void); 134 135 /** 136 * Configure automatic close() behavior of nmsg inputs and outputs. The 137 * default behavior is for nmsg_input_close(), nmsg_output_close(), and 138 * nmsg_io_destroy() to automatically close the underlying file descriptors 139 * for libnmsg inputs and outputs. 140 * 141 * \param autoclose False to disable automatic close() behavior, true to 142 * re-enable. 143 */ 144 void nmsg_set_autoclose(bool autoclose); 145 146 /** 147 * Set debug level. If the debug level is greater than 0, some libnmsg 148 * functions will emit debugging information to stderr. Higher values increase 149 * verbosity. 150 */ 151 void nmsg_set_debug(int debug); 152 153 /** 154 * Retrieve the current debug level. 155 */ 156 int nmsg_get_debug(void); 157 158 /** 159 * Retrieve the semantic library version as a string. 160 */ 161 const char *nmsg_get_version(void); 162 163 /** 164 * Retrieve the semantic library version as a packed integer. The number is a 165 * combination of the major, minor, and patchelevel numbers as per: 166 * MAJOR * 1000000 + MINOR * 1000 + PATCHLEVEL. 167 */ 168 uint32_t nmsg_get_version_number(void); 169 #ifdef __cplusplus 170 } 171 #endif 172 173 /** 174 \mainpage nmsg documentation 175 176 \section intro Introduction 177 178 The NMSG format is an efficient encoding of typed, structured data into 179 payloads which are packed into containers which can be transmitted over the 180 network or stored to disk. <tt>libnmsg</tt> is the reference implementation of 181 this format and provides an extensible interface for creating and parsing 182 messages in NMSG format. The NMSG format relies on Google <a 183 href="http://code.google.com/p/protobuf/">Protocol Buffers</a> to encode the 184 payload header. Individual NMSG payloads are distinguished by assigned vendor 185 ID and message type values and <tt>libnmsg</tt> provides a modular interface 186 for registering handlers for specific message types. <tt>libnmsg</tt> makes it 187 easy to build new message types using a <a 188 href="http://code.google.com/p/protobuf-c/">Protocol Buffers</a> compiler. 189 190 \see http://code.google.com/p/protobuf/ 191 \see http://code.google.com/p/protobuf-c/ 192 193 <hr> 194 195 \section libnmsg 196 197 <tt>libnmsg</tt> provides a reference C implementation of an NMSG parser and 198 generator. It contains core functions for reading and writing NMSG units and 199 can be extended at runtime by plugin modules implementing new message types. 200 201 \subsection core Core I/O functions 202 <div class="subsection"> 203 204 input.h and output.h provide the single-threaded input and output interfaces. 205 io.h provides a multi-threaded interface for multiplexing data between inputs 206 and outputs. message.h provides an interface for creating and inspecting 207 message payloads. 208 209 </div> 210 211 \subsection nmsg_msg nmsg_msg message module interface 212 <div class="subsection"> 213 214 The <b>nmsg_msg</b> message module interface is implemented by msgmod.h. Plugins 215 in external shared objects provide an <tt>nmsg_msgmod_t</tt> structure in order 216 to implement new message types. 217 218 </div> 219 220 \subsection aux Auxiliary functions 221 <div class="subsection"> 222 223 <ul> 224 <li>alias.h 225 <li>asprintf.h 226 <li>chalias.h 227 <li>ipdg.h 228 <li>pcap_input.h 229 <li>rate.h 230 <li>strbuf.h 231 <li>timespec.h 232 <li>zbuf.h 233 </ul> 234 235 </div> 236 237 <hr> 238 239 \section wire_format Wire format 240 241 An NMSG unit consists of a small fixed-length header which precedes a variable 242 length part that may contain one or more payloads or, if a single payload is too 243 large, the variable length part may contain a single message fragment. 244 245 NMSG is designed for transport over UDP sockets or storage in on-disk files. 246 Individual UDP datagrams may transport only a single NMSG unit, while the file 247 format is simply a series of NMSG units concatenated together. The variable 248 length part of an NMSG unit transported over UDP is usually much smaller than 249 those stored on disk (#NMSG_WBUFSZ_MAX versus #NMSG_WBUFSZ_JUMBO or 250 #NMSG_WBUFSZ_ETHER). NMSG units stored on disk also do not contain message 251 fragments. 252 253 The fixed-length NMSG header is ten octets long and consists of a magic value, a 254 bit field of flags, a protocol version number, and the length of the variable 255 length data part. 256 257 The variable length data part is interpreted as a Protocol Buffer message. 258 259 <table> 260 261 <tr> 262 <th>Octet 0-3</th> 263 <th>Octet 4</th> 264 <th>Octet 5</th> 265 <th>Octet 6-9</th> 266 <th>Remainder</th> 267 </tr> 268 269 <tr> 270 <td><center>Magic</center></td> 271 <td><center>Flags</center></td> 272 <td><center>Protocol Version</center></td> 273 <td><center>Length</center></td> 274 <td><center>Data</center></td> 275 </tr> 276 277 </table> 278 279 \subsection magic Magic value 280 <div class="subsection"> 281 282 The magic value (#NMSG_MAGIC) is always the four octet sequence 'N', 'M', 'S', 283 'G'. 284 285 </div> 286 287 \subsection flags Flags 288 <div class="subsection"> 289 290 This is a bit field of flags. Currently two values are defined. #NMSG_FLAG_ZLIB 291 indicates that the data content has been compressed. #NMSG_FLAG_FRAGMENT 292 indicates that the data content starts a special fragmentation header. 293 294 </div> 295 296 \subsubsection zlib NMSG_FLAG_ZLIB 297 <div class="subsubsection"> 298 299 This flag indicates that zlib compression has been applied to the variable 300 length part. If the #NMSG_FLAG_FRAGMENT flag is not also set, then the entire 301 variable length part should be deflated with zlib and interpreted as an 302 <b>NmsgPayload</b>. 303 304 </div> 305 306 \subsubsection frag NMSG_FLAG_FRAGMENT 307 <div class="subsubsection"> 308 309 This flag indicates that the variable length part should be interpreted as an 310 <b>NmsgFragment</b>. After reassembly, the data should be interpreted as an 311 <b>NmsgPayload</b>. If the #NMSG_FLAG_ZLIB flag is also set, then the 312 reassembled data should be deflated and then interpreted as an 313 <b>NmsgPayload</b>. 314 315 Note that when creating a compressed, fragmented NMSG unit, compression should 316 be applied <i>before</i> fragmentation. 317 318 </div> 319 320 \subsection version Protocol Version 321 <div class="subsection"> 322 323 This value (#NMSG_PROTOCOL_VERSION) is currently 2. 324 325 </div> 326 327 \subsection length Length 328 <div class="subsection"> 329 330 This value is an unsigned 32 bit integer in network byte order indicating the 331 length in octets of the variable length data part. 332 333 </div> 334 335 \subsection data Data 336 <div class="subsection"> 337 338 The variable length data part is encoded using <a 339 href="http://code.google.com/apis/protocolbuffers/docs/encoding.html">Google 340 Protocol Buffers</a>. The file <tt>nmsg/nmsg.proto</tt> in the source 341 distribution describes the two message types <b>Nmsg</b> and 342 <b>NmsgFragment</b> that can appear in NMSG units: 343 344 \include nmsg.proto 345 346 If no flags are set, then the data part is an <b>Nmsg</b> protobuf message. If 347 only the #NMSG_FLAG_ZLIB flag is set, then the data part is a zlib compressed 348 <b>Nmsg</b> protobuf message. The <b>Nmsg</b> protobuf message is a container 349 message for one or more <b>NmsgPayload</b> messages. If only the 350 #NMSG_FLAG_FRAGMENT flag is set, then the data part is an <b>NmsgFragment</b> 351 protobuf message. 352 353 </div> 354 355 \subsubsection nmsg Nmsg and NmsgPayload protobuf messages 356 <div class="subsubsection"> 357 358 <b>Nmsg</b> messages contain one or more <b>NmsgPayload</b> messages. 359 360 The <b>vid</b> field of <b>NmsgPayload</b> messages is the vendor ID. The 361 currently defined vendor IDs are listed in vendors.h and assigned by Farsight. 362 The <b>msgtype</b> field is a vendor-specific value and together the (<b>vid</b>, 363 <b>msgtype</b>) tuple defines the type of the data contained in the 364 <b>payload</b> field. 365 366 The time that the data encapsulated in the <b>payload</b> field was generated is 367 stored in the <b>time_sec</b> and <b>time_nsec</b> fields. The number of 368 nanoseconds since the Unix epoch is split across the two fields, with the 369 integer number of seconds stored in the <b>time_sec</b> field and the 370 nanoseconds part stored in the <b>time_nsec</b> field. 371 372 The <b>source</b>, <b>operator</b>, and <b>group</b> fields are optional fields 373 that can be used by cooperating senders and receivers to classify the payload. 374 375 </div> 376 377 \subsubsection nmsgfragment NmsgFragment protobuf messages 378 <div class="subsubsection"> 379 380 <b>NmsgFragment</b> messages are used to encapsulate fragments of an <b>Nmsg</b> 381 message if the serialized message is too large for the underlying transport. 382 This enables NMSG payloads to avoid the size restrictions of UDP/IP transport. 383 384 If a serialized <b>Nmsg</b> message is too large for the underlying transport, 385 it is split into fragments which are carried in the <b>fragment</b> field of a 386 sequence of <b>NmsgFragment</b> messages. A random value is selected for the 387 <b>id</b> field of these fragments. The 0-indexed <b>current</b> field indicates 388 the ordering of the fragments, and the <b>last</b> field indicates the total 389 number of fragments. Once a receiver has received all the fragments for a given 390 fragment <b>id</b>, the fragmented payload should be extracted from the 391 <b>payload</b> field of each fragment and concatenated into a buffer. 392 393 If the sender performed compression of the <b>Nmsg</b> message before 394 fragmentation, then all fragments should have the #NMSG_FLAG_ZLIB field set and 395 the receiver must perform decompression of the reassembled buffer. The result of 396 decompression should be interpreted as an <b>Nmsg</b> message. 397 398 If the sender did not perform compression before fragmentation, then the buffer 399 should be directly interpreted as an <b>Nmsg</b> message. 400 401 </div> 402 403 */ 404 405 #endif /* NMSG_H */ 406