1 /**
2 *@internal
3 *
4 * @file fbnetflow.c
5 *
6 * This implements a Netflow convertor for translating into IPFIX
7 * within the fixbuf structure
8 *
9 * ------------------------------------------------------------------------
10 * Copyright (C) 2008-2019 Carnegie Mellon University. All Rights Reserved.
11 * ------------------------------------------------------------------------
12 * Authors: Chris Inacio <inacio@cert.org>, Emily Sarneso <ecoff@cert.org>
13 * ------------------------------------------------------------------------
14 * @OPENSOURCE_LICENSE_START@
15 * libfixbuf 2.0
16 *
17 * Copyright 2018-2019 Carnegie Mellon University. All Rights Reserved.
18 *
19 * NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE
20 * ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS"
21 * BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND,
22 * EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT
23 * LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY,
24 * EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE
25 * MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF
26 * ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, TRADEMARK, OR
27 * COPYRIGHT INFRINGEMENT.
28 *
29 * Released under a GNU-Lesser GPL 3.0-style license, please see
30 * LICENSE.txt or contact permission@sei.cmu.edu for full terms.
31 *
32 * [DISTRIBUTION STATEMENT A] This material has been approved for
33 * public release and unlimited distribution. Please see Copyright
34 * notice for non-US Government use and distribution.
35 *
36 * Carnegie Mellon(R) and CERT(R) are registered in the U.S. Patent
37 * and Trademark Office by Carnegie Mellon University.
38 *
39 * DM18-0325
40 * @OPENSOURCE_LICENSE_END@
41 * ------------------------------------------------------------------------
42 */
43
44 #define _FIXBUF_SOURCE_
45 #include <fixbuf/private.h>
46
47 #include "fbcollector.h"
48
49 #include <pthread.h>
50
51
52 #ifndef FB_NETFLOW_DEBUG
53 # define FB_NETFLOW_DEBUG 0
54 #endif
55
56 #define NF_MAX_SEQ_DIFF 100
57 #define NF_OUT_OF_ORDER 10
58 #define NF_REBOOT_SECS 60 * 1000 /* 1 min in milliseconds */
59
60
61 #if HAVE_ALIGNED_ACCESS_REQUIRED
62
63 #define fb_ntohll(x) (x)
64 #define fb_htonll(x) fb_ntohll(x)
65
66 /**
67 * this set of macros for reading and writing U16's and U32's
68 * uses memcpy's to avoid tripping over alignment issues on
69 * platforms that cannot do unaligned access (e.g. SPARC, Alpha,
70 * etc). The next section, after the else, does not use memcpy's
71 * and operates just fine on architectures that don't crash
72 * from an unaligned access (x86, PowerPC, etc.)
73 */
74
75 /**
76 * READU16INC
77 *
78 * read U16 and increment
79 *
80 * read a U16 value from ptr, it assumes the pointer
81 * is properly aligned and pointing at the correct
82 * place; and then increment ptr appropriately
83 * according to its size to adjust for reading a
84 * U16; increments it 2-bytes
85 *
86 * does network to host translation
87 */
88 #define READU16INC(ptr,assignee) { \
89 uint16_t *ru16_t16ptr = (uint16_t *)(ptr); \
90 uint16_t ru16_t16val = 0; \
91 memcpy(&ru16_t16val, ru16_t16ptr, sizeof(uint16_t)); \
92 assignee = g_ntohs(ru16_t16val); \
93 ptr += sizeof(*ru16_t16ptr) / sizeof(*ptr); \
94 }
95
96 #define READU16(ptr,assignee) { \
97 uint16_t *ru16_t16ptr = (uint16_t *)(ptr); \
98 uint16_t ru16_t16val = 0; \
99 memcpy(&ru16_t16val, ru16_t16ptr, sizeof(uint16_t)); \
100 assignee = g_ntohs(ru16_t16val); \
101 }
102
103 #define WRITEU16(ptr,value) { \
104 uint16_t *ru16_t16ptr = (uint16_t *)(ptr); \
105 uint16_t ru16_t16val = g_htons(value); \
106 memcpy(ru16_t16ptr, &ru16_t16val, sizeof(uint16_t)); \
107 }
108
109
110 /**
111 * READU32INC
112 *
113 * read U32 and increment ptr
114 *
115 * read a U32 from the ptr given in ptr, it assumes
116 * it is aligned and positioned correctly, then
117 * increment the ptr appropriately based on its
118 * size and having read a U32 to increment it
119 * 4-bytes ahead
120 *
121 * does network to host translation
122 */
123 #define READU32INC(ptr,assignee) { \
124 uint32_t *ru32_t32ptr = (uint32_t *)(ptr); \
125 uint32_t ru32_t32val = 0; \
126 memcpy(&ru32_t32val, ru32_t32ptr, sizeof(uint32_t)); \
127 assignee = g_ntohl(ru32_t32val); \
128 ptr += sizeof(*ru32_t32ptr) / sizeof(*ptr); \
129 }
130
131
132 #define READU32(ptr,assignee) { \
133 uint32_t *ru32_t32ptr = (uint32_t *)(ptr); \
134 uint32_t ru32_t32val = 0; \
135 memcpy(&ru32_t32val, ru32_t32ptr, sizeof(uint32_t)); \
136 assignee = g_ntohl(ru32_t32val); \
137 }
138
139 #define WRITEU32(ptr,value) { \
140 uint32_t *ru32_t32ptr = (uint32_t *)(ptr); \
141 uint32_t ru32_t32val = g_htonl(value); \
142 memcpy(ru32_t32ptr, &ru32_t32val, sizeof(uint32_t)); \
143 }
144
145 #else
146
147
148 #define fb_ntohll(x) \
149 ((((uint64_t)g_ntohl((uint32_t)((x) & 0xffffffff))) << 32) \
150 | g_ntohl((uint32_t)(((x) >> 32) & 0xffffffff)))
151 #define fb_htonll(x) fb_ntohll(x)
152
153
154 /**
155 * READU16INC
156 *
157 * read U16 and increment
158 *
159 * read a U16 value from ptr, it assumes the pointer
160 * is properly aligned and pointing at the correct
161 * place; and then increment ptr appropriately
162 * according to its size to adjust for reading a
163 * U16; increments it 2-bytes
164 *
165 * does network to host translation
166 */
167 #define READU16INC(ptr,assignee) { \
168 uint16_t *ru16_t16ptr = (uint16_t *)(ptr); \
169 uint16_t ru16_t16val = 0; \
170 ru16_t16val = g_ntohs(*ru16_t16ptr); \
171 assignee = ru16_t16val; \
172 ptr += sizeof(*ru16_t16ptr) / sizeof(*ptr); \
173 }
174
175 #define READU16(ptr,assignee) { \
176 uint16_t *ru16_t16ptr = (uint16_t *)(ptr); \
177 uint16_t ru16_t16val = 0; \
178 ru16_t16val = g_ntohs(*ru16_t16ptr); \
179 assignee = ru16_t16val; \
180 }
181
182 #define WRITEU16(ptr,value) { \
183 uint16_t *ru16_t16ptr = (uint16_t *)(ptr); \
184 *ru16_t16ptr = g_htons(value); \
185 }
186
187
188 /**
189 * READU32INC
190 *
191 * read U32 and increment ptr
192 *
193 * read a U32 from the ptr given in ptr, it assumes
194 * it is aligned and positioned correctly, then
195 * increment the ptr appropriately based on its
196 * size and having read a U32 to increment it
197 * 4-bytes ahead
198 *
199 * does network to host translation
200 */
201 #define READU32INC(ptr,assignee) { \
202 uint32_t *ru32_t32ptr = (uint32_t *)(ptr); \
203 uint32_t ru32_t32val = 0; \
204 ru32_t32val = g_ntohl(*ru32_t32ptr); \
205 assignee = ru32_t32val; \
206 ptr += sizeof(*ru32_t32ptr) / sizeof(*ptr); \
207 }
208
209
210 #define READU32(ptr,assignee) { \
211 uint32_t *ru32_t32ptr = (uint32_t *)(ptr); \
212 uint32_t ru32_t32val = 0; \
213 ru32_t32val = g_ntohl(*ru32_t32ptr); \
214 assignee = ru32_t32val; \
215 }
216
217 #define WRITEU32(ptr,value) { \
218 uint32_t *ru32_t32ptr = (uint32_t *)(ptr); \
219 *ru32_t32ptr = g_htonl(value); \
220 }
221
222 #endif
223
224
225 /** mini hash table for Netflow V9 */
226 typedef struct fbCollectorNetflowV9TemplateHash_st {
227 /** id of the stored template, should be zeroed if not in use */
228 uint16_t templateId;
229 /** length of the template in question, zero is reserved for an
230 unused field */
231 uint16_t templateLength;
232 /** boolean flag set if template ID represents an options template */
233 gboolean optionsTemplate;
234 /** boolean flag set if we added sysuptime field to template */
235 gboolean addSysUpTime;
236 } fbCollectorNetflowV9TemplateHash_t;
237
238 typedef struct fbCollectorNetflowV9Session_st {
239 /** template hash */
240 GHashTable *templateHash;
241 /** potential missed packets */
242 uint32_t netflowMissed;
243 /** current netflow seq num */
244 uint32_t netflowSeqNum;
245 /** current ipfix seq num */
246 uint32_t ipfixSeqNum;
247 } fbCollectorNetflowV9Session_t;
248
249 /** defines the extra state needed to convert from NetflowV9 to IPFIX */
250 struct fbCollectorNetflowV9State_st {
251 uint64_t sysUpTime;
252 uint32_t observation_id;
253 fbSession_t *sessionptr;
254 fbCollectorNetflowV9Session_t *session;
255 /* need to keep templates per domain */
256 GHashTable *domainHash;
257 pthread_mutex_t ts_lock;
258 };
259
260 /**
261 * templateHashDestroyHelper
262 *
263 * helps destroy the template hash by translating between the
264 * GLib GDestroyNotify function type definition and using the
265 * GLib slice free function
266 *
267 * @param datum pointer to the structure to be destroyed
268 *
269 */
templateHashDestroyHelper(gpointer datum)270 static void templateHashDestroyHelper(
271 gpointer datum)
272 {
273 g_slice_free(fbCollectorNetflowV9TemplateHash_t, datum);
274 }
275
domainHashDestroyHelper(gpointer datum)276 static void domainHashDestroyHelper(
277 gpointer datum)
278 {
279 if (((fbCollectorNetflowV9Session_t *)datum)->templateHash) {
280 g_hash_table_destroy(((fbCollectorNetflowV9Session_t *)datum)->templateHash);
281 }
282 g_slice_free(fbCollectorNetflowV9Session_t, datum);
283 }
284
285
286
287 /*#################################################
288 *
289 * netflow v9 functions for the collector, used
290 * to optionally read
291 *
292 *#################################################*/
293
294 /**
295 * fbCollectorDecodeV9MsgVL
296 *
297 * parses the header of a V9 message and determines
298 * how much needs to be read in order to complete
299 * the message, (at least in theory)
300 *
301 * @param collector a pointer to the collector state
302 * structure
303 * @param hdr a pointer to the beginning of the buffer
304 * to parse as a message (get converted back
305 * into a uint8_t* and used as such)
306 * @param b_len length of the buffer passed in for the
307 * hdr
308 * @param m_len length of the message header that still
309 * needs to be read (always set to zero, since
310 * this reads the entire message)
311 * @param err a pointer to glib error structure, used
312 * if an error occurs during processing the
313 * stream
314 *
315 *
316 * @return number of octets to read to complete
317 * the message
318 */
fbCollectorDecodeV9MsgVL(fbCollector_t * collector,fbCollectorMsgVL_t * hdr,size_t b_len,uint16_t * m_len,GError ** err)319 static gboolean fbCollectorDecodeV9MsgVL(
320 fbCollector_t *collector,
321 fbCollectorMsgVL_t *hdr,
322 size_t b_len,
323 uint16_t *m_len,
324 GError **err)
325 {
326 uint16_t recordCount;
327 uint8_t *dataBuf;
328 uint8_t *bufOffset;
329 uint64_t unix_secs;
330 uint64_t sysuptime;
331 int rc;
332 unsigned int loop;
333 uint16_t setLength;
334 struct fbCollectorNetflowV9State_st *transState =
335 (struct fbCollectorNetflowV9State_st *)collector->translatorState;
336 struct setHeader_st {
337 uint16_t setId;
338 uint16_t setLength;
339 } *setHeaderPtr;
340
341
342 if (0x0009 != g_ntohs(hdr->n_version)) {
343 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
344 "Illegal NetflowV9 Message version 0x%04x; "
345 "input is probably not a NetflowV9 Message stream.",
346 g_ntohs(hdr->n_version));
347 *m_len = 0;
348 return FALSE;
349 }
350
351 /* so this is hopefully a netflow message; I should now be able to
352 read the entire message, it can't be larger than 1 packet, (the
353 question becomes, what's a packet size) */
354
355 recordCount = htons(hdr->n_len);
356 dataBuf = (uint8_t *)hdr;
357 bufOffset = dataBuf + sizeof(hdr);
358
359 /* read the rest of the v9 header, ugly trick ahead:
360 gonna read in the uptime, and then throw away the read
361 and finish reading the rest of the header, dumping the
362 uptime, because we don't really want it */
363 if ((unsigned int)((bufOffset-dataBuf) + 16) < b_len) {
364 g_set_error(err,FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
365 "Error buffer too small to read NetflowV9 message header");
366 *m_len = 0;
367 return FALSE;
368 }
369 if (TRUE == collector->bufferedStream) {
370 rc = fread(bufOffset, 1, 4, collector->stream.fp);
371 } else {
372 rc = read(collector->stream.fd, bufOffset, 4);
373 }
374
375 READU32(bufOffset, sysuptime);
376
377 if (4 != rc) {
378 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
379 "Could not complete read of the Netflow header");
380 *m_len = 0;
381 return FALSE;
382 }
383
384 if (TRUE == collector->bufferedStream) {
385 rc = fread(bufOffset, 1, 12, collector->stream.fp);
386 } else {
387 rc = read(collector->stream.fd, bufOffset, 12);
388 }
389
390 if (12 != rc) {
391 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
392 "Could not complete read of the Netflow header");
393 *m_len = 0;
394 return FALSE;
395 }
396
397 READU32(bufOffset, unix_secs);
398
399 bufOffset += 12;
400
401 /* now calculate time to put in element 160 */
402 /* convert unix_secs to milliseconds then subtract sysuptime
403 sysuptime = (milliseconds since reboot) */
404
405 transState->sysUpTime = (unix_secs * 1000) - sysuptime;
406 transState->sysUpTime = fb_htonll(transState->sysUpTime);
407
408 /* so we don't really care about what is in the different sets,
409 at this point, we just want to scan through recordCount
410 number of them and read the length from each, and sum it,
411 then we get to rewind the file offset so that we can go
412 record-by-record back out to the application */
413
414 for (loop = 0; loop < recordCount; loop++) {
415
416 if ((unsigned int)((bufOffset-dataBuf) + 4) < b_len) {
417 g_set_error(err,FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
418 "Error buffer to small to read NetflowV9 message");
419 *m_len = 0;
420 return FALSE;
421 }
422
423 if (TRUE == collector->bufferedStream) {
424 rc = fread(bufOffset, 1, 4, collector->stream.fp);
425 } else {
426 rc = read(collector->stream.fd, bufOffset, 4);
427 }
428
429 if (4 != rc) {
430 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
431 "Error reading set header in NetflowV9 message"
432 " expected read of 4 received %d", rc);
433 *m_len = 0;
434 return FALSE;
435 }
436
437 setHeaderPtr = (struct setHeader_st *)bufOffset;
438 bufOffset += 4;
439 setLength = g_ntohs(setHeaderPtr->setLength);
440
441 if ((unsigned int)((bufOffset-dataBuf) + setLength) < b_len) {
442 g_set_error(err,FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
443 "Error buffer to small to read NetflowV9 message");
444 *m_len = 0;
445 return FALSE;
446 }
447
448 if (TRUE == collector->bufferedStream) {
449 rc = fread(bufOffset, 1, setLength, collector->stream.fp);
450 } else {
451 rc = read(collector->stream.fd, bufOffset, setLength);
452 }
453
454 if (setLength != rc) {
455 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
456 "Error reading NetflowV9 set payload");
457 *m_len = 0;
458 return FALSE;
459 }
460 bufOffset += setLength;
461 }
462
463
464 *m_len = 0;
465 return TRUE;
466 }
467
468 /**
469 * fbCollectorMessageHeaderV9
470 *
471 * this converts a NetFlow V9 header into something every so slightly
472 * closer to an IPFIX header; it dumps the up time with a big memcpy
473 *
474 * @param collector pointer to the collector state structure
475 * @param buffer pointer to the message buffer
476 * @param b_len length of the buffer passed in
477 * @param m_len pointer to the length of the resultant buffer
478 * @param err pointer to a GLib error structure
479 *
480 * @return TRUE on success, FALSE on error
481 */
fbCollectorMessageHeaderV9(fbCollector_t * collector,uint8_t * buffer,size_t b_len,uint16_t * m_len,GError ** err)482 static gboolean fbCollectorMessageHeaderV9(
483 fbCollector_t *collector,
484 uint8_t *buffer,
485 size_t b_len,
486 uint16_t *m_len,
487 GError **err)
488 {
489 uint16_t tempRead16;
490 uint64_t unix_secs;
491 uint64_t sysuptime;
492 struct fbCollectorNetflowV9State_st *transState =
493 (struct fbCollectorNetflowV9State_st *)collector->translatorState;
494
495
496 if (b_len < 20) {
497 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
498 "Invalid NetFlow V9 Header. Buffer Length too short. "
499 "Length: %u", (unsigned int)b_len);
500 return FALSE;
501 }
502
503 /* first make sure the message seems like a NetFlow V9 message */
504 tempRead16 = g_ntohs(*((uint16_t *)buffer));
505
506 if (0x0009 != tempRead16) {
507 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
508 "invalid version number for NetFlow V9, expecting 0x0009,"
509 " received %#06x", tempRead16);
510 return FALSE;
511 }
512
513 READU32((buffer + 4), sysuptime);
514 READU32((buffer + 8), unix_secs);
515 READU32((buffer + 16), collector->obdomain);
516 collector->time = time(NULL);
517
518 /* convert to milliseconds - subtract sysuptime to get time of reboot */
519 transState->sysUpTime = (unix_secs * 1000) - sysuptime;
520 transState->sysUpTime = fb_htonll(transState->sysUpTime);
521
522 /* memcpy is no good here because src & dst overlap */
523 memmove((buffer + 4), (buffer + 8), (b_len - 8));
524
525 /* return that we shortened the buffer */
526 *m_len = b_len - 4;
527
528 return TRUE;
529 }
530
531
532
533 /**
534 * netflowDataTemplateParse
535 *
536 * this parses a NetFlow V9 template and stores the results
537 * into the template hash for this session. It only stores
538 * the template ID and the length of the resulting record.
539 * it will error out on malformed templates and data records
540 * which are not common between IPFIX and NetFlow V9
541 *
542 * @param collector pointer to the collector state record
543 * @param dataBuf pointer to the buffer holding the template def
544 * points <b>after</b> the set ID and set length
545 * @param recordLength pointer to the set header length field
546 * @param msgBuf pointer to the start of the netflow PDU
547 * @param msgLen pointer to the length of the whole msg
548 * @param err GError pointer to store the error if one occurs
549 *
550 * @return Number of Templates Parsed
551 *
552 */
netflowDataTemplateParse(fbCollector_t * collector,uint8_t * dataBuf,uint16_t * recordLength,uint8_t * msgBuf,size_t * msgLen,GError ** err)553 static int netflowDataTemplateParse(
554 fbCollector_t *collector,
555 uint8_t *dataBuf,
556 uint16_t *recordLength,
557 uint8_t *msgBuf,
558 size_t *msgLen,
559 GError **err)
560 {
561 uint16_t templateId = 0;
562 uint16_t fieldCount = 0;
563 uint8_t *bufPtr = dataBuf;
564 uint16_t targetRecSize = 0;
565 uint16_t recLength = g_ntohs(*recordLength);
566 uint16_t lengthParsed = 4; /* to account for set header */
567 uint8_t *fieldCountPtr;
568 unsigned int loop;
569 uint16_t temp;
570 int tmplcount = 0;
571 #ifdef FB_MAP_POST_TO_REVERSE
572 uint8_t addReversePenFix = 0;
573 #endif
574 gboolean addSysUpTime = FALSE;
575 struct fbCollectorNetflowV9State_st *transState =
576 (struct fbCollectorNetflowV9State_st *)collector->translatorState;
577 struct fbCollectorNetflowV9TemplateHash_st *newTemplate = NULL;
578 fbCollectorNetflowV9Session_t *currentSession = transState->session;
579
580 if ((recLength < 8) || 0 != (recLength % 4)) {
581 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
582 "NetFlow template record is either too short or not a "
583 "multiple of 4 octets, (recLength = %u)", recLength);
584 return 0;
585 }
586
587 while ( lengthParsed < recLength ) {
588
589 /* read the template ID */
590 READU16INC(bufPtr, templateId);
591
592 fieldCountPtr = bufPtr;
593 /* read the number of data records in the template */
594 READU16INC(bufPtr, fieldCount);
595
596 /* lets keep a count of how far we've read into the rec */
597 lengthParsed += 4;
598
599 /* subtract 8 from the record length to account for the set
600 header,(type & length 16-bits each)-but this can contain
601 more than 1! */
602
603 if (fieldCount > ((recLength - lengthParsed) / 4)) {
604 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
605 "NetFlow V9 Field Count is greater than remaining "
606 "record length");
607 return 0;
608 }
609
610 /* iterate through each type entry in the template
611 record, make sure the IE model number is within
612 what we can handle and then record the length to
613 build up the length of each template number */
614
615 for (loop = 0; loop < fieldCount; loop++) {
616
617 READU16(bufPtr, temp);
618 if ( 0 == temp ) {
619 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
620 "NetFlow V9 template data record model type is not"
621 " common with IPFIX IE model: Unknown element:%d",
622 temp);
623 return 0;
624 } else if ( temp > 346 ) {
625 /* convert Netflowv9 Arbitrary Element to IPFIX Generic */
626 /* except for 40005 & 33002 (event ids that are in infomodel)*/
627 if ( temp == 40005 ) {
628 WRITEU16(bufPtr, FB_CISCO_ASA_EVENT_ID);
629 } else if ( temp == 33002 ) {
630 WRITEU16(bufPtr, FB_CISCO_ASA_EVENT_XTRA);
631 } else if ( temp == 40001 ) {
632 /* postNATSourceIPv4Address */
633 WRITEU16(bufPtr, 225);
634 } else if ( temp == 40002 ) {
635 /* postNATDestinationIPv4Address */
636 WRITEU16(bufPtr, 226);
637 } else if ( temp == 40003 ) {
638 /* postNAPTSourceTransportPort */
639 WRITEU16(bufPtr, 227);
640 } else if ( temp == 40004 ) {
641 /* postNAPTDestinationTransportPort */
642 WRITEU16(bufPtr, 228);
643 } else {
644 WRITEU16(bufPtr, FB_CISCO_GENERIC);
645 }
646 }
647
648 #ifdef FB_MAP_POST_TO_REVERSE
649 /*
650 * The code inside the #ifdef FB_MAP_POST_TO_REVERSE
651 * blocks in this function maps post*DeltaCount values to
652 * reverse*DeltaCount values. The code was written when
653 * NetFlow v9 support was initially added to fixbuf.
654 */
655
656 /*
657 because the IPFIX standard info model is broken, RFC 7012, we
658 have to convert certain info model elements from their NetFlow
659 v9 values into appropriate IPFIX numbers.
660 (Cisco will be none too pleased)
661 */
662
663 /* convert V9 out bytes field into IPFIX reverseOctetDeltaCount */
664 if (23 == temp) {
665 WRITEU16(bufPtr, (IPFIX_ENTERPRISE_BIT | 1));
666 addReversePenFix = 1;
667 }
668 /* convert V9 out pkts field into IPFIX reversePacketDeltaCount */
669 if (24 == temp) {
670 WRITEU16(bufPtr, (IPFIX_ENTERPRISE_BIT | 2));
671 addReversePenFix = 1;
672 }
673 #endif /* FB_MAP_POST_TO_REVERSE */
674
675 if ((21 == temp) || (22 == temp)) {
676 /* need at add element 160 for sysuptime */
677 addSysUpTime = TRUE;
678 }
679
680 bufPtr += sizeof(uint16_t);
681 lengthParsed += sizeof(uint16_t);
682
683 /* record how long each element is */
684 READU16INC(bufPtr, temp);
685
686 targetRecSize += temp;
687 lengthParsed += sizeof(uint16_t);
688
689 #ifdef FB_MAP_POST_TO_REVERSE
690 /* if we're supposed to add the reverse PEN for a NetFlow V9 ->
691 IPFIX info model change, do it now */
692 if (0 != addReversePenFix) {
693 addReversePenFix = 0;
694 if (FB_MSGLEN_MAX <= (*msgLen + sizeof(uint32_t))) {
695 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
696 "NetFlow V9 unable to convert information model"
697 " elements, no space");
698 return 0;
699 }
700 /* make space for the extra 32-bit PEN */
701 memmove((bufPtr + sizeof(uint32_t)), bufPtr,
702 (*msgLen - (bufPtr - msgBuf)));
703
704 /* write the reverse PEN into the template */
705 WRITEU32(bufPtr, FB_IE_PEN_REVERSE);
706 bufPtr += sizeof(uint32_t);
707 lengthParsed += sizeof(uint32_t);
708
709 /* update the length of this set in the set header */
710 WRITEU16(recordLength,
711 (g_ntohs(*recordLength) + sizeof(uint32_t)));
712
713 /* update the length that this record is; we error check this
714 value later */
715
716 recLength += sizeof(uint32_t);
717
718 /* also increase the msglen, to note the change */
719 *msgLen += sizeof(uint32_t);
720 }
721 #endif /* FB_MAP_POST_TO_REVERSE */
722 }
723
724 /* add the SysUpTime field (IPFIX element 160) to the template */
725 if (addSysUpTime) {
726 if (FB_MSGLEN_MAX <= (*msgLen + sizeof(uint32_t))) {
727 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
728 "NetFlow V9 unable to convert information model "
729 "time elements, no space");
730 return 0;
731 }
732 memmove((bufPtr + sizeof(uint32_t)), bufPtr,
733 (*msgLen - (bufPtr - msgBuf)));
734 /* add (IPFIX info element 160) */
735 WRITEU16(bufPtr, 160);
736 bufPtr += sizeof(uint16_t);
737 /* length is 8 */
738 WRITEU16(bufPtr, 8);
739 bufPtr += sizeof(uint16_t);
740 lengthParsed += sizeof(uint32_t);
741 /* modify the length of this message */
742 WRITEU16(recordLength,(g_ntohs(*recordLength) + sizeof(uint32_t)));
743 recLength += sizeof(uint32_t);
744 *msgLen += sizeof(uint32_t);
745 /* change fieldcount to add this field */
746 WRITEU16(fieldCountPtr, fieldCount + 1);
747 }
748
749 newTemplate = g_slice_new(fbCollectorNetflowV9TemplateHash_t);
750 if (NULL == newTemplate) {
751 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_TMPL,
752 "could not allocate NetFlow v9 template storage "
753 "record");
754 return 0;
755 }
756
757 newTemplate->templateId = templateId;
758 newTemplate->templateLength = targetRecSize;
759 newTemplate->optionsTemplate = FALSE;
760 if (addSysUpTime) {
761 newTemplate->addSysUpTime = TRUE;
762 } else {
763 newTemplate->addSysUpTime = FALSE;
764 }
765 addSysUpTime = FALSE;
766
767 /* put the template into the hash, check/replace the template
768 that is there if this template number already exists */
769 if (currentSession->templateHash == NULL) {
770 currentSession->templateHash =
771 g_hash_table_new_full(g_direct_hash, NULL,
772 NULL, templateHashDestroyHelper);
773
774 if (NULL == currentSession->templateHash) {
775 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
776 "failure to allocate hash table for NetFlow "
777 "session");
778 return FALSE;
779 }
780 }
781
782 g_hash_table_insert(currentSession->templateHash,
783 GUINT_TO_POINTER(templateId), newTemplate);
784 tmplcount++;
785
786 #if FB_NETFLOW_DEBUG == 1
787 fprintf(stderr, "template inserted into hash: templateId %d,"
788 " templateSize: %d, Domain: %04x, SysUpTime %d, "
789 "fieldCount: %d \n",
790 templateId, targetRecSize, transState->observation_id,
791 newTemplate->addSysUpTime, fieldCount);
792 #endif
793 targetRecSize = 0; /* running tmpl size needs reset */
794
795 }
796
797 /* return the amount of templates we added so we can keep a track
798 of the total number of records parsed */
799 return tmplcount;
800 }
801
802
803 /**
804 * netflowOptionsTemplateParse
805 *
806 * this parses a NetFlow V9 options template and stores the results
807 * into the template hash for this session. It only stores
808 * the template ID and the length of the resulting record.
809 * it will error out on malformed templates
810 *
811 * @param collector pointer to the collector state record
812 * @param dataBuf pointer to the buffer holding the template def
813 * points <b>after</b> the set ID and set length
814 * @param recordLength a pointer to the length of the flowSet
815 * @param err GError pointer to store the error if one occurs
816 *
817 * @return number of templates parsed
818 *
819 */
netflowOptionsTemplateParse(fbCollector_t * collector,uint8_t * dataBuf,uint16_t * recordLength,GError ** err)820 static int netflowOptionsTemplateParse(
821 fbCollector_t *collector,
822 uint8_t *dataBuf,
823 uint16_t *recordLength,
824 GError **err)
825 {
826 uint16_t templateId = 0;
827 uint8_t *recOffset = dataBuf;
828 uint8_t *optScopeLenPtr;
829 uint8_t *optLenPtr;
830 uint16_t lengthParsed = 4; /* for set header */
831 uint16_t optScopeLen, optLen, type, fieldLen;
832 uint16_t templateLength = 0;
833 uint16_t recLength = g_ntohs(*recordLength);
834 unsigned int loop;
835 unsigned int fieldCount;
836 int tmplcount = 0;
837 struct fbCollectorNetflowV9State_st *transState =
838 (struct fbCollectorNetflowV9State_st *)collector->translatorState;
839 struct fbCollectorNetflowV9TemplateHash_st *newTemplate = NULL;
840 fbCollectorNetflowV9Session_t *currentSession = transState->session;
841
842 while (lengthParsed < recLength) {
843
844 fieldCount = 0;
845
846 if ((recLength - lengthParsed) < 6) {
847 /* not enough for an options header - probably extra padding */
848 return tmplcount;
849 }
850
851 /* read the template ID */
852 READU16INC(recOffset,templateId);
853 lengthParsed += sizeof(uint16_t);
854
855
856 /* keep a placeholder for the options Scope Length because we need
857 to change it to number of Scope Elements for IPFIX */
858 optScopeLenPtr = recOffset;
859
860 /* read the length of option scope bytes */
861 READU16INC(recOffset,optScopeLen);
862 lengthParsed += sizeof(uint16_t);
863
864 /* keep a placeholder for the options Length because we need to change
865 it to number of elements */
866 optLenPtr = recOffset;
867
868 /* read the length of the option bytes */
869 READU16INC(recOffset,optLen);
870 lengthParsed += sizeof(uint16_t);
871
872 if (!optLen && !optScopeLen) {
873 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
874 "Options Template has No Length. "
875 "(optScopeLen %u, optLen %u).", optScopeLen,
876 optLen);
877 return 0;
878 }
879
880 #if FB_NETFLOW_DEBUG == 1
881 fprintf(stderr, "Options Tmpl ID:%d; ScopeLen %d; Len %d\n",
882 templateId, optScopeLen, optLen); /* debug */
883 #endif
884
885 /* check the option scope length + option length to check to make sure
886 that this record is sane - just make sure there is enough room*/
887 if (recLength < (optScopeLen + optLen + lengthParsed)) {
888 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
889 "Record Length is too short for options fields "
890 "(recLength %u optScopeLen %u optLen %u len parsed %d)",
891 recLength, optScopeLen, optLen, lengthParsed);
892 return 0;
893 }
894
895 /* Get past Option Scope Elements */
896
897 for (loop = 0; loop < optScopeLen; loop += 4) {
898 /* replace option scope type with messageScope info element*/
899 WRITEU16(recOffset,263);
900 recOffset+=sizeof(uint16_t);
901
902 READU16INC(recOffset,fieldLen);
903 fieldCount++;
904 lengthParsed += 4;
905 templateLength += fieldLen;
906 }
907
908 /* IPFIX requires a scope count
909 (otherwise - why is it an options tmpl?) */
910 /* Cisco doesn't require a scope count - so make sure we have 1 */
911 if (!fieldCount) {
912 WRITEU16(optLenPtr,1);
913 } else {
914 /* In the IPFIX Options Template,
915 Scope Field Count is actually the 3rd item
916 in the Options header */
917 WRITEU16(optLenPtr,fieldCount);
918 }
919
920 /* Get past Options Elements */
921 for (loop = 0; loop < optLen; loop += 4) {
922
923 READU16INC(recOffset,type);
924 READU16INC(recOffset,fieldLen);
925 lengthParsed += 4;
926 fieldCount++;
927 templateLength += fieldLen;
928 }
929
930 /* In IPFIX, Field Count is the total number of fields (including
931 Scope Fields. */
932
933 WRITEU16(optScopeLenPtr,fieldCount);
934
935 newTemplate = g_slice_new(fbCollectorNetflowV9TemplateHash_t);
936 if (NULL == newTemplate) {
937 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_TMPL,
938 "could not allocate NetFlow v9 template storage record");
939 return FALSE;
940 }
941
942 newTemplate->templateId = templateId;
943 newTemplate->templateLength = templateLength;
944 newTemplate->optionsTemplate = TRUE;
945 newTemplate->addSysUpTime = FALSE;
946 /* if there is no TemplateHash this is the first template we
947 are receiving in the current domain. Create a Hash for the domain.*/
948 if (currentSession->templateHash == NULL) {
949
950 currentSession->templateHash =
951 g_hash_table_new_full(g_direct_hash, NULL,
952 NULL, templateHashDestroyHelper);
953
954 if (NULL == currentSession->templateHash) {
955 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
956 "failure to allocate hash table for NetFlow "
957 "session");
958 return FALSE;
959 }
960 }
961
962 g_hash_table_insert(currentSession->templateHash,
963 GUINT_TO_POINTER(templateId), newTemplate);
964
965 #if FB_NETFLOW_DEBUG == 1
966 fprintf(stderr, "Options template inserted into hash: templateId %d,"
967 " templateSize: %d, Domain: %04x, SysUpTime %d, "
968 "fieldCount: %u \n",
969 templateId, templateLength, transState->observation_id,
970 newTemplate->addSysUpTime, fieldCount);
971 #endif
972
973 templateLength = 0; /* reset to 0 after each tmpl */
974
975 tmplcount++;
976 }
977
978 /* return the number of templates added to keep track of record count */
979 return tmplcount;
980 }
981
982
983 /**
984 * fbCollectorPostProcV9
985 *
986 * converts a buffer that was read as a netflow V9
987 * into a buffer that "conforms" to IPFIX for the
988 * rest of fixbuf to process it
989 *
990 * @param collector pointer to the collector state structure
991 * @param dataBuf pointer to the netflow PDU
992 * @param bufLen the size from UDP of the message
993 * @param err glib error set when FALSE is returned with an
994 * informative message
995 *
996 * @return TRUE on success FALSE on error
997 */
fbCollectorPostProcV9(fbCollector_t * collector,uint8_t * dataBuf,size_t * bufLen,GError ** err)998 static gboolean fbCollectorPostProcV9(
999 fbCollector_t *collector,
1000 uint8_t *dataBuf,
1001 size_t *bufLen,
1002 GError **err)
1003 {
1004 uint16_t recordCount;
1005 uint16_t recordCounter = 0;
1006 int i;
1007 uint8_t *msgOsetPtr = dataBuf;
1008 uint16_t padding = 0;
1009 uint16_t *lengthCountPtr = NULL;
1010 uint16_t *recLengthPtr = NULL;
1011 uint32_t netflowSeqNum;
1012 uint32_t ipfixRecordCount = 0;
1013 struct fbCollectorNetflowV9State_st *transState =
1014 (struct fbCollectorNetflowV9State_st *)collector->translatorState;
1015 uint32_t timeStamp;
1016 uint32_t obsDomain;
1017 uint16_t version;
1018 uint32_t *seqNumPtr;
1019 uint8_t tmpls_parsed;
1020 uint16_t setId;
1021 uint16_t recordLength;
1022 fbCollectorNetflowV9Session_t *currentSession = NULL;
1023 #if FB_NETFLOW_DEBUG == 1
1024 uint16_t flowSet = 0;
1025 uint16_t dLoop; /* debug */
1026 /* debug */
1027 {
1028 uint8_t *d_tPtr8 = (uint8_t *)dataBuf;
1029 uint16_t *d_tPtr = (uint16_t *)dataBuf;
1030 uint16_t d_setid = g_ntohs(*d_tPtr);
1031 d_tPtr++;
1032 uint16_t d_setLen = g_ntohs(*d_tPtr);
1033 fprintf(stderr, "\nversion: %d count: %d\n",
1034 d_setid, d_setLen );
1035 for (dLoop=0; dLoop < *bufLen; dLoop++) {
1036 fprintf(stderr, "0x%02x ", (d_tPtr8)[dLoop]);
1037 if (0 == (dLoop+1)%4 && 0 != dLoop) fprintf(stderr, "\n");
1038 }
1039 }
1040
1041 #endif /* FB_NETFLOW_DEBUG */
1042
1043 /* the buffer header has been partially converted into
1044 IPFIX, mostly meaning that the extra uptime has been
1045 dropped; now the count has to be converted into length,
1046 and all the different records need to be updated
1047 appropriately */
1048
1049 READU16(msgOsetPtr, version);
1050 WRITEU16(msgOsetPtr, 0x0a);
1051 msgOsetPtr += sizeof(uint16_t);
1052
1053 lengthCountPtr = (uint16_t *)msgOsetPtr;
1054 READU16INC(msgOsetPtr, recordCount);
1055 READU32INC(msgOsetPtr, timeStamp);
1056
1057 seqNumPtr = (uint32_t *)msgOsetPtr;
1058 READU32INC(msgOsetPtr, netflowSeqNum);
1059
1060 #if FB_NETFLOW_DEBUG
1061 fprintf(stderr, "Sequence number %u\n", netflowSeqNum);
1062 #endif
1063
1064 /* read the observation domain */
1065 READU32INC(msgOsetPtr, obsDomain);
1066
1067 pthread_mutex_lock(&transState->ts_lock);
1068
1069 transState->observation_id = obsDomain;
1070
1071 if (transState->sessionptr != collector->udp_head->session) {
1072 /* lookup template Hash Table per Domain */
1073 transState->session =
1074 g_hash_table_lookup(transState->domainHash,
1075 collector->udp_head->session);
1076 if (transState->session == NULL) {
1077 transState->session = g_slice_new0(fbCollectorNetflowV9Session_t);
1078 g_hash_table_insert(transState->domainHash,
1079 (gpointer)collector->udp_head->session,
1080 transState->session);
1081 }
1082 transState->sessionptr = collector->udp_head->session;
1083 }
1084
1085 currentSession = transState->session;
1086
1087 /* seq num logic */
1088 if (currentSession->netflowSeqNum != netflowSeqNum) {
1089 int seq_diff = netflowSeqNum - currentSession->netflowSeqNum;
1090 #ifndef FB_SUPPRESS_LOGS
1091 g_log(G_LOG_DOMAIN, G_LOG_LEVEL_WARNING,
1092 "NetFlow V9 sequence number mismatch for domain 0x%04x, "
1093 "expecting 0x%04x received 0x%04x", obsDomain,
1094 currentSession->netflowSeqNum, netflowSeqNum);
1095 #endif
1096 if (currentSession->netflowSeqNum) {
1097 if (seq_diff > 0) {
1098 if (seq_diff > NF_MAX_SEQ_DIFF) {
1099 /* check for reboot */
1100 if (transState->sysUpTime > NF_REBOOT_SECS) {
1101 /* probably not a reboot so account for missed */
1102 currentSession->netflowMissed += seq_diff;
1103 } /* else - reboot? don't add to missed count */
1104 } else {
1105 currentSession->netflowMissed += seq_diff;
1106 }
1107 currentSession->netflowSeqNum = netflowSeqNum;
1108 } else {
1109 /* out of order or reboot? */
1110 if ((currentSession->netflowSeqNum - netflowSeqNum) >
1111 NF_OUT_OF_ORDER)
1112 {
1113 /* this may be a reboot - it's pretty out of seq. */
1114 currentSession->netflowSeqNum = netflowSeqNum;
1115 } else {
1116 /* this is in accepted range for out of sequence */
1117 /* account for not missing. don't reset sequence number */
1118 /* But subtract one so when we add one below, it evens out */
1119 if (currentSession->netflowMissed) {
1120 currentSession->netflowMissed -= 1;
1121 }
1122 currentSession->netflowSeqNum -= 1;
1123 }
1124 }
1125 } else {
1126 /* this is the first one we received in this session */
1127 currentSession->netflowSeqNum = netflowSeqNum;
1128 }
1129 }
1130
1131 /* iterate through the flowsets */
1132 /* a flowset can contain more than 1 record */
1133 /* recordLength is the TOTAL Length of the flowset - not each record */
1134
1135 while ( msgOsetPtr < (dataBuf + *bufLen) ) {
1136
1137 /* read the set ID, and adjust the reserved ones from
1138 Netflow to IPFIX */
1139 READU16INC(msgOsetPtr, setId);
1140 recLengthPtr = (uint16_t*)msgOsetPtr;
1141 READU16INC(msgOsetPtr, recordLength);
1142
1143 #if FB_NETFLOW_DEBUG == 1
1144 fprintf(stderr, "FlowSet %u; SetId %u; Length %u\n",
1145 ++flowSet, setId, recordLength); /* debug */
1146 #endif
1147
1148 if (recordLength < 4) {
1149 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1150 "Invalid Netflow %s Record Length (%u < 4)",
1151 ((1 == setId) ? "Options" : "Data"), recordLength);
1152 pthread_mutex_unlock(&transState->ts_lock);
1153 return FALSE;
1154 }
1155 /* Check to make sure we won't overrun buffer - Add 4 for set header */
1156 if (recordLength > ((dataBuf + *bufLen + 4) - msgOsetPtr)) {
1157 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1158 "Malformed NetFlow Record: %s Length (%d) is "
1159 "larger than remaining buffer length (%ld)",
1160 (1 == setId) ? "Options" : "Record",
1161 recordLength, ((dataBuf + *bufLen + 4) - msgOsetPtr));
1162 pthread_mutex_unlock(&transState->ts_lock);
1163 return FALSE;
1164 }
1165
1166 if (0 == setId) {
1167 /* TEMPLATE RECORD */
1168 /* Template SET ID = 0 in netflow, 2 in IPFIX */
1169 WRITEU16(msgOsetPtr-2*sizeof(uint16_t), 2);
1170
1171 tmpls_parsed = netflowDataTemplateParse(collector, msgOsetPtr,
1172 recLengthPtr,
1173 dataBuf, bufLen, err);
1174 if (!tmpls_parsed) {
1175 pthread_mutex_unlock(&transState->ts_lock);
1176 return FALSE;
1177 }
1178
1179 recordCounter += tmpls_parsed;
1180
1181 /* adjust the message pointer to skip over the payload of
1182 the template record, read the size from the record
1183 because it may have been updated in the template parse
1184 call (take into account that the pointer is 4-bytes
1185 into the record already (type & length)) */
1186 msgOsetPtr += (g_ntohs(*recLengthPtr) - (2 * sizeof(uint16_t)));
1187
1188 } else if (1 == setId) {
1189 /* OPTIONS TEMPLATE */
1190 /* Template SET ID = 3 for IPFIX */
1191 WRITEU16(msgOsetPtr-2*sizeof(uint16_t),3);
1192
1193 tmpls_parsed = netflowOptionsTemplateParse(collector, msgOsetPtr,
1194 recLengthPtr, err);
1195 if (!tmpls_parsed) {
1196 /* Needs to contain at least 1 */
1197 pthread_mutex_unlock(&transState->ts_lock);
1198 return FALSE;
1199 }
1200
1201 recordCounter += tmpls_parsed;
1202
1203 /* adjust the message pointer to skip over the payload of
1204 the template record (take into account that the pointer is
1205 4-bytes into the record already (type & length)) */
1206 msgOsetPtr += recordLength - (2 * sizeof(uint16_t));
1207
1208 } else if (setId < 256) {
1209 /* data records must be 256 or higher */
1210 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1211 "NetFlow record type (%u) is not supported", setId);
1212 pthread_mutex_unlock(&transState->ts_lock);
1213 return FALSE;
1214
1215 } else {
1216 /* DATA */
1217 struct fbCollectorNetflowV9TemplateHash_st *derTemplate = NULL;
1218 uint16_t numberRecordsInSet = 0;
1219 uintptr_t bigSetId = (uintptr_t) setId;
1220
1221 if (NULL == currentSession->templateHash) {
1222 /* return if this is the last FlowSet in the packet */
1223 if ((dataBuf + *bufLen) <= (msgOsetPtr - 4 + recordLength)) {
1224 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1225 "No Templates Present for this session."
1226 " %u Flows Lost.", recordCount-recordCounter);
1227 currentSession->netflowSeqNum++;
1228 pthread_mutex_unlock(&transState->ts_lock);
1229 return FALSE;
1230 }
1231 /* else, remove these bytes from the packet */
1232 #if FB_NETFLOW_DEBUG == 1
1233 fprintf(stderr, "remove data set with no template\n");
1234 #endif
1235
1236 g_log(G_LOG_DOMAIN, G_LOG_LEVEL_WARNING,
1237 "No Templates Present for this session.");
1238
1239 msgOsetPtr -= 4;
1240 memmove(msgOsetPtr, (msgOsetPtr + recordLength),
1241 *bufLen - ((msgOsetPtr + recordLength) - dataBuf));
1242 *bufLen -= recordLength;
1243
1244 } else if ((derTemplate =
1245 g_hash_table_lookup(currentSession->templateHash,
1246 (gconstpointer)bigSetId)) == NULL)
1247 {
1248 if ((dataBuf + *bufLen) <= (msgOsetPtr - 4 + recordLength)) {
1249 /* return if this is the last FlowSet in the packet */
1250 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1251 "No Template %#06x Present for this Session."
1252 " %u Flows Lost.", setId,
1253 (recordCount-recordCounter));
1254 currentSession->netflowSeqNum++;
1255 pthread_mutex_unlock(&transState->ts_lock);
1256 return FALSE;
1257 }
1258 /* else, remove these bytes from the packet */
1259 #if FB_NETFLOW_DEBUG == 1
1260 fprintf(stderr, "remove data set with no template\n");
1261 #endif
1262
1263 g_log(G_LOG_DOMAIN, G_LOG_LEVEL_WARNING,
1264 "No Template %#06x Present for Session", setId);
1265
1266 msgOsetPtr -= 4;
1267 memmove((msgOsetPtr), (msgOsetPtr + recordLength),
1268 *bufLen - ((msgOsetPtr + recordLength) - dataBuf));
1269 *bufLen -= recordLength;
1270
1271 } else {
1272
1273 numberRecordsInSet = (recordLength - 4) /
1274 derTemplate->templateLength;
1275 /* 4 for set id and length */
1276 padding = ((recordLength - 4) % derTemplate->templateLength);
1277
1278 #if FB_NETFLOW_DEBUG == 1
1279 fprintf(stderr,
1280 "number of data records in set %#06x is %d (%#x)\n",
1281 setId, numberRecordsInSet, numberRecordsInSet);
1282 #endif
1283 if (numberRecordsInSet == 0) {
1284 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1285 "NetFlow Data Record with 0 Records");
1286 pthread_mutex_unlock(&transState->ts_lock);
1287 return FALSE;
1288 }
1289
1290 recordCounter += numberRecordsInSet;
1291 ipfixRecordCount += numberRecordsInSet;
1292
1293 /* now check if need to add sysuptime to the record */
1294 if (derTemplate->addSysUpTime) {
1295 for (i = 0; i < numberRecordsInSet; i++) {
1296 msgOsetPtr += derTemplate->templateLength;
1297 if (FB_MSGLEN_MAX <= (*bufLen + sizeof(uint64_t))) {
1298 g_set_error(err, FB_ERROR_DOMAIN,
1299 FB_ERROR_NETFLOWV9,
1300 "NetFlow V9 unable to convert "
1301 "information model "
1302 "time elements, no space");
1303 pthread_mutex_unlock(&transState->ts_lock);
1304 return FALSE;
1305 }
1306
1307 memmove((msgOsetPtr + sizeof(uint64_t)), msgOsetPtr,
1308 (*bufLen - (msgOsetPtr - dataBuf)));
1309 /* add sysUpTime to flow record */
1310 memcpy(msgOsetPtr, &(transState->sysUpTime),
1311 sizeof(uint64_t));
1312 msgOsetPtr += sizeof(uint64_t);
1313 *bufLen += sizeof(uint64_t);
1314 }
1315 msgOsetPtr += padding;
1316 *recLengthPtr = g_htons(recordLength +
1317 (numberRecordsInSet *
1318 sizeof(uint64_t)));
1319 } else {
1320 /* subtract 4 since we already incremented msgOsetPtr 4
1321 for id & length */
1322 msgOsetPtr += recordLength - 4;
1323 }
1324 }
1325 }
1326 }
1327
1328 /* fixup the length value (from record count)*/
1329 *lengthCountPtr = g_htons(msgOsetPtr - dataBuf);
1330
1331 if ((msgOsetPtr - dataBuf) < (long)*bufLen) {
1332 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1333 "NetFlow Record Length Mismatch: (buffer has "
1334 "%u, processed %u)", (unsigned int)(*bufLen),
1335 ntohs(*lengthCountPtr));
1336 currentSession->netflowSeqNum++;
1337 pthread_mutex_unlock(&transState->ts_lock);
1338 return FALSE;
1339 }
1340
1341 /* fixup the sequence number */
1342 *seqNumPtr = g_htonl(currentSession->ipfixSeqNum);
1343
1344 /* increment the ipfix record count with the number of relevent
1345 records we observed*/
1346 currentSession->ipfixSeqNum += ipfixRecordCount;
1347
1348 /* increment the sequence number for the netflow side */
1349 currentSession->netflowSeqNum++;
1350
1351 if (recordCount != recordCounter) {
1352 #ifndef FB_SUPPRESS_LOGS
1353 g_log(G_LOG_DOMAIN, G_LOG_LEVEL_WARNING,
1354 "NetFlow V9 Record Count Discrepancy. "
1355 "Reported: %d. Found: %d.",
1356 recordCount, recordCounter);
1357 #endif
1358 }
1359
1360 #if FB_NETFLOW_DEBUG == 1
1361 /* debug */
1362 {
1363 uint8_t *d_tPtr8 = (uint8_t *)dataBuf;
1364 uint16_t *d_tPtr = (uint16_t *)dataBuf;
1365 uint16_t d_setid = g_ntohs(*d_tPtr);
1366 d_tPtr++;
1367 uint16_t d_setLen = g_ntohs(*d_tPtr);
1368 fprintf(stderr, "\nversion: %d count: %d\n",
1369 d_setid, d_setLen );
1370 for (dLoop=0; dLoop < *bufLen; dLoop++) {
1371 fprintf(stderr, "0x%02x ", (d_tPtr8)[dLoop]);
1372 if (0 == (dLoop+1)%4 && 0 != dLoop) fprintf(stderr, "\n");
1373 }
1374 }
1375 #endif
1376
1377 pthread_mutex_unlock(&transState->ts_lock);
1378
1379 return TRUE;
1380 }
1381
1382
1383
1384
1385
1386 /**
1387 * fbCollectorTransCloseV9
1388 *
1389 * frees the state included as part of the collector when the
1390 * Netflow V9 translator is enabled
1391 *
1392 * @param collector, pointer to the collector state structure
1393 *
1394 */
fbCollectorTransCloseV9(fbCollector_t * collector)1395 static void fbCollectorTransCloseV9(
1396 fbCollector_t *collector)
1397 {
1398 struct fbCollectorNetflowV9State_st *transState =
1399 (struct fbCollectorNetflowV9State_st *)collector->translatorState;
1400
1401 /* this should destroy each entry in the template */
1402 g_hash_table_destroy(transState->domainHash);
1403 transState->domainHash = NULL;
1404
1405 pthread_mutex_destroy(&transState->ts_lock);
1406
1407 if (NULL != collector->translatorState) {
1408 g_free(collector->translatorState);
1409 }
1410
1411 collector->translatorState = NULL;
1412 return;
1413 }
1414
1415 /**
1416 * fbCollectorTimeoutNetflowSession
1417 *
1418 * this timeouts sessions when we haven't seen messages for > 30 mins.
1419 *
1420 * @param collector pointer to collector state.
1421 * @param session pointer to session to timeout.
1422 *
1423 */
fbCollectorTimeOutSessionV9(fbCollector_t * collector,fbSession_t * session)1424 static void fbCollectorTimeOutSessionV9(
1425 fbCollector_t *collector,
1426 fbSession_t *session)
1427 {
1428 struct fbCollectorNetflowV9State_st *transState =
1429 (struct fbCollectorNetflowV9State_st *)collector->translatorState;
1430 fbCollectorNetflowV9Session_t *nfsession = NULL;
1431
1432 if (transState == NULL) {
1433 return;
1434 }
1435
1436 pthread_mutex_lock(&transState->ts_lock);
1437
1438 nfsession = g_hash_table_lookup(transState->domainHash, session);
1439 if (nfsession == NULL) {
1440 /* don't need to free! */
1441 pthread_mutex_unlock(&transState->ts_lock);
1442 return;
1443 }
1444
1445 /* remove this session, free the state */
1446 g_hash_table_remove(transState->domainHash, session);
1447
1448 if (session == transState->sessionptr) {
1449 transState->sessionptr = NULL;
1450 transState->session = NULL;
1451 }
1452
1453 pthread_mutex_unlock(&transState->ts_lock);
1454
1455 }
1456
1457
1458
1459 /**
1460 *fbCollectorSetNetflowV9Translator
1461 *
1462 * this sets the collector input translator
1463 * to convert NetFlowV9 into IPFIX for the
1464 * given collector
1465 *
1466 * @param collector pointer to the collector state
1467 * to perform Netflow V9 conversion on
1468 * @param err GError structure that holds the error
1469 * message if an error occurs
1470 *
1471 *
1472 * @return TRUE on success, FALSE on error
1473 */
fbCollectorSetNetflowV9Translator(fbCollector_t * collector,GError ** err)1474 gboolean fbCollectorSetNetflowV9Translator(
1475 fbCollector_t *collector,
1476 GError **err)
1477 {
1478 GHashTable *hashTable = NULL;
1479 struct fbCollectorNetflowV9State_st *nflowState =
1480 g_malloc(sizeof(struct fbCollectorNetflowV9State_st));
1481
1482 if (NULL == nflowState) {
1483 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_TRANSMISC,
1484 "failure to allocate Netflow V9 translator state");
1485 return FALSE;
1486 }
1487
1488
1489 hashTable = g_hash_table_new_full(g_direct_hash, NULL, NULL,
1490 domainHashDestroyHelper);
1491
1492 if (NULL == hashTable) {
1493 g_set_error(err, FB_ERROR_DOMAIN, FB_ERROR_NETFLOWV9,
1494 "failure to allocate sequence no. hash table for "
1495 "Netflow Session");
1496 return FALSE;
1497 }
1498
1499 #if FB_NETFLOW_DEBUG == 1
1500 fprintf(stderr, "Hash table address is %p for collector %p\n",
1501 hashTable, collector); /* debug */
1502 #endif
1503
1504 nflowState->domainHash = hashTable;
1505 nflowState->observation_id = 0;
1506 nflowState->sessionptr = NULL;
1507 pthread_mutex_init(&nflowState->ts_lock, NULL);
1508
1509 return fbCollectorSetTranslator(collector, fbCollectorPostProcV9,
1510 fbCollectorDecodeV9MsgVL, fbCollectorMessageHeaderV9,
1511 fbCollectorTransCloseV9, fbCollectorTimeOutSessionV9, nflowState, err);
1512
1513 }
1514
1515
1516 /**
1517 * fbCollectorGetNetflowMissed
1518 *
1519 * This returns the number of potential missed export packets
1520 * If peer is set, we search for a match and return the number of missed
1521 * packets for that ip/obdomain pair. If peer is not set then we just
1522 * return the most recent UDP connection stats. If peer is set and we don't
1523 * have a match, we just return 0.
1524 * we can't return the number of missed flow records since Netflow v9
1525 * increases sequence numbers by the number of export packets it has sent,
1526 * NOT the number of flow records (like IPFIX and netflow v5 does).
1527 *
1528 * @param collector
1529 * @param peer address of exporter to lookup
1530 * @param peerlen sizeof(peer)
1531 * @param obdomain observation domain of peer exporter
1532 * @return number of missed packets
1533 *
1534 */
1535
fbCollectorGetNetflowMissed(fbCollector_t * collector,struct sockaddr * peer,size_t peerlen,uint32_t obdomain)1536 uint32_t fbCollectorGetNetflowMissed(
1537 fbCollector_t *collector,
1538 struct sockaddr *peer,
1539 size_t peerlen,
1540 uint32_t obdomain)
1541 {
1542 struct fbCollectorNetflowV9State_st *ts = NULL;
1543 fbUDPConnSpec_t *udp = NULL;
1544 fbSession_t *session = NULL;
1545 fbCollectorNetflowV9Session_t *ts_session = NULL;
1546 uint32_t missed = 0;
1547
1548 if (!collector) {
1549 return 0;
1550 }
1551
1552 if (peer) {
1553 udp = collector->udp_head;
1554 while (udp) {
1555 /* loop through and find the match */
1556 if (udp->obdomain == obdomain) {
1557 if (!memcmp(&(udp->peer), peer, (peerlen > udp->peerlen) ?
1558 udp->peerlen : peerlen))
1559 {
1560 /* we have a match - set session */
1561 session = udp->session;
1562 break;
1563 }
1564 }
1565 udp = udp->next;
1566 }
1567 } else {
1568 /* set to most recent */
1569 session = collector->udp_head->session;
1570 }
1571
1572 if (!session) {
1573 return 0;
1574 }
1575
1576 ts = (struct fbCollectorNetflowV9State_st *)collector->translatorState;
1577
1578 if (ts == NULL) {
1579 g_warning("NFv9 Translator not set on collector.");
1580 return 0;
1581 }
1582
1583 pthread_mutex_lock(&ts->ts_lock);
1584
1585 if (ts->sessionptr != session) {
1586 /* lookup template Hash Table per Domain */
1587 ts_session = g_hash_table_lookup(ts->domainHash, session);
1588 } else {
1589 ts_session = ts->session;
1590 }
1591
1592 if (ts_session) {
1593 missed = ts_session->netflowMissed;
1594 }
1595
1596 pthread_mutex_unlock(&ts->ts_lock);
1597
1598 return missed;
1599 }
1600