1 /*
2 * Copyright (c) 2009-2020, Peter Haag
3 * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * * Neither the name of the author nor the names of its contributors may be
15 * used to endorse or promote products derived from this software without
16 * specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #ifdef HAVE_CONFIG_H
33 #include "config.h"
34 #endif
35
36 #include <stdio.h>
37 #include <sys/types.h>
38 #include <sys/socket.h>
39 #include <unistd.h>
40 #include <stdlib.h>
41 #include <netinet/in.h>
42 #include <arpa/inet.h>
43 #include <errno.h>
44 #include <string.h>
45
46 #ifdef HAVE_STDINT_H
47 #include <stdint.h>
48 #endif
49
50 #include "util.h"
51 #include "nfdump.h"
52 #include "nffile.h"
53 #include "nfx.h"
54 #include "nfnet.h"
55 #include "output_raw.h"
56 #include "bookkeeper.h"
57 #include "collector.h"
58 #include "exporter.h"
59 #include "netflow_v1.h"
60
61 extern extension_descriptor_t extension_descriptor[];
62
63 /* module limited globals */
64 static int verbose;
65 static extension_info_t v1_extension_info; // common for all v1 records
66 static uint16_t v1_output_record_size;
67
68 // All required extension to save full v1 records
69 static uint16_t v1_full_map[] = { EX_IO_SNMP_2, EX_NEXT_HOP_v4, EX_ROUTER_IP_v4, EX_RECEIVED, 0 };
70
71 typedef struct v1_block_s {
72 uint32_t srcaddr;
73 uint32_t dstaddr;
74 uint32_t dPkts;
75 uint32_t dOctets;
76 uint32_t data[1]; // link to next record
77 } v1_block_t;
78 #define V1_BLOCK_DATA_SIZE (sizeof(v1_block_t) - sizeof(uint32_t))
79
80 typedef struct exporter_v1_s {
81 // identical to exporter_t
82 struct exporter_v1_s *next;
83
84 // exporter information
85 exporter_info_record_t info;
86
87 uint64_t packets; // number of packets sent by this exporter
88 uint64_t flows; // number of flow records sent by this exporter
89 uint32_t sequence_failure; // number of sequence failues
90 uint32_t padding_errors; // number of sequence failues
91
92 sampler_t *sampler;
93 // End of exporter_t
94
95 // extension map
96 extension_map_t *extension_map;
97
98 } exporter_v1_t;
99
100 static inline exporter_v1_t *GetExporter(FlowSource_t *fs, netflow_v1_header_t *header);
101
102 /* functions */
103
104 #include "nffile_inline.c"
105
Init_v1(int v)106 int Init_v1(int v) {
107 int i, id, map_index;
108 int extension_size;
109 uint16_t map_size;
110
111 verbose = v;
112 // prepare v1 extension map
113 v1_extension_info.map = NULL;
114 v1_extension_info.next = NULL;
115 v1_extension_info.offset_cache = NULL;
116 v1_extension_info.ref_count = 0;
117
118 extension_size = 0;
119 // default map - 0 extensions
120 map_size = sizeof(extension_map_t);
121 i=0;
122 dbg_printf("v1 map: map size start: %u\n", map_size);
123 while ( (id = v1_full_map[i]) != 0 ) {
124 if ( extension_descriptor[id].enabled ) {
125 extension_size += extension_descriptor[id].size;
126 map_size += sizeof(uint16_t);
127 dbg_printf("v1 map: enabled extension %u\n", id);
128 }
129 i++;
130 }
131 dbg_printf("v1 map: map size so far: %u\n", map_size);
132
133 // extension_size contains the sum of all optional extensions
134 // caculate the record size
135 v1_output_record_size = COMMON_RECORD_DATA_SIZE + V1_BLOCK_DATA_SIZE + extension_size;
136
137 // align 32 bits
138 if ( ( map_size & 0x3 ) != 0 )
139 map_size += 2;
140
141 // Create a netflow v1 extension map
142 v1_extension_info.map = (extension_map_t *)malloc((size_t)map_size);
143 if ( !v1_extension_info.map ) {
144 LogError("Process_v1: malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno));
145 return 0;
146 }
147
148 v1_extension_info.map->type = ExtensionMapType;
149 v1_extension_info.map->size = map_size;
150 v1_extension_info.map->map_id = INIT_ID;
151 v1_extension_info.map->extension_size = extension_size;
152
153 // see netflow_v1.h for extension map description
154 map_index = 0;
155 i=0;
156 while ( (id = v1_full_map[i]) != 0 ) {
157 if ( extension_descriptor[id].enabled )
158 v1_extension_info.map->ex_id[map_index++] = id;
159 i++;
160 }
161 v1_extension_info.map->ex_id[map_index] = 0;
162
163 return 1;
164 } // End of Init_v1
165
166 /*
167 * functions used for receiving netflow v1 records
168 */
169
170
GetExporter(FlowSource_t * fs,netflow_v1_header_t * header)171 static inline exporter_v1_t *GetExporter(FlowSource_t *fs, netflow_v1_header_t *header) {
172 exporter_v1_t **e = (exporter_v1_t **)&(fs->exporter_data);
173 uint16_t version = ntohs(header->version);
174 #define IP_STRING_LEN 40
175 char ipstr[IP_STRING_LEN];
176
177 // search the appropriate exporter engine
178 while ( *e ) {
179 if ( (*e)->info.version == version &&
180 (*e)->info.ip.V6[0] == fs->ip.V6[0] && (*e)->info.ip.V6[1] == fs->ip.V6[1])
181 return *e;
182 e = &((*e)->next);
183 }
184
185 // nothing found
186 *e = (exporter_v1_t *)malloc(sizeof(exporter_v1_t));
187 if ( !(*e)) {
188 LogError("Process_v1: malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno));
189 return NULL;
190 }
191 memset((void *)(*e), 0, sizeof(exporter_v1_t));
192 (*e)->info.header.type = ExporterInfoRecordType;
193 (*e)->info.header.size = sizeof(exporter_info_record_t);
194 (*e)->info.version = version;
195 (*e)->info.id = 0;
196 (*e)->info.ip = fs->ip;
197 (*e)->info.sa_family = fs->sa_family;
198 (*e)->next = NULL;
199 (*e)->packets = 0;
200 (*e)->flows = 0;
201 (*e)->sequence_failure = 0;
202 (*e)->padding_errors = 0;
203 (*e)->sampler = NULL;
204
205 // copy the v1 extension map
206 (*e)->extension_map = (extension_map_t *)malloc(v1_extension_info.map->size);
207 if ( !(*e)->extension_map ) {
208 LogError("Process_v1: malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror (errno));
209 free(*e);
210 *e = NULL;
211 return NULL;
212 }
213 memcpy((void *)(*e)->extension_map, (void *)v1_extension_info.map, v1_extension_info.map->size);
214
215 if ( !AddExtensionMap(fs, (*e)->extension_map) ) {
216 // bad - we must free this map and fail - otherwise data can not be read any more
217 free((*e)->extension_map);
218 free(*e);
219 *e = NULL;
220 return NULL;
221 }
222
223 (*e)->info.sysid = 0;
224 FlushInfoExporter(fs, &((*e)->info));
225
226 if ( fs->sa_family == AF_INET ) {
227 uint32_t _ip = htonl(fs->ip.V4);
228 inet_ntop(AF_INET, &_ip, ipstr, sizeof(ipstr));
229 } else if ( fs->sa_family == AF_INET6 ) {
230 uint64_t _ip[2];
231 _ip[0] = htonll(fs->ip.V6[0]);
232 _ip[1] = htonll(fs->ip.V6[1]);
233 inet_ntop(AF_INET6, &_ip, ipstr, sizeof(ipstr));
234 } else {
235 strncpy(ipstr, "<unknown>", IP_STRING_LEN);
236 }
237
238 dbg_printf("New Exporter: v1 SysID: %u, Extension ID: %i, IP: %s, \n",
239 (*e)->info.sysid, (*e)->extension_map->map_id, ipstr);
240 LogError("Process_v1: SysID: %u, New exporter: IP: %s\n", (*e)->info.sysid, ipstr);
241
242 return (*e);
243
244 } // End of GetExporter
245
Process_v1(void * in_buff,ssize_t in_buff_cnt,FlowSource_t * fs)246 void Process_v1(void *in_buff, ssize_t in_buff_cnt, FlowSource_t *fs) {
247 netflow_v1_header_t *v1_header;
248 netflow_v1_record_t *v1_record;
249 exporter_v1_t *exporter;
250 extension_map_t *extension_map;
251 common_record_t *common_record;
252 uint64_t start_time, end_time, boot_time;
253 uint32_t First, Last;
254 uint16_t count;
255 uint8_t flags;
256 int i, done, flow_record_length;
257 ssize_t size_left;
258 char *string;
259
260 // map v1 data structure to input buffer
261 v1_header = (netflow_v1_header_t *)in_buff;
262
263 exporter = GetExporter(fs, v1_header);
264 if ( !exporter ) {
265 LogError("Process_v1: Exporter NULL: Abort v1 record processing");
266 return;
267 }
268 flags = 0;
269
270 exporter->packets++;
271
272 extension_map = exporter->extension_map;
273 flow_record_length = NETFLOW_V1_RECORD_LENGTH;
274
275 // this many data to process
276 size_left = in_buff_cnt;
277
278 common_record = fs->nffile->buff_ptr;
279 done = 0;
280 while ( !done ) {
281 v1_block_t *v1_block;
282
283 /* Process header */
284
285 // count check
286 count = ntohs(v1_header->count);
287 if ( count > NETFLOW_V1_MAX_RECORDS ) {
288 LogError("Process_v1: Unexpected record count in header: %i. Abort v1 record processing", count);
289 fs->nffile->buff_ptr = (void *)common_record;
290 return;
291 }
292
293 // input buffer size check for all expected records
294 if ( size_left < ( NETFLOW_V1_HEADER_LENGTH + count * flow_record_length) ) {
295 LogError("Process_v1: Not enough data to process v1 record. Abort v1 record processing");
296 fs->nffile->buff_ptr = (void *)common_record;
297 return;
298 }
299
300 // output buffer size check for all expected records
301 if ( !CheckBufferSpace(fs->nffile, count * v1_output_record_size) ) {
302 // fishy! - should never happen. maybe disk full?
303 LogError("Process_v1: output buffer size error. Abort v1 record processing");
304 return;
305 }
306
307 // map output record to memory buffer
308 common_record = (common_record_t *)fs->nffile->buff_ptr;
309 v1_block = (v1_block_t *)common_record->data;
310
311 v1_header->SysUptime = ntohl(v1_header->SysUptime);
312 v1_header->unix_secs = ntohl(v1_header->unix_secs);
313 v1_header->unix_nsecs = ntohl(v1_header->unix_nsecs);
314
315 /* calculate boot time in msec */
316 boot_time = ((uint64_t)(v1_header->unix_secs)*1000 +
317 ((uint64_t)(v1_header->unix_nsecs) / 1000000) ) - (uint64_t)(v1_header->SysUptime);
318
319 // process all records
320 v1_record = (netflow_v1_record_t *)((pointer_addr_t)v1_header + NETFLOW_V1_HEADER_LENGTH);
321
322 /* loop over each records associated with this header */
323 for (i = 0; i < count; i++) {
324 pointer_addr_t bsize;
325 void *data_ptr;
326 uint8_t *s1, *s2;
327 int j, id;
328 // header data
329 common_record->flags = flags;
330 common_record->type = CommonRecordType;
331 common_record->exporter_sysid = exporter->info.sysid;
332 common_record->ext_map = extension_map->map_id;
333 common_record->size = v1_output_record_size;
334
335 // v1 common fields
336 common_record->srcport = ntohs(v1_record->srcport);
337 common_record->dstport = ntohs(v1_record->dstport);
338 common_record->tcp_flags = v1_record->tcp_flags;
339 common_record->prot = v1_record->prot;
340 common_record->tos = v1_record->tos;
341 common_record->fwd_status = 0;
342 common_record->reserved = 0;
343
344 // v1 typed data as fixed struct v1_block
345 v1_block->srcaddr = ntohl(v1_record->srcaddr);
346 v1_block->dstaddr = ntohl(v1_record->dstaddr);
347 v1_block->dPkts = ntohl(v1_record->dPkts);
348 v1_block->dOctets = ntohl(v1_record->dOctets);
349
350 // process optional extensions
351 data_ptr = (void *)v1_block->data;
352 j = 0;
353 while ( (id = extension_map->ex_id[j]) != 0 ) {
354 switch (id) {
355 case EX_IO_SNMP_2: { // 2 byte input/output interface index
356 tpl_ext_4_t *tpl = (tpl_ext_4_t *)data_ptr;
357 tpl->input = ntohs(v1_record->input);
358 tpl->output = ntohs(v1_record->output);
359 data_ptr = (void *)tpl->data;
360 } break;
361 case EX_NEXT_HOP_v4: { // IPv4 next hop
362 tpl_ext_9_t *tpl = (tpl_ext_9_t *)data_ptr;
363 tpl->nexthop = ntohl(v1_record->nexthop);
364 data_ptr = (void *)tpl->data;
365 } break;
366 case EX_ROUTER_IP_v4: { // IPv4 router address
367 tpl_ext_23_t *tpl = (tpl_ext_23_t *)data_ptr;
368 tpl->router_ip = fs->ip.V4;
369 data_ptr = (void *)tpl->data;
370 ClearFlag(common_record->flags, FLAG_IPV6_EXP);
371 } break;
372 case EX_RECEIVED: {
373 tpl_ext_27_t *tpl = (tpl_ext_27_t *)data_ptr;
374 tpl->received = (uint64_t)((uint64_t)fs->received.tv_sec * 1000LL) + (uint64_t)((uint64_t)fs->received.tv_usec / 1000LL);
375 data_ptr = (void *)tpl->data;
376 } break;
377
378 default:
379 // this should never happen, as v1 has no other extensions
380 LogError("Process_v1: Unexpected extension %i for v1 record. Skip extension", id);
381 }
382 j++;
383 }
384
385 // Time issues
386 First = ntohl(v1_record->First);
387 Last = ntohl(v1_record->Last);
388
389 if ( First > Last ) {
390 /* First in msec, in case of msec overflow, between start and end */
391 start_time = boot_time - 0x100000000LL + (uint64_t)First;
392 } else {
393 start_time = boot_time + (uint64_t)First;
394 }
395
396 /* end time in msecs */
397 end_time = (uint64_t)Last + boot_time;
398
399 // if overflow happened after flow ended but before got exported
400 if ( Last > v1_header->SysUptime ) {
401 start_time -= 0x100000000LL;
402 end_time -= 0x100000000LL;
403 }
404
405 common_record->first = start_time/1000;
406 common_record->msec_first = start_time - common_record->first*1000;
407
408 common_record->last = end_time/1000;
409 common_record->msec_last = end_time - common_record->last*1000;
410
411 // update first_seen, last_seen
412 if ( start_time < fs->first_seen )
413 fs->first_seen = start_time;
414 if ( end_time > fs->last_seen )
415 fs->last_seen = end_time;
416
417
418 // Update stats
419 switch (common_record->prot) {
420 case IPPROTO_ICMP:
421 fs->nffile->stat_record->numflows_icmp++;
422 fs->nffile->stat_record->numpackets_icmp += v1_block->dPkts;
423 fs->nffile->stat_record->numbytes_icmp += v1_block->dOctets;
424 // fix odd CISCO behaviour for ICMP port/type in src port
425 if ( common_record->srcport != 0 ) {
426 s1 = (uint8_t *)&(common_record->srcport);
427 s2 = (uint8_t *)&(common_record->dstport);
428 s2[0] = s1[1];
429 s2[1] = s1[0];
430 common_record->srcport = 0;
431 }
432 break;
433 case IPPROTO_TCP:
434 fs->nffile->stat_record->numflows_tcp++;
435 fs->nffile->stat_record->numpackets_tcp += v1_block->dPkts;
436 fs->nffile->stat_record->numbytes_tcp += v1_block->dOctets;
437 break;
438 case IPPROTO_UDP:
439 fs->nffile->stat_record->numflows_udp++;
440 fs->nffile->stat_record->numpackets_udp += v1_block->dPkts;
441 fs->nffile->stat_record->numbytes_udp += v1_block->dOctets;
442 break;
443 default:
444 fs->nffile->stat_record->numflows_other++;
445 fs->nffile->stat_record->numpackets_other += v1_block->dPkts;
446 fs->nffile->stat_record->numbytes_other += v1_block->dOctets;
447 }
448 exporter->flows++;
449 fs->nffile->stat_record->numflows++;
450 fs->nffile->stat_record->numpackets += v1_block->dPkts;
451 fs->nffile->stat_record->numbytes += v1_block->dOctets;
452
453 if ( verbose ) {
454 master_record_t master_record;
455 memset((void *)&master_record, 0, sizeof(master_record_t));
456 ExpandRecord_v2((common_record_t *)common_record, &v1_extension_info, &(exporter->info), &master_record);
457 flow_record_to_raw(&master_record, &string, 0);
458 printf("%s\n", string);
459 }
460
461 // advance to next input flow record
462 v1_record = (netflow_v1_record_t *)((pointer_addr_t)v1_record + flow_record_length);
463
464 if ( ((pointer_addr_t)data_ptr - (pointer_addr_t)common_record) != v1_output_record_size ) {
465 printf("Panic size check: ptr diff: %llu, record size: %u\n", (unsigned long long)((pointer_addr_t)data_ptr - (pointer_addr_t)common_record), v1_output_record_size );
466 abort();
467 }
468 // advance to next output record
469 common_record = (common_record_t *)data_ptr;
470 v1_block = (v1_block_t *)common_record->data;
471
472 // buffer size sanity check - should never happen, but check it anyway
473 bsize = (pointer_addr_t)common_record - (pointer_addr_t)fs->nffile->block_header - sizeof(data_block_header_t);
474 if ( bsize > BUFFSIZE ) {
475 LogError("### Software error ###: %s line %d", __FILE__, __LINE__);
476 LogError("Process_v1: Output buffer overflow! Flush buffer and skip records.");
477 LogError("Buffer size: size: %u, bsize: %llu > %u", fs->nffile->block_header->size, (unsigned long long)bsize, BUFFSIZE);
478 // reset buffer
479 fs->nffile->block_header->size = 0;
480 fs->nffile->block_header->NumRecords = 0;
481 fs->nffile->buff_ptr = (void *)((pointer_addr_t)fs->nffile->block_header + sizeof(data_block_header_t) );
482 return;
483 }
484
485 } // End of foreach v1 record
486
487 // update file record size ( -> output buffer size )
488 fs->nffile->block_header->NumRecords += count;
489 fs->nffile->block_header->size += count * v1_output_record_size;
490 fs->nffile->buff_ptr = (void *)common_record;
491
492 // still to go for this many input bytes
493 size_left -= NETFLOW_V1_HEADER_LENGTH + count * flow_record_length;
494
495 // next header
496 v1_header = (netflow_v1_header_t *)v1_record;
497
498 // should never be < 0
499 done = size_left <= 0;
500
501 } // End of while !done
502
503 return;
504
505 } /* End of Process_v1 */
506
507