1 /*
2 * Copyright (c) 2009-2020, Peter Haag
3 * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * * Neither the name of the author nor the names of its contributors may be
15 * used to endorse or promote products derived from this software without
16 * specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #include "config.h"
33
34 #include <stdio.h>
35 #include <unistd.h>
36 #include <stdlib.h>
37 #include <stdarg.h>
38 #include <errno.h>
39 #include <time.h>
40 #include <string.h>
41 #include <ctype.h>
42 #include <sys/types.h>
43 #include <sys/socket.h>
44 #include <sys/param.h>
45 #include <netinet/in.h>
46 #include <arpa/inet.h>
47 #include <sys/stat.h>
48 #include <fcntl.h>
49
50 #ifdef HAVE_STDINT_H
51 #include <stdint.h>
52 #endif
53
54 #include "util.h"
55 #include "nfdump.h"
56 #include "nffile.h"
57 #include "nfx.h"
58 #include "exporter.h"
59 #include "flist.h"
60 #include "panonymizer.h"
61
62 // module limited globals
63 extension_map_list_t *extension_map_list;
64
65 /* Function Prototypes */
66 static void usage(char *name);
67
68 static inline void AnonRecord(master_record_t *master_record);
69
70 static void process_data(void *wfile);
71
72 /* Functions */
73
74 #define NEED_PACKRECORD 1
75 #include "nffile_inline.c"
76 #undef NEED_PACKRECORD
77
usage(char * name)78 static void usage(char *name) {
79 printf("usage %s [options] \n"
80 "-h\t\tthis text you see right here\n"
81 "-K <key>\tAnonymize IP addresses using CryptoPAn with key <key>.\n"
82 "-r\t\tread input from file\n"
83 "-M <expr>\tRead input from multiple directories.\n"
84 "-R <expr>\tRead input from sequence of files.\n"
85 "-w <file>\tName of output file. Defaults to input file.\n"
86 , name);
87 } /* usage */
88
AnonRecord(master_record_t * master_record)89 static inline void AnonRecord(master_record_t *master_record) {
90 extension_map_t *extension_map = master_record->map_ref;
91 int i;
92
93 // Required extension 1 - IP addresses
94 if ( (master_record->flags & FLAG_IPV6_ADDR) != 0 ) { // IPv6
95 // IPv6
96 uint64_t anon_ip[2];
97 anonymize_v6(master_record->V6.srcaddr, anon_ip);
98 master_record->V6.srcaddr[0] = anon_ip[0];
99 master_record->V6.srcaddr[1] = anon_ip[1];
100
101 anonymize_v6(master_record->V6.dstaddr, anon_ip);
102 master_record->V6.dstaddr[0] = anon_ip[0];
103 master_record->V6.dstaddr[1] = anon_ip[1];
104
105 } else {
106 // IPv4
107 master_record->V4.srcaddr = anonymize(master_record->V4.srcaddr);
108 master_record->V4.dstaddr = anonymize(master_record->V4.dstaddr);
109 }
110
111 // Process optional extensions
112 i=0;
113 while ( extension_map->ex_id[i] ) {
114 switch (extension_map->ex_id[i++]) {
115 case EX_AS_2: // srcas/dstas 2 byte
116 master_record->srcas = 0;
117 master_record->dstas = 0;
118 break;
119 case EX_AS_4: // srcas/dstas 4 byte
120 master_record->srcas = 0;
121 master_record->dstas = 0;
122 break;
123 case EX_NEXT_HOP_v4:
124 master_record->ip_nexthop.V4 = anonymize(master_record->ip_nexthop.V4);
125 break;
126 case EX_NEXT_HOP_v6: {
127 uint64_t anon_ip[2];
128 anonymize_v6(master_record->ip_nexthop.V6, anon_ip);
129 master_record->ip_nexthop.V6[0] = anon_ip[0];
130 master_record->ip_nexthop.V6[1] = anon_ip[1];
131 } break;
132 case EX_NEXT_HOP_BGP_v4:
133 master_record->bgp_nexthop.V4 = anonymize(master_record->bgp_nexthop.V4);
134 break;
135 case EX_NEXT_HOP_BGP_v6: {
136 uint64_t anon_ip[2];
137 anonymize_v6(master_record->bgp_nexthop.V6, anon_ip);
138 master_record->bgp_nexthop.V6[0] = anon_ip[0];
139 master_record->bgp_nexthop.V6[1] = anon_ip[1];
140 } break;
141 case EX_ROUTER_IP_v4:
142 master_record->ip_router.V4 = anonymize(master_record->ip_router.V4);
143 break;
144 case EX_ROUTER_IP_v6: {
145 uint64_t anon_ip[2];
146 anonymize_v6(master_record->ip_router.V6, anon_ip);
147 master_record->ip_router.V6[0] = anon_ip[0];
148 master_record->ip_router.V6[1] = anon_ip[1];
149 } break;
150 #ifdef NSEL
151 case EX_NSEL_XLATE_IP_v4:
152 master_record->xlate_src_ip.V4 = anonymize(master_record->xlate_src_ip.V4);
153 master_record->xlate_dst_ip.V4 = anonymize(master_record->xlate_dst_ip.V4);
154 break;
155 case EX_NSEL_XLATE_IP_v6: {
156 uint64_t anon_ip[2];
157 anonymize_v6(master_record->xlate_src_ip.V6, anon_ip);
158 master_record->xlate_src_ip.V6[0] = anon_ip[0];
159 master_record->xlate_src_ip.V6[1] = anon_ip[1];
160 anonymize_v6(master_record->xlate_dst_ip.V6, anon_ip);
161 master_record->xlate_dst_ip.V6[0] = anon_ip[0];
162 master_record->xlate_dst_ip.V6[1] = anon_ip[1];
163 } break;
164 #endif
165 }
166 }
167
168 } // End of AnonRecord
169
170
process_data(void * wfile)171 static void process_data(void *wfile) {
172 master_record_t master_record;
173 common_record_t *flow_record;
174 nffile_t *nffile_r;
175 nffile_t *nffile_w;
176 int i, done, ret, cnt, verbose;
177 char outfile[MAXPATHLEN], *cfile;
178
179 setbuf(stderr, NULL);
180 cnt = 1;
181 verbose = 1;
182
183 // Get the first file handle
184 nffile_r = GetNextFile(NULL, 0, 0);
185 if ( !nffile_r ) {
186 LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
187 return;
188 }
189 if ( nffile_r == EMPTY_LIST ) {
190 LogError("Empty file list. No files to process\n");
191 return;
192 }
193
194 cfile = GetCurrentFilename();
195 if ( !cfile ) {
196 if ( nffile_r->fd == 0 ) { // stdin
197 outfile[0] = '-';
198 outfile[1] = '\0';
199 verbose = 0;
200 } else {
201 LogError("(NULL) input file name error in %s line %d\n", __FILE__, __LINE__);
202 return;
203 }
204 } else {
205 // prepare output file
206 snprintf(outfile,MAXPATHLEN-1, "%s-tmp", cfile);
207 outfile[MAXPATHLEN-1] = '\0';
208 if ( verbose )
209 fprintf(stderr, " %i Processing %s\r", cnt++, cfile);
210 }
211
212 if ( wfile )
213 nffile_w = OpenNewFile(wfile, NULL, FILE_COMPRESSION(nffile_r), 1, NULL);
214 else
215 nffile_w = OpenNewFile(outfile, NULL, FILE_COMPRESSION(nffile_r), 1, NULL);
216
217 if ( !nffile_w ) {
218 if ( nffile_r ) {
219 CloseFile(nffile_r);
220 DisposeFile(nffile_r);
221 }
222 return;
223 }
224
225 memcpy((void *)nffile_w->stat_record, (void *)nffile_r->stat_record, sizeof(stat_record_t));
226
227 done = 0;
228 while ( !done ) {
229 // get next data block from file
230 ret = ReadBlock(nffile_r);
231
232 switch (ret) {
233 case NF_CORRUPT:
234 case NF_ERROR:
235 if ( ret == NF_CORRUPT )
236 LogError("Skip corrupt data file '%s'\n",GetCurrentFilename());
237 else
238 LogError("Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) );
239 // fall through - get next file in chain
240 case NF_EOF: {
241 nffile_t *next;
242 if ( nffile_w->block_header->NumRecords ) {
243 if ( WriteBlock(nffile_w) <= 0 ) {
244 LogError("Failed to write output buffer to disk: '%s'" , strerror(errno));
245 }
246 }
247 if ( wfile == NULL ) {
248 CloseUpdateFile(nffile_w, nffile_r->file_header->ident);
249 if ( rename(outfile, cfile) < 0 ) {
250 LogError("\nrename() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
251 LogError("Abort processing.\n");
252 return;
253 }
254 }
255
256 next = GetNextFile(nffile_r, 0, 0);
257 if ( next == EMPTY_LIST ) {
258 done = 1;
259 continue;
260 }
261 if ( next == NULL ) {
262 LogError("Unexpected end of file list\n");
263 done = 1;
264 continue;
265 }
266
267 cfile = GetCurrentFilename();
268 if ( !cfile ) {
269 LogError("(NULL) input file name error in %s line %d\n", __FILE__, __LINE__);
270 return;
271 }
272 LogError(" %i Processing %s\r", cnt++, cfile);
273
274 if ( wfile == NULL ) {
275 snprintf(outfile,MAXPATHLEN-1, "%s-tmp", cfile);
276 outfile[MAXPATHLEN-1] = '\0';
277
278 nffile_w = OpenNewFile(outfile, nffile_w, FILE_COMPRESSION(nffile_r), 1, NULL);
279 if ( !nffile_w ) {
280 if ( nffile_r ) {
281 CloseFile(nffile_r);
282 DisposeFile(nffile_r);
283 }
284 return;
285 }
286 memcpy((void *)nffile_w->stat_record, (void *)nffile_r->stat_record, sizeof(stat_record_t));
287 } else {
288 SumStatRecords(nffile_w->stat_record, nffile_r->stat_record);
289 }
290
291 // continue with next file
292 continue;
293
294 } break; // not really needed
295 }
296
297 if ( nffile_r->block_header->id != DATA_BLOCK_TYPE_2 ) {
298 fprintf(stderr, "Can't process block type %u. Skip block.\n", nffile_r->block_header->id);
299 continue;
300 }
301
302 flow_record = nffile_r->buff_ptr;
303 uint32_t sumSize = 0;
304 for ( i=0; i < nffile_r->block_header->NumRecords; i++ ) {
305 if ( (sumSize + flow_record->size) > ret ) {
306 LogError("Corrupt data file. Inconsistent block size in %s line %d\n", __FILE__, __LINE__);
307 exit(255);
308 }
309 sumSize += flow_record->size;
310
311 switch ( flow_record->type ) {
312 case CommonRecordType: {
313 uint32_t map_id = flow_record->ext_map;
314 if ( extension_map_list->slot[map_id] == NULL ) {
315 LogError("Corrupt data file! No such extension map id: %u. Skip record", flow_record->ext_map );
316 } else {
317 ExpandRecord_v2( flow_record, extension_map_list->slot[flow_record->ext_map], NULL, &master_record);
318
319 // update number of flows matching a given map
320 extension_map_list->slot[map_id]->ref_count++;
321
322 AnonRecord(&master_record);
323 PackRecord(&master_record, nffile_w);
324 }
325
326 } break;
327 case ExtensionMapType: {
328 extension_map_t *map = (extension_map_t *)flow_record;
329
330 int ret = Insert_Extension_Map(extension_map_list, map);
331 switch (ret) {
332 case 0:
333 break; // map already known and flushed
334 case 1:
335 AppendToBuffer(nffile_w, (void *)map, map->size);
336 break;
337 default:
338 LogError("Corrupt data file. Unable to decode at %s line %d\n", __FILE__, __LINE__);
339 exit(255);
340 }
341 } break;
342 case LegacyRecordType1:
343 case LegacyRecordType2:
344 case ExporterInfoRecordType:
345 case ExporterStatRecordType:
346 case SamplerInfoRecordype:
347 // Silently skip exporter/sampler records
348 break;
349
350 default: {
351 fprintf(stderr, "Skip unknown record type %i\n", flow_record->type);
352 }
353 }
354 // Advance pointer by number of bytes for netflow record
355 flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
356
357 } // for all records
358
359 } // while
360
361 PackExtensionMapList(extension_map_list);
362 if ( wfile != NULL )
363 CloseUpdateFile(nffile_w, nffile_r->file_header->ident);
364
365 if ( nffile_r ) {
366 CloseFile(nffile_r);
367 DisposeFile(nffile_r);
368 }
369
370 DisposeFile(nffile_w);
371
372 LogError("\n");
373 LogError("Processed %i files.\n", --cnt);
374
375 } // End of process_data
376
377
main(int argc,char ** argv)378 int main( int argc, char **argv ) {
379 char *rfile, *Rfile, *wfile, *Mdirs;
380 int c;
381 char CryptoPAnKey[32];
382
383 rfile = Rfile = Mdirs = wfile = NULL;
384 while ((c = getopt(argc, argv, "K:L:r:M:R:w:")) != EOF) {
385 switch (c) {
386 case 'h':
387 usage(argv[0]);
388 exit(0);
389 break;
390 break;
391 case 'K':
392 if ( !ParseCryptoPAnKey(optarg, CryptoPAnKey) ) {
393 fprintf(stderr, "Invalid key '%s' for CryptoPAn!\n", optarg);
394 exit(255);
395 }
396 PAnonymizer_Init((uint8_t *)CryptoPAnKey);
397 break;
398 case 'L':
399 if ( !InitLog(0, "argv[0]", optarg, 0) )
400 exit(255);
401 break;
402 case 'r':
403 rfile = optarg;
404 if ( strcmp(rfile, "-") == 0 )
405 rfile = NULL;
406 break;
407 case 'M':
408 Mdirs = optarg;
409 break;
410 case 'R':
411 Rfile = optarg;
412 break;
413 case 'w':
414 wfile = optarg;
415 break;
416 default:
417 usage(argv[0]);
418 exit(0);
419 }
420 }
421
422 if ( rfile && Rfile ) {
423 fprintf(stderr, "-r and -R are mutually exclusive. Please specify either -r or -R\n");
424 exit(255);
425 }
426 if ( Mdirs && !(rfile || Rfile) ) {
427 fprintf(stderr, "-M needs either -r or -R to specify the file or file list. Add '-R .' for all files in the directories.\n");
428 exit(255);
429 }
430
431 extension_map_list = InitExtensionMaps(NEEDS_EXTENSION_LIST);
432
433 SetupInputFileSequence(Mdirs, rfile, Rfile);
434
435 process_data(wfile);
436
437 FreeExtensionMaps(extension_map_list);
438
439 return 0;
440 }
441