1 /*
2 * Copyright (c) 2009-2020, Peter Haag
3 * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * * Neither the name of the author nor the names of its contributors may be
15 * used to endorse or promote products derived from this software without
16 * specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 /*
33 * nfreader is sample code for reading nfdump binary files.
34 * It accepts the standard nfdump file select options -r, -M and -R
35 * Therefore it allows you to loop over multiple files and process the netflow record.
36 *
37 * Insert your code in the process_data function after the call to ExpandRecord
38 * To build the binary: first compile nfdump as usual.
39 * Then compile nfreader:
40 *
41 * make nfreader
42 *
43 * This compiles this code and links the required nfdump files
44 * If you do it by hand:
45 *
46 * gcc -I.. -o nfreader nfreader.c -lnfdump
47 *
48 */
49
50 #include "config.h"
51
52 #include <stdio.h>
53 #include <unistd.h>
54 #include <stdlib.h>
55 #include <stdarg.h>
56 #include <errno.h>
57 #include <time.h>
58 #include <string.h>
59 #include <ctype.h>
60 #include <sys/types.h>
61 #include <sys/socket.h>
62 #include <netinet/in.h>
63 #include <arpa/inet.h>
64 #include <sys/stat.h>
65 #include <fcntl.h>
66
67 #ifdef HAVE_STDINT_H
68 #include <stdint.h>
69 #endif
70
71 #include "util.h"
72 #include "nfdump.h"
73 #include "nffile.h"
74 #include "nfx.h"
75 #include "bookkeeper.h"
76 #include "collector.h"
77 #include "exporter.h"
78 #include "flist.h"
79
80 // module limited globals
81 extension_map_list_t *extension_map_list;
82
83 extern exporter_t **exporter_list;
84
85 /* Function Prototypes */
86 static void usage(char *name);
87
88 static void print_record(void *record, char *s );
89
90 static void process_data(void);
91
92 /* Functions */
93
94 #include "nffile_inline.c"
95
usage(char * name)96 static void usage(char *name) {
97 printf("usage %s [options] \n"
98 "-h\t\tthis text you see right here\n"
99 "-r\t\tread input from file\n"
100 "-M <expr>\tRead input from multiple directories.\n"
101 "-R <expr>\tRead input from sequence of files.\n"
102 , name);
103 } /* usage */
104
print_record(void * record,char * s)105 static void print_record(void *record, char *s ) {
106 char as[40], ds[40], datestr1[64], datestr2[64];
107 time_t when;
108 struct tm *ts;
109 master_record_t *r = (master_record_t *)record;
110
111 if ( (r->flags & FLAG_IPV6_ADDR ) != 0 ) { // IPv6
112 r->V6.srcaddr[0] = htonll(r->V6.srcaddr[0]);
113 r->V6.srcaddr[1] = htonll(r->V6.srcaddr[1]);
114 r->V6.dstaddr[0] = htonll(r->V6.dstaddr[0]);
115 r->V6.dstaddr[1] = htonll(r->V6.dstaddr[1]);
116 inet_ntop(AF_INET6, r->V6.srcaddr, as, sizeof(as));
117 inet_ntop(AF_INET6, r->V6.dstaddr, ds, sizeof(ds));
118 } else { // IPv4
119 r->V4.srcaddr = htonl(r->V4.srcaddr);
120 r->V4.dstaddr = htonl(r->V4.dstaddr);
121 inet_ntop(AF_INET, &r->V4.srcaddr, as, sizeof(as));
122 inet_ntop(AF_INET, &r->V4.dstaddr, ds, sizeof(ds));
123 }
124 as[40-1] = 0;
125 ds[40-1] = 0;
126
127 when = r->first;
128 ts = localtime(&when);
129 strftime(datestr1, 63, "%Y-%m-%d %H:%M:%S", ts);
130
131 when = r->last;
132 ts = localtime(&when);
133 strftime(datestr2, 63, "%Y-%m-%d %H:%M:%S", ts);
134
135 snprintf(s, 1023, "\n"
136 "Flow Record: \n"
137 " srcaddr = %16s\n"
138 " dstaddr = %16s\n"
139 " first = %10u [%s]\n"
140 " last = %10u [%s]\n"
141 " msec_first = %5u\n"
142 " msec_last = %5u\n"
143 " prot = %3u\n"
144 " srcport = %5u\n"
145 " dstport = %5u\n"
146 " dPkts = %10llu\n"
147 " dOctets = %10llu\n"
148 ,
149 as, ds, r->first, datestr1, r->last, datestr2, r->msec_first, r->msec_last,
150 r->prot, r->srcport, r->dstport,
151 (unsigned long long)r->dPkts, (unsigned long long)r->dOctets);
152
153 s[1024-1] = 0;
154
155 } // End of print_record
156
157
process_data(void)158 static void process_data(void) {
159 master_record_t master_record;
160 common_record_t *flow_record;
161 nffile_t *nffile;
162 int i, done, ret;
163
164 // Get the first file handle
165 nffile = GetNextFile(NULL, 0, 0);
166 if ( !nffile ) {
167 LogError("GetNextFile() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
168 return;
169 }
170 if ( nffile == EMPTY_LIST ) {
171 LogError("Empty file list. No files to process\n");
172 return;
173 }
174
175 done = 0;
176 while ( !done ) {
177 // get next data block from file
178 ret = ReadBlock(nffile);
179
180 switch (ret) {
181 case NF_CORRUPT:
182 case NF_ERROR:
183 if ( ret == NF_CORRUPT )
184 fprintf(stderr, "Skip corrupt data file '%s'\n",GetCurrentFilename());
185 else
186 fprintf(stderr, "Read error in file '%s': %s\n",GetCurrentFilename(), strerror(errno) );
187 // fall through - get next file in chain
188 case NF_EOF: {
189 nffile_t *next = GetNextFile(nffile, 0, 0);
190 if ( next == EMPTY_LIST ) {
191 done = 1;
192 }
193 if ( next == NULL ) {
194 done = 1;
195 LogError("Unexpected end of file list\n");
196 }
197 // else continue with next file
198 continue;
199
200 } break; // not really needed
201 }
202
203 if ( nffile->block_header->id != DATA_BLOCK_TYPE_2 ) {
204 fprintf(stderr, "Can't process block type %u. Skip block.\n", nffile->block_header->id);
205 continue;
206 }
207
208 flow_record = nffile->buff_ptr;
209 uint32_t sumSize = 0;
210 for ( i=0; i < nffile->block_header->NumRecords; i++ ) {
211 char string[1024];
212 if ( (sumSize + flow_record->size) > ret || (flow_record->size < sizeof(record_header_t)) ) {
213 LogError("Corrupt data file. Inconsistent block size in %s line %d\n", __FILE__, __LINE__);
214 exit(255);
215 }
216 sumSize += flow_record->size;
217
218 switch ( flow_record->type ) {
219 case CommonRecordType: {
220 uint32_t map_id = flow_record->ext_map;
221 exporter_t *exp_info = exporter_list[flow_record->exporter_sysid];
222 if ( extension_map_list->slot[map_id] == NULL ) {
223 snprintf(string, 1024, "Corrupt data file! No such extension map id: %u. Skip record", flow_record->ext_map );
224 string[1023] = '\0';
225 } else {
226 ExpandRecord_v2( flow_record, extension_map_list->slot[flow_record->ext_map],
227 exp_info ? &(exp_info->info) : NULL, &master_record);
228
229 // update number of flows matching a given map
230 extension_map_list->slot[map_id]->ref_count++;
231
232 /*
233 * insert hier your calls to your processing routine
234 * master_record now contains the next flow record as specified in nffile.c
235 * for example you can print each record:
236 *
237 */
238 print_record(&master_record, string);
239 printf("%s\n", string);
240 }
241
242 } break;
243 case ExtensionMapType: {
244 extension_map_t *map = (extension_map_t *)flow_record;
245
246 int ret = Insert_Extension_Map(extension_map_list, map);
247 switch (ret) {
248 case 0:
249 // map already known and flushed
250 break;
251 case 1:
252 // new map
253 break;
254 default:
255 LogError("Corrupt data file. Unable to decode at %s line %d\n", __FILE__, __LINE__);
256 exit(255);
257 }
258
259 } break;
260 case ExporterInfoRecordType:
261 case ExporterStatRecordType:
262 case SamplerInfoRecordype:
263 // Silently skip exporter records
264 break;
265 default: {
266 fprintf(stderr, "Skip unknown record type %i\n", flow_record->type);
267 }
268 }
269
270 // Advance pointer by number of bytes for netflow record
271 flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
272
273 } // for all records
274
275 } // while
276
277 CloseFile(nffile);
278 DisposeFile(nffile);
279
280 PackExtensionMapList(extension_map_list);
281
282 } // End of process_data
283
284
main(int argc,char ** argv)285 int main( int argc, char **argv ) {
286 char *rfile, *Rfile, *Mdirs;
287 int c;
288
289 rfile = Rfile = Mdirs = NULL;
290 while ((c = getopt(argc, argv, "L:r:M:R:")) != EOF) {
291 switch (c) {
292 case 'h':
293 usage(argv[0]);
294 exit(0);
295 break;
296 break;
297 case 'L':
298 if ( !InitLog(0, "argv[0]", optarg, 0) )
299 exit(255);
300 break;
301 case 'r':
302 rfile = optarg;
303 if ( strcmp(rfile, "-") == 0 )
304 rfile = NULL;
305 break;
306 case 'M':
307 Mdirs = optarg;
308 break;
309 case 'R':
310 Rfile = optarg;
311 break;
312 default:
313 usage(argv[0]);
314 exit(0);
315 }
316 }
317
318 if ( rfile && Rfile ) {
319 fprintf(stderr, "-r and -R are mutually exclusive. Please specify either -r or -R\n");
320 exit(255);
321 }
322 if ( Mdirs && !(rfile || Rfile) ) {
323 fprintf(stderr, "-M needs either -r or -R to specify the file or file list. Add '-R .' for all files in the directories.\n");
324 exit(255);
325 }
326
327 extension_map_list = InitExtensionMaps(NEEDS_EXTENSION_LIST);
328 if ( !InitExporterList() ) {
329 exit(255);
330 }
331
332 SetupInputFileSequence(Mdirs, rfile, Rfile);
333
334 process_data();
335
336 FreeExtensionMaps(extension_map_list);
337
338 return 0;
339 }
340