1 /*
2 * Copyright (c) 2009-2020, Peter Haag
3 * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * * Neither the name of the author nor the names of its contributors may be
15 * used to endorse or promote products derived from this software without
16 * specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #include "config.h"
33
34 #include <sys/types.h>
35 #include <sys/socket.h>
36 #include <netinet/in.h>
37 #include <arpa/inet.h>
38 #include <sys/uio.h>
39 #include <time.h>
40 #include <unistd.h>
41 #include <string.h>
42 #include <fcntl.h>
43 #include <sys/stat.h>
44 #include <sys/param.h>
45 #include <stdio.h>
46 #include <errno.h>
47 #include <unistd.h>
48 #include <stdlib.h>
49 #include <bzlib.h>
50
51 #ifdef HAVE_STDINT_H
52 #include <stdint.h>
53 #endif
54
55 #include "util.h"
56 #include "nfdump.h"
57 #include "minilzo.h"
58 #include "lz4.h"
59 #include "flist.h"
60 #include "nffile.h"
61 #include "nffileV2.h"
62
63 /* global vars */
64
65 // required for idet filter in nftree.c
66 char *CurrentIdent;
67
68
69 // LZO params
70 #define HEAP_ALLOC(var,size) \
71 lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ]
72
73 static HEAP_ALLOC(wrkmem,LZO1X_1_MEM_COMPRESS);
74 static int lzo_initialized = 0;
75 static int lz4_initialized = 0;
76 static int bz2_initialized = 0;
77
78 static int LZO_initialize(void);
79
80 static int LZ4_initialize(void);
81
82 static int BZ2_initialize(void);
83
84 static void BZ2_prep_stream (bz_stream*);
85
86 static int OpenRaw(char *filename, stat_record_t *stat_record, int *compressed);
87
88 extern char *nf_error;
89
90 /* function prototypes */
91 static nffile_t *NewFile(void);
92
93 /* function definitions */
94
SumStatRecords(stat_record_t * s1,stat_record_t * s2)95 void SumStatRecords(stat_record_t *s1, stat_record_t *s2) {
96
97 s1->numflows += s2->numflows;
98 s1->numbytes += s2->numbytes;
99 s1->numpackets += s2->numpackets;
100 s1->numflows_tcp += s2->numflows_tcp;
101 s1->numflows_udp += s2->numflows_udp;
102 s1->numflows_icmp += s2->numflows_icmp;
103 s1->numflows_other += s2->numflows_other;
104 s1->numbytes_tcp += s2->numbytes_tcp;
105 s1->numbytes_udp += s2->numbytes_udp;
106 s1->numbytes_icmp += s2->numbytes_icmp;
107 s1->numbytes_other += s2->numbytes_other;
108 s1->numpackets_tcp += s2->numpackets_tcp;
109 s1->numpackets_udp += s2->numpackets_udp;
110 s1->numpackets_icmp += s2->numpackets_icmp;
111 s1->numpackets_other += s2->numpackets_other;
112 s1->sequence_failure += s2->sequence_failure;
113
114 if ( s2->first_seen < s1->first_seen ) {
115 s1->first_seen = s2->first_seen;
116 s1->msec_first = s2->msec_first;
117 }
118 if ( s2->first_seen == s1->first_seen &&
119 s2->msec_first < s1->msec_first )
120 s1->msec_first = s2->msec_first;
121
122 if ( s2->last_seen > s1->last_seen ) {
123 s1->last_seen = s2->last_seen;
124 s1->msec_last = s2->msec_last;
125 }
126 if ( s2->last_seen == s1->last_seen &&
127 s2->msec_last > s1->msec_last )
128 s1->msec_last = s2->msec_last;
129
130 } // End of SumStatRecords
131
132
LZO_initialize(void)133 static int LZO_initialize(void) {
134
135 if (lzo_init() != LZO_E_OK) {
136 // this usually indicates a compiler bug - try recompiling
137 // without optimizations, and enable `-DLZO_DEBUG' for diagnostics
138 LogError("Compression lzo_init() failed.\n");
139 return 0;
140 }
141 lzo_initialized = 1;
142
143 return 1;
144
145 } // End of LZO_initialize
146
LZ4_initialize(void)147 static int LZ4_initialize (void) {
148
149 int lz4_buff_size = LZ4_compressBound(BUFFSIZE + sizeof (data_block_header_t));
150 if ( lz4_buff_size > (2 * BUFFSIZE) ) {
151 LogError ("LZ4_compressBound() error in %s line %d: Buffer too small\n", __FILE__, __LINE__);
152 return 0;
153 }
154 lz4_initialized = 1;
155
156 return 1;
157
158 } // End of LZ4_initialize
159
BZ2_initialize(void)160 static int BZ2_initialize (void) {
161
162 bz2_initialized = 1;
163
164 return 1;
165
166 } // End of BZ2_initialize
167
BZ2_prep_stream(bz_stream * bs)168 static void BZ2_prep_stream (bz_stream* bs)
169 {
170 bs->bzalloc = NULL;
171 bs->bzfree = NULL;
172 bs->opaque = NULL;
173 } // End of BZ2_prep_stream
174
Compress_Block_LZO(nffile_t * nffile)175 static int Compress_Block_LZO(nffile_t *nffile) {
176 unsigned char __LZO_MMODEL *in;
177 unsigned char __LZO_MMODEL *out;
178 lzo_uint in_len;
179 lzo_uint out_len;
180 int r;
181
182 in = (unsigned char __LZO_MMODEL *)(nffile->buff_pool[0] + sizeof(data_block_header_t));
183 out = (unsigned char __LZO_MMODEL *)(nffile->buff_pool[1] + sizeof(data_block_header_t));
184 in_len = nffile->block_header->size;
185 r = lzo1x_1_compress(in,in_len,out,&out_len,wrkmem);
186
187 if (r != LZO_E_OK) {
188 LogError("Compress_Block_LZO() error compression failed in %s line %d: LZ4 : %d\n", __FILE__, __LINE__, r);
189 return -1;
190 }
191
192 // copy header
193 memcpy(nffile->buff_pool[1], nffile->buff_pool[0], sizeof(data_block_header_t));
194 ((data_block_header_t *)nffile->buff_pool[1])->size = out_len;
195
196 // swap buffers
197 void *_tmp = nffile->buff_pool[1];
198 nffile->buff_pool[1] = nffile->buff_pool[0];
199 nffile->buff_pool[0] = _tmp;
200
201 nffile->block_header = nffile->buff_pool[0];
202
203 return 1;
204
205 } // End of Compress_Block_LZO
206
Uncompress_Block_LZO(nffile_t * nffile)207 static int Uncompress_Block_LZO(nffile_t *nffile) {
208 unsigned char __LZO_MMODEL *in;
209 unsigned char __LZO_MMODEL *out;
210 lzo_uint in_len;
211 lzo_uint out_len;
212 int r;
213
214 in = (unsigned char __LZO_MMODEL *)(nffile->buff_pool[0] + sizeof(data_block_header_t));
215 out = (unsigned char __LZO_MMODEL *)(nffile->buff_pool[1] + sizeof(data_block_header_t));
216 in_len = nffile->block_header->size;
217 out_len = nffile->buff_size;
218
219 if ( in_len == 0 ) {
220 LogError("Uncompress_Block_LZO() header length error in %s line %d\n", __FILE__, __LINE__);
221 return -1;
222 }
223 r = lzo1x_decompress_safe(in,in_len,out,&out_len,NULL);
224 if (r != LZO_E_OK ) {
225 /* this should NEVER happen */
226 LogError("Uncompress_Block_LZO() error decompression failed in %s line %d: LZO error: %d\n", __FILE__, __LINE__, r);
227 return -1;
228 }
229
230 // copy header
231 memcpy(nffile->buff_pool[1], nffile->buff_pool[0], sizeof(data_block_header_t));
232 ((data_block_header_t *)nffile->buff_pool[1])->size = out_len;
233
234 // swap buffers
235 void *_tmp = nffile->buff_pool[1];
236 nffile->buff_pool[1] = nffile->buff_pool[0];
237 nffile->buff_pool[0] = _tmp;
238
239 nffile->block_header = nffile->buff_pool[0];
240 nffile->buff_ptr = nffile->buff_pool[0] + sizeof(data_block_header_t);
241
242 return 1;
243
244 } // End of Uncompress_Block_LZO
245
Compress_Block_LZ4(nffile_t * nffile)246 static int Compress_Block_LZ4(nffile_t *nffile) {
247
248 const char *in = (const char *)(nffile->buff_pool[0] + sizeof(data_block_header_t));
249 char *out = (char *)(nffile->buff_pool[1] + sizeof(data_block_header_t));
250 int in_len = nffile->block_header->size;
251
252 int out_len = LZ4_compress_default(in, out, in_len, nffile->buff_size);
253 if (out_len == 0 ) {
254 LogError("Compress_Block_LZ4() error compression aborted in %s line %d: LZ4 : buffer too small\n", __FILE__, __LINE__);
255 return -1;
256 }
257 if (out_len < 0 ) {
258 LogError("Compress_Block_LZ4() error compression failed in %s line %d: LZ4 : %d\n", __FILE__, __LINE__, out_len);
259 return -1;
260 }
261
262 // copy header
263 memcpy(nffile->buff_pool[1], nffile->buff_pool[0], sizeof(data_block_header_t));
264 ((data_block_header_t *)nffile->buff_pool[1])->size = out_len;
265
266 // swap buffers
267 void *_tmp = nffile->buff_pool[1];
268 nffile->buff_pool[1] = nffile->buff_pool[0];
269 nffile->buff_pool[0] = _tmp;
270
271 nffile->block_header = nffile->buff_pool[0];
272
273 return 1;
274
275 } // End of Compress_Block_LZ4
276
Uncompress_Block_LZ4(nffile_t * nffile)277 static int Uncompress_Block_LZ4(nffile_t *nffile) {
278
279 const char *in = (const char *)(nffile->buff_pool[0] + sizeof(data_block_header_t));
280 char *out = (char *)(nffile->buff_pool[1] + sizeof(data_block_header_t));
281 int in_len = nffile->block_header->size;
282
283 int out_len = LZ4_decompress_safe(in, out, in_len, nffile->buff_size);
284 if (out_len == 0 ) {
285 LogError("LZ4_decompress_safe() error compression aborted in %s line %d: LZ4 : buffer too small\n", __FILE__, __LINE__);
286 return -1;
287 }
288 if (out_len < 0 ) {
289 LogError("LZ4_decompress_safe() error compression failed in %s line %d: LZ4 : %d\n", __FILE__, __LINE__, out_len);
290 return -1;
291 }
292
293 // copy header
294 memcpy(nffile->buff_pool[1], nffile->buff_pool[0], sizeof(data_block_header_t));
295 ((data_block_header_t *)nffile->buff_pool[1])->size = out_len;
296
297 // swap buffers
298 void *_tmp = nffile->buff_pool[1];
299 nffile->buff_pool[1] = nffile->buff_pool[0];
300 nffile->buff_pool[0] = _tmp;
301
302 nffile->block_header = nffile->buff_pool[0];
303 nffile->buff_ptr = nffile->buff_pool[0] + sizeof(data_block_header_t);
304
305 return 1;
306
307 } // End of Uncompress_Block_LZ4
308
Compress_Block_BZ2(nffile_t * nffile)309 static int Compress_Block_BZ2(nffile_t *nffile) {
310 bz_stream bs;
311
312 BZ2_prep_stream (&bs);
313 BZ2_bzCompressInit (&bs, 9, 0, 0);
314
315 bs.next_in = (char*)(nffile->buff_pool[0] + sizeof(data_block_header_t));
316 bs.next_out = (char*)(nffile->buff_pool[1] + sizeof(data_block_header_t));
317 bs.avail_in = nffile->block_header->size;
318 bs.avail_out = nffile->buff_size;
319
320 for (;;) {
321 int r = BZ2_bzCompress (&bs, BZ_FINISH);
322 if (r == BZ_FINISH_OK) continue;
323 if (r != BZ_STREAM_END) {
324 LogError("Compress_Block_BZ2() error compression failed in %s line %d: LZ4 : %d\n", __FILE__, __LINE__, r);
325 return -1;
326 }
327 break;
328 }
329
330 // copy header
331 memcpy(nffile->buff_pool[1], nffile->buff_pool[0], sizeof(data_block_header_t));
332 ((data_block_header_t *)nffile->buff_pool[1])->size = bs.total_out_lo32;
333
334 // swap buffers
335 void *_tmp = nffile->buff_pool[1];
336 nffile->buff_pool[1] = nffile->buff_pool[0];
337 nffile->buff_pool[0] = _tmp;
338
339 nffile->block_header = nffile->buff_pool[0];
340
341 BZ2_bzCompressEnd (&bs);
342
343 return 1;
344
345 } // End of Compress_Block_BZ2
346
Uncompress_Block_BZ2(nffile_t * nffile)347 static int Uncompress_Block_BZ2(nffile_t *nffile) {
348 bz_stream bs;
349
350 BZ2_prep_stream (&bs);
351 BZ2_bzDecompressInit (&bs, 0, 0);
352
353 bs.next_in = (char*)(nffile->buff_pool[0] + sizeof(data_block_header_t));
354 bs.next_out = (char*)(nffile->buff_pool[1] + sizeof(data_block_header_t));
355 bs.avail_in = nffile->block_header->size;
356 bs.avail_out = nffile->buff_size;
357
358 for (;;) {
359 int r = BZ2_bzDecompress (&bs);
360 if (r == BZ_OK) {
361 continue;
362 } else if (r != BZ_STREAM_END) {
363 BZ2_bzDecompressEnd (&bs);
364 return NF_CORRUPT;
365 } else {
366 break;
367 }
368 }
369
370 // copy header
371 memcpy(nffile->buff_pool[1], nffile->buff_pool[0], sizeof(data_block_header_t));
372 ((data_block_header_t *)nffile->buff_pool[1])->size = bs.total_out_lo32;
373
374 // swap buffers
375 void *_tmp = nffile->buff_pool[1];
376 nffile->buff_pool[1] = nffile->buff_pool[0];
377 nffile->buff_pool[0] = _tmp;
378
379 nffile->block_header = nffile->buff_pool[0];
380 nffile->buff_ptr = nffile->buff_pool[0] + sizeof(data_block_header_t);
381
382 BZ2_bzDecompressEnd (&bs);
383
384 return 1;
385
386 } // End of Uncompress_Block_BZ2
387
OpenFile(char * filename,nffile_t * nffile)388 nffile_t *OpenFile(char *filename, nffile_t *nffile){
389 struct stat stat_buf;
390 int ret, allocated;
391
392 if ( !nffile ) {
393 nffile = NewFile();
394 if ( nffile == NULL ) {
395 return NULL;
396 }
397 allocated = 1;
398 } else
399 allocated = 0;
400
401
402 if ( filename == NULL ) {
403 // stdin
404 // Zero Stat
405 nffile->fd = STDIN_FILENO;
406 } else {
407 // regular file
408 if ( stat(filename, &stat_buf) ) {
409 LogError("Can't stat '%s': %s\n", filename, strerror(errno));
410 if ( allocated ) {
411 DisposeFile(nffile);
412 return NULL;
413 }
414 }
415
416 if (!S_ISREG(stat_buf.st_mode) ) {
417 LogError("'%s' is not a file\n", filename);
418 if ( allocated ) {
419 DisposeFile(nffile);
420 return NULL;
421 }
422 }
423
424 // printf("Statfile %s\n",filename);
425 nffile->fd = open(filename, O_RDONLY);
426 if ( nffile->fd < 0 ) {
427 LogError("Error open file: %s\n", strerror(errno));
428 if ( allocated ) {
429 DisposeFile(nffile);
430 return NULL;
431 }
432 }
433
434 }
435
436 ret = read(nffile->fd, (void *)nffile->file_header, sizeof(file_header_t));
437 if ( nffile->file_header->magic != MAGIC ) {
438 LogError("Open file '%s': bad magic: 0x%X\n", filename ? filename : "<stdin>", nffile->file_header->magic );
439 CloseFile(nffile);
440 if ( allocated ) {
441 DisposeFile(nffile);
442 return NULL;
443 }
444 }
445
446 if ( nffile->file_header->version != LAYOUT_VERSION_1 ) {
447 LogError("Open file %s: bad version: %u\n", filename, nffile->file_header->version );
448 CloseFile(nffile);
449 if ( allocated ) {
450 DisposeFile(nffile);
451 return NULL;
452 }
453 }
454
455 ret = read(nffile->fd, (void *)nffile->stat_record, sizeof(stat_record_t));
456 if ( ret < 0 ) {
457 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
458 CloseFile(nffile);
459 if ( allocated ) {
460 DisposeFile(nffile);
461 return NULL;
462 }
463 }
464
465 CurrentIdent = nffile->file_header->ident;
466
467 int compression = FILE_COMPRESSION(nffile);
468 switch (compression) {
469 case NOT_COMPRESSED:
470 break;
471 case LZO_COMPRESSED:
472 if ( !lzo_initialized && !LZO_initialize() && allocated ) {
473 DisposeFile(nffile);
474 return NULL;
475 }
476 break;
477 case LZ4_COMPRESSED:
478 if ( !lz4_initialized && !LZ4_initialize() && allocated ) {
479 DisposeFile(nffile);
480 return NULL;
481 }
482 break;
483 case BZ2_COMPRESSED:
484 if ( !bz2_initialized && !BZ2_initialize() && allocated ) {
485 DisposeFile(nffile);
486 return NULL;
487 }
488 break;
489 }
490
491 return nffile;
492
493 } // End of OpenFile
494
CloseFile(nffile_t * nffile)495 void CloseFile(nffile_t *nffile){
496
497 if ( !nffile )
498 return;
499
500 // do not close stdout
501 if ( nffile->fd )
502 close(nffile->fd);
503
504 } // End of CloseFile
505
ChangeIdent(char * filename,char * Ident)506 int ChangeIdent(char *filename, char *Ident) {
507 file_header_t FileHeader;
508 struct stat stat_buf;
509 int fd;
510
511 if ( filename == NULL )
512 return 0;
513
514 if ( stat(filename, &stat_buf) ) {
515 LogError("Can't stat '%s': %s\n", filename, strerror(errno));
516 return -1;
517 }
518
519 if (!S_ISREG(stat_buf.st_mode) ) {
520 LogError("'%s' is not a file\n", filename);
521 return -1;
522 }
523
524 fd = open(filename, O_RDWR);
525 if ( fd < 0 ) {
526 LogError("Error open file: %s\n", strerror(errno));
527 return fd;
528 }
529
530 if ( read(fd, (void *)&FileHeader, sizeof(FileHeader)) < 0 ) {
531 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
532 close(fd);
533 return -1;
534 }
535 if ( FileHeader.magic != MAGIC ) {
536 LogError("Open file '%s': bad magic: 0x%X\n", filename, FileHeader.magic );
537 close(fd);
538 return -1;
539 }
540 if ( FileHeader.version != LAYOUT_VERSION_1 ) {
541 LogError("Open file %s: bad version: %u\n", filename, FileHeader.version );
542 close(fd);
543 return -1;
544 }
545
546 strncpy(FileHeader.ident, Ident, IDENTLEN);
547 FileHeader.ident[IDENTLEN - 1] = 0;
548
549 if ( lseek(fd, 0, SEEK_SET) < 0 ) {
550 LogError("lseek() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
551 close(fd);
552 return -1;
553 }
554
555 if ( write(fd, (void *)&FileHeader, sizeof(file_header_t)) <= 0 ) {
556 LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
557 }
558
559 if ( close(fd) < 0 ) {
560 LogError("close() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
561 return -1;
562 }
563
564 return 0;
565
566 } // End of ChangeIdent
567
568
PrintStat(stat_record_t * s)569 void PrintStat(stat_record_t *s) {
570
571 if ( s == NULL )
572 return;
573
574 // format info: make compiler happy with conversion to (unsigned long long),
575 // which does not change the size of the parameter
576 printf("Ident: %s\n", CurrentIdent);
577 printf("Flows: %llu\n", (unsigned long long)s->numflows);
578 printf("Flows_tcp: %llu\n", (unsigned long long)s->numflows_tcp);
579 printf("Flows_udp: %llu\n", (unsigned long long)s->numflows_udp);
580 printf("Flows_icmp: %llu\n", (unsigned long long)s->numflows_icmp);
581 printf("Flows_other: %llu\n", (unsigned long long)s->numflows_other);
582 printf("Packets: %llu\n", (unsigned long long)s->numpackets);
583 printf("Packets_tcp: %llu\n", (unsigned long long)s->numpackets_tcp);
584 printf("Packets_udp: %llu\n", (unsigned long long)s->numpackets_udp);
585 printf("Packets_icmp: %llu\n", (unsigned long long)s->numpackets_icmp);
586 printf("Packets_other: %llu\n", (unsigned long long)s->numpackets_other);
587 printf("Bytes: %llu\n", (unsigned long long)s->numbytes);
588 printf("Bytes_tcp: %llu\n", (unsigned long long)s->numbytes_tcp);
589 printf("Bytes_udp: %llu\n", (unsigned long long)s->numbytes_udp);
590 printf("Bytes_icmp: %llu\n", (unsigned long long)s->numbytes_icmp);
591 printf("Bytes_other: %llu\n", (unsigned long long)s->numbytes_other);
592 printf("First: %u\n", s->first_seen);
593 printf("Last: %u\n", s->last_seen);
594 printf("msec_first: %u\n", s->msec_first);
595 printf("msec_last: %u\n", s->msec_last);
596 printf("Sequence failures: %u\n", s->sequence_failure);
597 } // End of PrintStat
598
NewFile(void)599 static nffile_t *NewFile(void) {
600 nffile_t *nffile;
601 int i;
602
603 // Create struct
604 nffile = calloc(1, sizeof(nffile_t));
605 if ( !nffile ) {
606 LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
607 return NULL;
608 }
609 nffile->buff_ptr = NULL;
610 nffile->fd = 0;
611
612 // Init file header
613 nffile->file_header = calloc(1, sizeof(file_header_t));
614 if ( !nffile->file_header ) {
615 LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
616 return NULL;
617 }
618 nffile->file_header->magic = MAGIC;
619 nffile->file_header->version = LAYOUT_VERSION_1;
620 nffile->file_header->flags = 0;
621 nffile->file_header->NumBlocks = 0;
622
623 nffile->stat_record = calloc(1, sizeof(stat_record_t));
624 if ( !nffile->stat_record ) {
625 LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
626 return NULL;
627 }
628
629 // init data buffer
630 nffile->buff_size = 2 * BUFFSIZE;
631 for (i=0; i<NUM_BUFFS; i++ ) {
632 // allocate twice of BUFFSIZE initially - should be ok, otherwise expand
633 nffile->buff_pool[i] = malloc(nffile->buff_size);
634 if ( !nffile->buff_pool[i] ) {
635 LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
636 return NULL;
637 }
638 }
639
640 nffile->block_header = nffile->buff_pool[0];
641 nffile->block_header->size = 0;
642 nffile->block_header->NumRecords = 0;
643 nffile->block_header->id = DATA_BLOCK_TYPE_2;
644 nffile->block_header->flags = 0;
645
646 nffile->buff_ptr = (void *)((pointer_addr_t)nffile->block_header + sizeof(data_block_header_t));
647
648 return nffile;
649
650 } // End of NewFile
651
DisposeFile(nffile_t * nffile)652 nffile_t *DisposeFile(nffile_t *nffile) {
653 int i;
654
655 free(nffile->file_header);
656 free(nffile->stat_record);
657
658 for (i=0; i<NUM_BUFFS; i++ ) {
659 free(nffile->buff_pool[i]);
660 }
661
662 return NULL;
663 } // End of DisposeFile
664
OpenNewFile(char * filename,nffile_t * nffile,int compress,int anonymized,char * ident)665 nffile_t *OpenNewFile(char *filename, nffile_t *nffile, int compress, int anonymized, char *ident) {
666 size_t len;
667 int fd, flags;
668
669 switch (compress) {
670 case NOT_COMPRESSED:
671 flags = FLAG_NOT_COMPRESSED;
672 break;
673 case LZO_COMPRESSED:
674 flags = FLAG_LZO_COMPRESSED;
675 if ( !lzo_initialized && !LZO_initialize() ) {
676 LogError("Failed to initialize LZO compression");
677 return NULL;
678 }
679 break;
680 case LZ4_COMPRESSED:
681 flags = FLAG_LZ4_COMPRESSED;
682 if ( !lz4_initialized && !LZ4_initialize() ) {
683 LogError("Failed to initialize LZ4 compression");
684 return NULL;
685 }
686 break;
687 case BZ2_COMPRESSED:
688 flags = FLAG_BZ2_COMPRESSED;
689 if ( !bz2_initialized && !BZ2_initialize() ) {
690 LogError("Failed to initialize BZ2 compression");
691 return NULL;
692 }
693 break;
694 default:
695 LogError("Unknown compression ID: %i\n", compress);
696 return NULL;
697 }
698
699 fd = 0;
700 if ( strcmp(filename, "-") == 0 ) { // output to stdout
701 fd = STDOUT_FILENO;
702 } else {
703 fd = open(filename, O_CREAT | O_RDWR | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH );
704 if ( fd < 0 ) {
705 LogError("Failed to open file %s: '%s'" , filename, strerror(errno));
706 return NULL;
707 }
708 }
709
710 // Allocate new struct if not given
711 if ( nffile == NULL ) {
712 nffile = NewFile();
713 if ( nffile == NULL ) {
714 return NULL;
715 }
716 }
717
718 nffile->fd = fd;
719
720 if ( anonymized )
721 SetFlag(flags, FLAG_ANONYMIZED);
722
723 nffile->file_header->flags = flags;
724
725 if ( nffile->stat_record ) {
726 memset((void *)nffile->stat_record, 0, sizeof(stat_record_t));
727 nffile->stat_record->first_seen = 0x7fffffff;
728 nffile->stat_record->msec_first = 999;
729 }
730
731 if ( ident ) {
732 strncpy(nffile->file_header->ident, ident, IDENTLEN);
733 nffile->file_header->ident[IDENTLEN - 1] = 0;
734 }
735
736 nffile->file_header->NumBlocks = 0;
737 len = sizeof(file_header_t);
738 if ( write(nffile->fd, (void *)nffile->file_header, len) < len ) {
739 LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
740 close(nffile->fd);
741 nffile->fd = 0;
742 return NULL;
743 }
744
745 // write empty stat record - ist updated when file gets closed
746 len = sizeof(stat_record_t);
747 if ( write(nffile->fd, (void *)nffile->stat_record, len) < len ) {
748 LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
749 close(nffile->fd);
750 nffile->fd = 0;
751 return NULL;
752 }
753
754 return nffile;
755
756 } /* End of OpenNewFile */
757
AppendFile(char * filename)758 nffile_t *AppendFile(char *filename) {
759 nffile_t *nffile;
760
761 // try to open the existing file
762 nffile = OpenFile(filename, NULL);
763 if ( !nffile )
764 return NULL;
765
766 // file is valid - re-open the file mode RDWR
767 close(nffile->fd);
768 nffile->fd = open(filename, O_RDWR | O_APPEND, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH );
769 if ( nffile->fd < 0 ) {
770 LogError("Failed to open file %s: '%s'" , filename, strerror(errno));
771 DisposeFile(nffile);
772 return NULL;
773 }
774
775 // init output data buffer
776 nffile->block_header = malloc(BUFFSIZE + sizeof(data_block_header_t));
777 if ( !nffile->block_header ) {
778 LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
779 close(nffile->fd);
780 DisposeFile(nffile);
781 return NULL;
782 }
783 nffile->block_header->size = 0;
784 nffile->block_header->NumRecords = 0;
785 nffile->block_header->id = DATA_BLOCK_TYPE_2;
786 nffile->block_header->flags = 0;
787 nffile->buff_ptr = (void *)((pointer_addr_t)nffile->block_header + sizeof(data_block_header_t));
788
789 int compression = FILE_COMPRESSION(nffile);
790 switch (compression) {
791 case NOT_COMPRESSED:
792 break;
793 case LZO_COMPRESSED:
794 if ( !lzo_initialized && !LZO_initialize() ) {
795 LogError("Failed to initialize LZO compression");
796 close(nffile->fd);
797 DisposeFile(nffile);
798 return NULL;
799 }
800 break;
801 case LZ4_COMPRESSED:
802 if ( !lz4_initialized && !LZ4_initialize() ) {
803 LogError("Failed to initialize LZ4 compression");
804 close(nffile->fd);
805 DisposeFile(nffile);
806 return NULL;
807 }
808 break;
809 case BZ2_COMPRESSED:
810 if ( !bz2_initialized && !BZ2_initialize() ) {
811 LogError("Failed to initialize BZ2 compression");
812 close(nffile->fd);
813 DisposeFile(nffile);
814 return NULL;
815 }
816 break;
817 }
818
819 return nffile;
820
821 } /* End of AppendFile */
822
RenameAppend(char * from,char * to)823 int RenameAppend(char *from, char *to) {
824 int fd_to, fd_from, ret;
825 int compressed_to, compressed_from;
826 stat_record_t stat_record_to, stat_record_from;
827 data_block_header_t *block_header;
828 void *p;
829
830 fd_to = OpenRaw(to, &stat_record_to, &compressed_to);
831 if ( fd_to == 0 ) {
832 // file does not exists, use rename
833 return rename(from, to) == 0 ? 1 : 0;
834 }
835
836 fd_from = OpenRaw(from, &stat_record_from, &compressed_from);
837 if ( fd_from <= 0 ) {
838 // file does not exists - strange
839 close(fd_to);
840 return 0;
841 }
842
843 // both files open - append data
844 ret = lseek(fd_to, 0, SEEK_END);
845 if ( ret < 0 ) {
846 LogError("lseek() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
847 close(fd_from);
848 close(fd_to);
849 return 0;
850 }
851
852 block_header = malloc(sizeof(data_block_header_t) + BUFFSIZE);
853 if ( !block_header ) {
854 LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno));
855 close(fd_from);
856 close(fd_to);
857 return 0;
858 }
859 p = (void *)((void *)block_header + sizeof(data_block_header_t));
860
861 while (1) {
862 ret = read(fd_from, (void *)block_header, sizeof(data_block_header_t));
863 if ( ret == 0 )
864 // EOF
865 break;
866
867 if ( ret < 0 ) {
868 // that's bad! difficult to recover. stat will be inconsistent
869 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
870 break;
871 }
872
873 // read data block
874 ret = read(fd_from, p, block_header->size);
875 if ( ret != block_header->size ) {
876 // that's bad! difficult to recover. stat will be inconsistent
877 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
878 break;
879 }
880 // append data block
881 ret = write(fd_to, block_header, sizeof(data_block_header_t) + block_header->size);
882 if ( ret < 0 ) {
883 // that's bad! difficult to recover. stat will be inconsistent
884 LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
885 break;
886 }
887 }
888
889 SumStatRecords(&stat_record_to, &stat_record_from);
890 // both files open - append data
891 ret = lseek(fd_to, sizeof(file_header_t), SEEK_SET);
892 if ( ret < 0 ) {
893 LogError("lseek() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
894 close(fd_from);
895 close(fd_to);
896 return 0;
897 }
898
899 if ( write(fd_to, (void *)&stat_record_to, sizeof(stat_record_t)) <= 0 ) {
900 LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
901 close(fd_from);
902 close(fd_to);
903 return 0;
904 }
905
906 close(fd_from);
907 close(fd_to);
908 unlink(from);
909 return 1;
910
911 } // End of RenameAppend
912
OpenRaw(char * filename,stat_record_t * stat_record,int * compressed)913 static int OpenRaw(char *filename, stat_record_t *stat_record, int *compressed) {
914 struct stat stat_buf;
915 file_header_t file_header;
916 int fd, ret;
917
918 if ( stat(filename, &stat_buf) ) {
919 // file does not exists
920 return 0;
921 }
922
923 // file exists - should be a regular file
924 if (!S_ISREG(stat_buf.st_mode) ) {
925 // should nor really happen - catch it anyway
926 LogError("'%s' is not a regular file\n", filename);
927 return -1;
928 }
929
930 // file exists - append to existing
931 fd = open(filename, O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH );
932 if ( fd < 0 ) {
933 LogError("open() failed for file %s: '%s'" , filename, strerror(errno));
934 return -1;
935 }
936
937 ret = read(fd, (void *)&file_header, sizeof(file_header_t));
938 if ( ret < 0 ) {
939 LogError("read() failed for file %s: '%s'" , filename, strerror(errno));
940 close(fd);
941 return -1;
942 }
943
944 if ( file_header.magic != MAGIC ) {
945 LogError("Open file '%s': bad magic: 0x%X\n", filename, file_header.magic );
946 close(fd);
947 return -1;
948 }
949
950 if ( file_header.version != LAYOUT_VERSION_1 ) {
951 LogError("Open file %s: bad version: %u\n", filename, file_header.version );
952 close(fd);
953 return -1;
954 }
955
956 ret = read(fd, (void *)stat_record, sizeof(stat_record_t));
957 if ( ret < 0 ) {
958 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
959 close(fd);
960 return -1;
961 }
962
963 if ( file_header.flags & FLAG_LZO_COMPRESSED )
964 *compressed = FLAG_LZO_COMPRESSED;
965 else if ( file_header.flags & FLAG_LZ4_COMPRESSED )
966 *compressed = FLAG_LZ4_COMPRESSED;
967 else if ( file_header.flags & FLAG_BZ2_COMPRESSED )
968 *compressed = FLAG_BZ2_COMPRESSED;
969 else
970 *compressed = 0;
971
972 return fd;
973
974 } // End of OpenRaw
975
CloseUpdateFile(nffile_t * nffile,char * ident)976 int CloseUpdateFile(nffile_t *nffile, char *ident) {
977
978 if ( nffile->block_header->size ) {
979 int ret = WriteBlock(nffile);
980 if ( ret < 0 ) {
981 LogError("Failed to flush output buffer");
982 return 0;
983 }
984 }
985
986 if ( lseek(nffile->fd, 0, SEEK_SET) < 0 ) {
987 // lseek on stdout works if output redirected:
988 // e.g. -w - > outfile
989 // but fails on pipe e.g. -w - | ./nfdump ....
990 if ( nffile->fd == STDOUT_FILENO ) {
991 return 1;
992 } else {
993 LogError("lseek() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
994 close(nffile->fd);
995 return 0;
996 }
997 }
998
999 if ( ident ) {
1000 strncpy(nffile->file_header->ident, ident, IDENTLEN);
1001 } else {
1002 if ( strlen(nffile->file_header->ident) == 0 )
1003 strncpy(nffile->file_header->ident, IDENTNONE, IDENTLEN);
1004 }
1005
1006 if ( write(nffile->fd, (void *)nffile->file_header, sizeof(file_header_t)) <= 0 ) {
1007 LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1008 }
1009 if ( write(nffile->fd, (void *)nffile->stat_record, sizeof(stat_record_t)) <= 0 ) {
1010 LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1011 }
1012 if ( close(nffile->fd) < 0 ) {
1013 LogError("close() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1014 return 0;
1015 }
1016
1017 nffile->file_header->NumBlocks = 0;
1018
1019 return 1;
1020
1021 } /* End of CloseUpdateFile */
1022
ReadBlock(nffile_t * nffile)1023 int ReadBlock(nffile_t *nffile) {
1024 ssize_t ret, read_bytes, buff_bytes, request_size;
1025 void *read_ptr;
1026 uint32_t compression;
1027
1028 ret = read(nffile->fd, nffile->block_header, sizeof(data_block_header_t));
1029 if ( ret == 0 ) // EOF
1030 return NF_EOF;
1031
1032 if ( ret == -1 ) // ERROR
1033 return NF_ERROR;
1034
1035 // Check for sane buffer size
1036 if ( ret != sizeof(data_block_header_t) ) {
1037 // this is most likely a corrupt file
1038 LogError("Corrupt data file: Read %i bytes, requested %u\n", ret, sizeof(data_block_header_t));
1039 return NF_CORRUPT;
1040 }
1041
1042 // block header read successfully
1043 read_bytes = ret;
1044
1045 // Check for sane buffer size
1046 if ( nffile->block_header->size > BUFFSIZE ||
1047 nffile->block_header->size == 0 || nffile->block_header->NumRecords == 0) {
1048 // this is most likely a corrupt file
1049 LogError("Corrupt data file: Requested buffer size %u exceeds max. buffer size", nffile->block_header->size);
1050 return NF_CORRUPT;
1051 }
1052
1053 // check block compression - defaults to file compression setting
1054
1055 compression = FILE_COMPRESSION(nffile);
1056 ret = read(nffile->fd, nffile->buff_ptr, nffile->block_header->size);
1057 if ( ret == nffile->block_header->size ) {
1058 // we have the whole record and are done for now
1059 switch (compression) {
1060 case NOT_COMPRESSED:
1061 break;
1062 case LZO_COMPRESSED:
1063 if ( Uncompress_Block_LZO(nffile) < 0 )
1064 return NF_CORRUPT;
1065 break;
1066 case LZ4_COMPRESSED:
1067 if ( Uncompress_Block_LZ4(nffile) < 0 )
1068 return NF_CORRUPT;
1069 break;
1070 case BZ2_COMPRESSED:
1071 if ( Uncompress_Block_BZ2(nffile) < 0 )
1072 return NF_CORRUPT;
1073 break;
1074 }
1075 nffile->buff_ptr = (void *)((pointer_addr_t)nffile->block_header + sizeof(data_block_header_t));
1076 return read_bytes + nffile->block_header->size;
1077 }
1078
1079 if ( ret == 0 ) {
1080 // EOF not expected here - this should never happen, file may be corrupt
1081 LogError("ReadBlock() Corrupt data file: Unexpected EOF while reading data block.\n");
1082 return NF_CORRUPT;
1083 }
1084
1085 if ( ret == -1 ) { // ERROR
1086 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1087 return NF_ERROR;
1088 }
1089
1090 // Ups! - ret is != block_header->size
1091 // this was a short read - most likely reading from the stdin pipe
1092 // loop until we have requested size
1093
1094 buff_bytes = ret; // already in buffer
1095 request_size = nffile->block_header->size - buff_bytes; // still to go for this amount of data
1096
1097 read_ptr = (void *)((pointer_addr_t)nffile->buff_ptr + buff_bytes);
1098 do {
1099 ret = read(nffile->fd, read_ptr, request_size);
1100 if ( ret < 0 ) {
1101 // -1: Error - not expected
1102 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1103 return NF_ERROR;
1104 }
1105
1106 if ( ret == 0 ) {
1107 // 0: EOF - not expected
1108 LogError("read() corrupt data file: Unexpected EOF in %s line %d: %s\n", __FILE__, __LINE__);
1109 return NF_CORRUPT;
1110 }
1111
1112 buff_bytes += ret;
1113 request_size = nffile->block_header->size - buff_bytes;
1114
1115 if ( request_size > 0 ) {
1116 // still a short read - continue in read loop
1117 read_ptr = (void *)((pointer_addr_t)nffile->buff_ptr + buff_bytes);
1118 }
1119 } while ( request_size > 0 );
1120
1121 switch (compression) {
1122 case NOT_COMPRESSED:
1123 break;
1124 case LZO_COMPRESSED:
1125 if ( Uncompress_Block_LZO(nffile) < 0 )
1126 return NF_CORRUPT;
1127 break;
1128 case LZ4_COMPRESSED:
1129 if ( Uncompress_Block_LZ4(nffile) < 0 )
1130 return NF_CORRUPT;
1131 break;
1132 case BZ2_COMPRESSED:
1133 if ( Uncompress_Block_BZ2(nffile) < 0 )
1134 return NF_CORRUPT;
1135 break;
1136 }
1137
1138 nffile->buff_ptr = (void *)((pointer_addr_t)nffile->block_header + sizeof(data_block_header_t));
1139 return read_bytes + nffile->block_header->size;
1140
1141 } // End of ReadBlock
1142
WriteBlock(nffile_t * nffile)1143 int WriteBlock(nffile_t *nffile) {
1144 int ret, compression;
1145
1146 // empty blocks need not to be stored
1147 if ( nffile->block_header->size == 0 )
1148 return 1;
1149
1150 compression = FILE_COMPRESSION(nffile);
1151 switch (compression) {
1152 case NOT_COMPRESSED:
1153 break;
1154 case LZO_COMPRESSED:
1155 if ( Compress_Block_LZO(nffile) < 0 ) return -1;
1156 break;
1157 case LZ4_COMPRESSED:
1158 if ( Compress_Block_LZ4(nffile) < 0 ) return -1;
1159 break;
1160 case BZ2_COMPRESSED:
1161 if ( Compress_Block_BZ2(nffile) < 0 ) return -1;
1162 break;
1163 }
1164
1165 ret = write(nffile->fd, (void *)nffile->block_header, sizeof(data_block_header_t) + nffile->block_header->size);
1166 if (ret > 0) {
1167 nffile->block_header->size = 0;
1168 nffile->block_header->NumRecords = 0;
1169 nffile->buff_ptr = (void *)((pointer_addr_t) nffile->block_header + sizeof (data_block_header_t));
1170 nffile->file_header->NumBlocks++;
1171 }
1172
1173 return ret;
1174
1175 } // End of WriteBlock
1176
ModifyCompressFile(char * rfile,char * Rfile,int compress)1177 void ModifyCompressFile(char * rfile, char *Rfile, int compress) {
1178 int i, anonymized, compression;
1179 ssize_t ret;
1180 nffile_t *nffile_r, *nffile_w;
1181 stat_record_t *_s;
1182 char *filename, outfile[MAXPATHLEN];
1183
1184 SetupInputFileSequence(NULL, rfile, Rfile);
1185
1186 nffile_r = NULL;
1187 while (1) {
1188 nffile_r = GetNextFile(nffile_r, 0, 0);
1189
1190 // last file
1191 if ( nffile_r == EMPTY_LIST )
1192 break;
1193
1194 filename = GetCurrentFilename();
1195
1196 if ( !nffile_r || !filename) {
1197 break;
1198 }
1199
1200 compression = FILE_COMPRESSION(nffile_r);
1201 if ( compression == compress ) {
1202 printf("File %s is already same compression methode\n", filename);
1203 continue;
1204 }
1205
1206 // tmp filename for new output file
1207 snprintf(outfile, MAXPATHLEN, "%s-tmp", filename);
1208 outfile[MAXPATHLEN-1] = '\0';
1209
1210 anonymized = IP_ANONYMIZED(nffile_r);
1211
1212 // allocate output file
1213 nffile_w = OpenNewFile(outfile, NULL, compress, anonymized, NULL);
1214 if ( !nffile_w ) {
1215 CloseFile(nffile_r);
1216 DisposeFile(nffile_r);
1217 break;;
1218 }
1219
1220 // swap stat records :)
1221 _s = nffile_r->stat_record;
1222 nffile_r->stat_record = nffile_w->stat_record;
1223 nffile_w->stat_record = _s;
1224
1225 for ( i=0; i < nffile_r->file_header->NumBlocks; i++ ) {
1226 ret = ReadBlock(nffile_r);
1227 if ( ret < 0 ) {
1228 LogError("Error while reading data block. Abort.\n");
1229 CloseFile(nffile_r);
1230 DisposeFile(nffile_r);
1231 CloseFile(nffile_w);
1232 DisposeFile(nffile_w);
1233 unlink(outfile);
1234 return;
1235 }
1236
1237 // swap buffers
1238 void *_tmp = nffile_r->buff_pool[0];
1239 nffile_r->buff_pool[0] = nffile_w->buff_pool[0];
1240 nffile_w->buff_pool[0] = _tmp;
1241 nffile_w->block_header = nffile_w->buff_pool[0];
1242 nffile_r->block_header = nffile_r->buff_pool[0];
1243 nffile_r->buff_ptr = (void *)((pointer_addr_t)nffile_r->block_header + sizeof(data_block_header_t));
1244
1245 if ( WriteBlock(nffile_w) <= 0 ) {
1246 LogError("Failed to write output buffer to disk: '%s'" , strerror(errno));
1247 CloseFile(nffile_r);
1248 DisposeFile(nffile_r);
1249 CloseFile(nffile_w);
1250 DisposeFile(nffile_w);
1251 unlink(outfile);
1252 return;
1253 }
1254 }
1255
1256 printf("File %s compression changed\n", filename);
1257 if ( !CloseUpdateFile(nffile_w, nffile_r->file_header->ident) ) {
1258 unlink(outfile);
1259 LogError("Failed to close file: '%s'" , strerror(errno));
1260 } else {
1261 unlink(filename);
1262 rename(outfile, filename);
1263 }
1264
1265 DisposeFile(nffile_w);
1266 }
1267
1268 } // End of ModifyCompressFile
1269
QueryFile(char * filename)1270 void QueryFile(char *filename) {
1271 int i;
1272 nffile_t *nffile;
1273 uint32_t num_records, type1, type2;
1274 struct stat stat_buf;
1275 ssize_t ret;
1276 off_t fsize;
1277
1278 if ( stat(filename, &stat_buf) ) {
1279 LogError("Can't stat '%s': %s\n", filename, strerror(errno));
1280 return;
1281 }
1282
1283 nffile = OpenFile(filename, NULL);
1284 if ( !nffile ) {
1285 return;
1286 }
1287
1288 num_records = 0;
1289 // set file size to current position ( file header )
1290 fsize = lseek(nffile->fd, 0, SEEK_CUR);
1291 type1 = 0;
1292 type2 = 0;
1293 printf("File : %s\n", filename);
1294 printf ("Version : %u - %s\n", nffile->file_header->version,
1295 FILE_IS_LZO_COMPRESSED (nffile) ? "lzo compressed" :
1296 FILE_IS_LZ4_COMPRESSED (nffile) ? "lz4 compressed" :
1297 FILE_IS_BZ2_COMPRESSED (nffile) ? "bz2 compressed" :
1298 "not compressed");
1299
1300 printf("Blocks : %u\n", nffile->file_header->NumBlocks);
1301 for ( i=0; i < nffile->file_header->NumBlocks; i++ ) {
1302 if ( (fsize + sizeof(data_block_header_t)) > stat_buf.st_size ) {
1303 LogError("Unexpected read beyond EOF! File corrupted. Abort.\n");
1304 LogError("Expected %u blocks, counted %i\n", nffile->file_header->NumBlocks, i);
1305 break;
1306 }
1307 ret = read(nffile->fd, (void *)nffile->block_header, sizeof(data_block_header_t));
1308 if ( ret < 0 ) {
1309 LogError("Error reading block %i: %s\n", i, strerror(errno));
1310 break;
1311 }
1312
1313 // Should never happen, as catched already in first check, but test it anyway ..
1314 if ( ret == 0 ) {
1315 LogError("Unexpected end of file reached. Expected %u blocks, counted %i\n", nffile->file_header->NumBlocks, i);
1316 break;
1317 }
1318 if ( ret < sizeof(data_block_header_t) ) {
1319 LogError("Short read: Expected %u bytes, read: %i\n", sizeof(data_block_header_t), ret);
1320 break;
1321 }
1322 fsize += sizeof(data_block_header_t);
1323
1324 num_records += nffile->block_header->NumRecords;
1325 switch ( nffile->block_header->id) {
1326 case DATA_BLOCK_TYPE_1:
1327 type1++;
1328 break;
1329 case DATA_BLOCK_TYPE_2:
1330 type2++;
1331 break;
1332 default:
1333 printf("block %i has unknown type %u\n", i, nffile->block_header->id);
1334 }
1335
1336 if ( (fsize + nffile->block_header->size ) > stat_buf.st_size ) {
1337 LogError("Expected to seek beyond EOF! File corrupted. Abort.\n");
1338 break;
1339 }
1340 fsize += nffile->block_header->size;
1341
1342 ret = lseek(nffile->fd, nffile->block_header->size, SEEK_CUR);
1343 if ( ret < 0 ) {
1344 LogError("Error seeking block %i: %s\n", i, strerror(errno));
1345 break;
1346 }
1347 if ( fsize != ret ) {
1348 LogError("Expected seek: Expected: %u, got: %u\n", fsize, ret);
1349 break;
1350 }
1351 }
1352
1353 if ( fsize < stat_buf.st_size ) {
1354 LogError("Extra data detected after regular blocks: %i bytes\n", stat_buf.st_size-fsize);
1355 }
1356
1357 printf(" Type 1 : %u\n", type1);
1358 printf(" Type 2 : %u\n", type2);
1359 printf("Records : %u\n", num_records);
1360
1361 CloseFile(nffile);
1362 DisposeFile(nffile);
1363
1364 } // End of QueryFile
1365
1366 // simple interface to get a statrecord from a file without nffile overhead
GetStatRecord(char * filename,stat_record_t * stat_record)1367 stat_record_t *GetStatRecord(char *filename, stat_record_t *stat_record) {
1368 file_header_t file_header;
1369 int fd, ret;
1370
1371 fd = open(filename, O_RDONLY);
1372 if ( fd < 0 ) {
1373 LogError("open() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1374 return NULL;
1375 }
1376
1377 ret = read(fd, (void *)&file_header, sizeof(file_header_t));
1378 if ( ret < 0 ) {
1379 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1380 close(fd);
1381 return NULL;
1382 }
1383
1384 if ( file_header.magic != MAGIC ) {
1385 LogError("Open file '%s': bad magic: 0x%X\n", filename ? filename : "<stdin>", file_header.magic );
1386 close(fd);
1387 return NULL;
1388 }
1389
1390 if ( file_header.version != LAYOUT_VERSION_1 ) {
1391 LogError("Open file %s: bad version: %u\n", filename, file_header.version );
1392 close(fd);
1393 return NULL;
1394 }
1395
1396 ret = read(fd, (void *)stat_record, sizeof(stat_record_t));
1397 if ( ret < 0 ) {
1398 LogError("read() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
1399 close(fd);
1400 return NULL;
1401 }
1402
1403 close(fd);
1404 return stat_record;
1405
1406 } // End of GetStatRecord
1407