1 /*
2 * ffe - flat file extractor
3 *
4 * Copyright (C) 2006 Timo Savinen
5 * This file is part of ffe.
6 *
7 * ffe is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * ffe is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with ffe; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 *
21 */
22
23 /* $Id: execute.c,v 1.118 2011-04-10 10:12:09 timo Exp $ */
24
25 #include "ffe.h"
26 #include <stdlib.h>
27 #include <string.h>
28 #include <ctype.h>
29 #include <wchar.h>
30 #ifdef HAVE_PRINTF_H
31 #include <printf.h>
32 #endif
33
34
35 #ifdef PACKAGE
36 static char *program = PACKAGE;
37 #else
38 static char *program = "ffe";
39 #endif
40
41 #define READ_LINE_LEN 33554432
42 #define READ_LINE_LEN_HIGH (READ_LINE_LEN - 524288)
43
44
45
46 #define GUESS_LINES 1000
47 #define GUESS_BUFFER 524288
48 #define FIELD_SIZE (128 * 1024)
49 #define WRITE_BUFFER (2 * 1024 * 1024)
50 #define JUSTIFY_STRING 128
51
52 extern int update_anon_info(struct structure *,char *);
53 extern void init_libgcrypt();
54
55
56 extern struct replace *replace;
57
58 struct input_file *files = NULL;
59 static struct input_file *current_file = NULL;
60 static FILE *input_fp = NULL;
61 static int ungetchar = -1;
62 static char *default_output_file = NULL;
63 static FILE *default_output_fp = NULL;
64
65 static char *output_file = NULL;
66 static FILE *output_fp = NULL;
67
68 static uint8_t *read_buffer = NULL;
69 static uint8_t *read_buffer_start = NULL;
70 static uint8_t *read_buffer_high_water = NULL;
71
72 static size_t last_consumed = 0; /* for binary reads */
73 static uint8_t *field_buffer = NULL;
74 static int field_buffer_size = FIELD_SIZE;
75 static int guess_lines = 0;
76
77 static int eocf = 0;
78 static int ccount = -1;
79 static int orig_ccount = -1;
80
81 static uint8_t justify_string[JUSTIFY_STRING];
82
83 /* write buffer definitions */
84 static uint8_t *write_buffer = NULL;
85 static int write_buffer_size = WRITE_BUFFER;
86 static uint8_t *write_pos;
87 static uint8_t *write_buffer_end;
88
89 /* file number counters */
90 static long int current_file_lineno;
91 static long int current_total_lineno;
92 static long long int current_offset = 0;
93 static long long int current_file_offset = 0;
94
95
96 /* Pipe management */
97 #define PIPE_OUTPUT_LEN 1048576
98 static uint8_t pipe_output[PIPE_OUTPUT_LEN];
99
100 /* examples of non matching lines */
101 #define NO_MATCH_LINES 1
102 static int no_matching_lines = 0;
103
104 /* header definition from structure */
105 static int headers;
106
107 char *current_file_name = NULL;
108
109 static char debug_file[128];
110 static FILE *debug_fp = NULL;
111 static long int debug_lineno = 0;
112
113 static uint8_t bcd_to_ascii_cap[]={'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','\000'};
114 static uint8_t hex_to_ascii_cap[]={'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
115
116 static uint8_t bcd_to_ascii_low[]={'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','\000'};
117 static uint8_t hex_to_ascii_low[]={'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
118
119 uint8_t *bcd_to_ascii;
120 uint8_t *hex_to_ascii;
121
122
123 static void print_binary_field(uint8_t,struct field *,uint8_t *);
124 static void print_fixed_field(uint8_t,struct field *,uint8_t *);
125
126 inline uint8_t
htocl(uint8_t hex)127 htocl(uint8_t hex)
128 {
129 return hex_to_ascii[hex & 0x0f];
130 }
131
132 inline uint8_t
htocb(uint8_t hex)133 htocb(uint8_t hex)
134 {
135 return hex_to_ascii[(hex >> 4) & 0x0f];
136 }
137
138 inline uint8_t
bcdtocl(uint8_t bcd)139 bcdtocl(uint8_t bcd)
140 {
141 return bcd_to_ascii[bcd & 0x0f];
142 }
143
144 inline uint8_t
bcdtocb(uint8_t bcd)145 bcdtocb(uint8_t bcd)
146 {
147 return bcd_to_ascii[(bcd >> 4) & 0x0f];
148 }
149
150
151 void
set_output_file(char * name)152 set_output_file(char *name)
153 {
154 if(name == NULL)
155 {
156 default_output_fp = stdout;
157 default_output_file = "(stdout)";
158 } else
159 {
160 default_output_fp = xfopen(name,"w");
161 default_output_file = name;
162 }
163 output_fp = default_output_fp;
164 output_file = default_output_file;
165 }
166
167 void
close_output_file()168 close_output_file()
169 {
170 struct output *o = output;
171 int stdoutclosed = 0;
172
173 if(default_output_fp == stdout) stdoutclosed = 1;
174 if(fclose(default_output_fp) != 0)
175 {
176 panic("Error closing file",default_output_file,strerror(errno));
177 }
178
179 while(o != NULL)
180 {
181 if(o->ofp != NULL)
182 {
183 if((o->ofp == stdout && !stdoutclosed) || o->ofp != stdout)
184 {
185 if(o->ofp == stdout) stdoutclosed = 1;
186 if(fclose(o->ofp) != 0)
187 {
188 panic("Error closing file",o->output_file,strerror(errno));
189 }
190 }
191 }
192 o = o->next;
193 }
194 }
195
196 void
set_input_file(char * name)197 set_input_file(char *name)
198 {
199 register struct input_file *f = files;
200
201 if(files == NULL)
202 {
203 files = xmalloc(sizeof(struct input_file));
204 f = files;
205 } else
206 {
207 while(f->next != NULL) f = f->next;
208 f->next = xmalloc(sizeof(struct input_file));
209 f = f->next;
210 }
211
212 f->next = NULL;
213 f->name = xstrdup(name);
214 f->lineno = 0;
215 }
216
217 static FILE *
open_input_stream(char * file,char type)218 open_input_stream(char *file,char type)
219 {
220 int fds[2];
221 pid_t pid;
222 FILE *ret = NULL;
223 char command[1024];
224
225 if(ffe_open != NULL && ffe_open[0] != '\000') // use preprocessor
226 {
227 #if defined(HAVE_WORKING_FORK) && defined(HAVE_DUP2) && defined(HAVE_PIPE)
228 sprintf(command,ffe_open,file);
229 if (pipe(fds) != 0) panic("Cannot create pipe",strerror(errno),NULL);
230 pid = fork();
231 if(pid == (pid_t) 0) /* Child */
232 {
233 close(fds[0]);
234 if(dup2(fds[1],STDOUT_FILENO) == -1) panic("dup2 error",strerror(errno),NULL);
235 if(execl(SHELL_CMD, "sh", "-c", command, NULL) == -1) panic("Starting a shell with execl failed",command,strerror(errno));
236 close(fds[1]);
237 _exit(EXIT_SUCCESS);
238 } else if(pid > (pid_t) 0)
239 {
240 close(fds[1]);
241 ret = fdopen(fds[0],"r");
242 if(ret == NULL) panic("Cannot read from command",command,strerror(errno));
243
244 ungetchar = fgetc(ret);
245
246 if(ungetchar == EOF) // check if pipe returns something, if not open file normally
247 {
248 ungetchar = -1;
249 fclose(ret);
250 ret = NULL;
251 }
252 } else
253 {
254 panic("Cannot fork",strerror(errno),NULL);
255 }
256 #else
257 panic("Input preprocessing is not supported in this system",NULL,NULL);
258 #endif
259 }
260
261 if(ret == NULL)
262 {
263 if(type == BINARY)
264 {
265 ret = xfopenb(file,"r");
266 } else
267 {
268 ret = xfopen(file,"r");
269 }
270 }
271
272 read_buffer = read_buffer_start;
273
274 return ret;
275 }
276
277
278 void
open_input_file(int stype)279 open_input_file(int stype)
280 {
281 read_buffer_start = xmalloc(READ_LINE_LEN);
282 read_buffer = read_buffer_start;
283 read_buffer_high_water = read_buffer_start + READ_LINE_LEN_HIGH;
284
285 field_buffer = xmalloc(field_buffer_size);
286
287 if(files->name[0] == '-' && !files->name[1])
288 {
289 input_fp = stdin;
290 files->name = "(stdin)";
291 } else
292 {
293 input_fp = open_input_stream(files->name,stype);
294 }
295 current_file = files;
296 current_file->lineno = 0;
297 current_file_name = current_file->name;
298 }
299
300 /* read input file until next newline is found
301 */
302 void
complete_line_in_bin_buffer(int * ccount)303 complete_line_in_bin_buffer(int *ccount)
304 {
305 int c;
306
307 if(read_buffer[*ccount-1] != '\n') // check if have newline as last char
308 {
309 do
310 {
311 c = fgetc(input_fp);
312 if(*ccount >= READ_LINE_LEN) panic("Input file cannot be guessed, use -s option",NULL,NULL);
313 if(c != EOF) read_buffer[(*ccount)++] = (uint8_t) c;
314 } while(c != EOF && c != '\n');
315 }
316 }
317
318
319 /* read from input stream.
320 Check if ungetchar contains a valid char and write it to buffer then read nmenb - 1 chars
321 NOTE! it is assumed that size == 1...
322 */
323 static size_t
uc_fread(uint8_t * ptr,size_t size,size_t nmemb,FILE * stream)324 uc_fread(uint8_t *ptr, size_t size, size_t nmemb, FILE *stream)
325 {
326 size_t ret = 0;
327
328 if(ungetchar != -1) // there is peeked char in ungetchar, write it to buffer and read the rest
329 {
330 *ptr = (uint8_t) ungetchar;
331 ungetchar = -1;
332 ptr++;
333 nmemb--;
334 ret = 1;
335 }
336
337 ret += fread(ptr,size,nmemb,stream);
338 return ret;
339 }
340
341 /* read line from input stream.
342 Check if ungetchar contains a valid char and write it to buffer and then read the rest
343 */
344 static int
uc_fgets(uint8_t * s,int size,FILE * stream)345 uc_fgets(uint8_t *s, int size, FILE *stream)
346 {
347 int ret;
348 char *orig_s = s;
349
350 if(ungetchar != -1) // there is peeked char in ungetchar, write it to buffer and read the rest
351 {
352 *s = (uint8_t) ungetchar;
353 ungetchar = -1;
354 if(*s == '\n')
355 {
356 s++;
357 *s = '\000';
358 return 1;
359 }
360 s++;
361 size--;
362 }
363
364 if(fgets(s,size,stream) != NULL)
365 {
366 ret = strlen(orig_s);
367 } else
368 {
369 ret = -1;
370 }
371
372 return ret;
373 }
374
375
376 /* find next lineend and return line length
377 */
378 static int
find_next_LF(uint8_t * start,int length)379 find_next_LF(uint8_t *start,int length)
380 {
381 register uint8_t *p;
382
383 p = memchr(start,'\n',length);
384
385 if(p == NULL)
386 {
387 start[length] = '\n'; // add missing LF
388 return length;
389 }
390 return p - start;
391 }
392
393
394
395 /* reads one file from input */
396 /* returns the line length */
397 /* return -1 on EOF */
398 int
read_input_line(int stype)399 read_input_line(int stype)
400 {
401 int retval;
402 size_t unused;
403
404 do
405 {
406 if(stype == BINARY)
407 {
408 current_offset += (long long) last_consumed;
409 current_file_offset += (long long) last_consumed;
410 }
411
412 if(ccount <= 0)
413 {
414 ccount = uc_fread(read_buffer_start,1,READ_LINE_LEN,input_fp);
415 if(ccount < READ_LINE_LEN) eocf = 1;
416 read_buffer = read_buffer_start;
417 } else
418 {
419 if(read_buffer + last_consumed >= read_buffer_high_water && !eocf)
420 {
421 unused = READ_LINE_LEN-(read_buffer-read_buffer_start)-last_consumed;
422
423 memmove(read_buffer_start,read_buffer+last_consumed,unused);
424 ccount = uc_fread(read_buffer_start+unused,1,READ_LINE_LEN - unused,input_fp);
425 if(ccount < READ_LINE_LEN - unused) eocf = 1;
426 read_buffer = read_buffer_start;
427 ccount += unused;
428 } else
429 {
430 read_buffer += last_consumed;
431 ccount -= last_consumed;
432 }
433 }
434
435 retval = ccount;
436
437 if(ccount > 0 && stype != BINARY)
438 {
439 retval = find_next_LF(read_buffer,ccount);
440 last_consumed = retval + (ccount > retval ? 1 : 0); // add lf
441 #ifdef WIN32
442 if(retval && read_buffer[retval - 1] == '\r') {
443 retval--;
444 if(retval && read_buffer[retval - 1] == '\r') retval--; /* There might be two CRs? */
445 }
446 #endif
447 } else
448 {
449 last_consumed = 0;
450 }
451
452
453 if(ccount == 0)
454 {
455 if(fclose(input_fp))
456 {
457 panic("Error closing file",files->name,strerror(errno));
458 }
459 current_file = current_file->next;
460 if(current_file != NULL)
461 {
462 if(current_file->name[0] == '-' && !current_file->name[1])
463 {
464 input_fp = stdin;
465 current_file->name = "(stdin)";
466 } else
467 {
468 input_fp = open_input_stream(current_file->name,stype);
469 if(stype == BINARY)
470 {
471 current_file_offset = 0;
472 }
473 }
474 eocf = 0;
475 last_consumed = 0;
476 current_file_name = current_file->name;
477 current_file->lineno = 0;
478 current_file_offset = 0;
479 } else
480 {
481 retval = -1;
482 }
483 } else
484 {
485 if(stype != BINARY) current_file->lineno++;
486 }
487 } while(ccount == 0 && current_file != NULL);
488
489 if(ccount > 0)
490 {
491 current_file_lineno = current_file->lineno;
492 }
493 return retval;
494 }
495
496 /* calculate field count from line containing separated fields */
497 int
get_field_count(uint8_t quote,uint8_t * type,uint8_t * line)498 get_field_count(uint8_t quote, uint8_t *type, uint8_t *line)
499 {
500 int inside_quote = 0;
501 int fields = 0;
502 register uint8_t *p = line;
503
504 if(type[0] != SEPARATED) return 0;
505
506 #ifdef WIN32
507 if (*p != '\n' || *p != '\r') fields++; /* at least one */
508 #else
509 if (*p != '\n') fields++; /* at least one */
510 #endif
511
512
513 #ifdef WIN32
514 while(*p != '\n' && *p != '\r')
515 #else
516 while(*p != '\n')
517 #endif
518 {
519 if(*p == type[1] && !inside_quote)
520 {
521 fields++;
522 if(type[2] == '*') while(*p == type[1]) p++;
523 }
524
525 if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && inside_quote && quote)
526 {
527 p++;
528 } else if(*p == quote && quote)
529 {
530 inside_quote = !inside_quote;
531 }
532 #ifdef WIN32
533 if(*p != '\n' && *p != '\r') p++;
534 #else
535 if(*p != '\n') p++;
536 #endif
537 }
538 return fields;
539 }
540
541 /* returns a pointer for one field of the fixed length record */
542 /* note first position is 1 */
543 uint8_t *
get_fixed_field(int position,int length,int line_length,uint8_t * line)544 get_fixed_field(int position, int length, int line_length,uint8_t *line)
545 {
546 register uint8_t *t,*s;
547
548 if(length >= field_buffer_size - 1)
549 {
550 field_buffer_size = length * 2;
551 field_buffer = xrealloc(field_buffer,field_buffer_size);
552 }
553
554 if(position <= line_length && position)
555 {
556 position--; /* to get first as zero */
557 s = line + position;
558 t = field_buffer;
559 memcpy(t,s,length);
560 t[length] = 0;
561 } else
562 {
563 field_buffer[0] = 0;
564 }
565 return field_buffer;
566 }
567
568 /* returns pointer to field in separated record */
569 uint8_t *
get_separated_field(int position,uint8_t quote,char * type,uint8_t * line)570 get_separated_field(int position, uint8_t quote,char *type, uint8_t *line)
571 {
572 register uint8_t *p = line;
573 int fieldno = 1;
574 int inside_quote = 0;
575 register int i = 0;
576
577 #ifdef WIN32
578 while(*p != '\n' && *p != '\r' && fieldno <= position)
579 #else
580 while(*p != '\n' && fieldno <= position)
581 #endif
582 {
583 if(*p == type[1] && !inside_quote)
584 {
585 fieldno++;
586 if(type[2] == '*') while(*p == type[1]) p++;
587 }
588
589 if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && inside_quote && quote)
590 {
591 p++;
592 } else if(*p == quote && quote)
593 {
594 inside_quote = !inside_quote;
595 if(inside_quote && p[1]) p++;
596 }
597
598 if(fieldno == position)
599 {
600 if(*p == type[1] || (*p == quote && quote))
601 {
602 if(inside_quote)
603 {
604 field_buffer[i] = *p;
605 i++;
606 }
607 } else
608 {
609 field_buffer[i] = *p;
610 i++;
611 }
612
613 if(i >= field_buffer_size - 1)
614 {
615 field_buffer_size = i * 2;
616 field_buffer = xrealloc(field_buffer,field_buffer_size);
617 }
618 }
619 #ifdef WIN32
620 if(*p != '\n' && *p != '\r') p++;
621 #else
622 if(*p != '\n') p++;
623 #endif
624 }
625 field_buffer[i] = 0;
626 return field_buffer;
627 }
628
629
630
631 /* read input stream once to read_buffer
632 if allready read (orig_ccount > -1)
633 reset pointrs and ccount
634 */
635 int
init_guessing()636 init_guessing()
637 {
638 if(orig_ccount == -1)
639 {
640 ccount = uc_fread(read_buffer_start,1,READ_LINE_LEN,input_fp);
641 orig_ccount = ccount;
642 if(ccount < READ_LINE_LEN) eocf = 1;
643 } else
644 {
645 ccount = orig_ccount;
646 }
647 read_buffer = read_buffer_start;
648 last_consumed = 0;
649
650 return ccount;
651 }
652
653 /* read one input line from buffer for guessing,
654 return line length
655 -1 if buffer consumed
656 */
657 static int
read_guess_line()658 read_guess_line()
659 {
660 int retval = -1;
661
662 read_buffer += last_consumed;
663 ccount -= last_consumed;
664
665 if(ccount > 0)
666 {
667 retval = find_next_LF(read_buffer,ccount);
668 last_consumed = retval + (ccount > retval ? 1 : 0);
669 current_file->lineno++;
670 current_file_lineno = current_file->lineno;
671 #ifdef WIN32
672 if(retval && read_buffer[retval - 1] == '\r') {
673 retval--;
674 if(retval && read_buffer[retval - 1] == '\r') retval--; /* There might be two CRs? */
675 }
676 #endif
677 }
678 return retval;
679 }
680
681 /* guessing done, reset ccount and pointer
682 */
683 static void
reset_guessing()684 reset_guessing()
685 {
686 ccount = orig_ccount;
687 read_buffer = read_buffer_start;
688 last_consumed = 0;
689 current_file_lineno = 0;
690 current_file->lineno = 0;
691 current_file_offset = 0;
692 }
693
694 /* calculates votes for record, length has the buffer len and buffer
695 contains the line to be examined
696 */
697 int
vote_record(uint8_t quote,char * type,int header,struct record * record,int length)698 vote_record(uint8_t quote,char *type,int header,struct record *record,int length)
699 {
700 register struct id *i = record->i;
701 int vote = 0,len;
702 int ids = 0;
703
704 while(i != NULL)
705 {
706 ids++;
707 switch(type[0])
708 {
709 case FIXED_LENGTH:
710 #ifdef HAVE_REGEX
711 if(i->regexp)
712 {
713 uint8_t *field;
714 field = get_fixed_field(i->position,length - i->position + 1,length,read_buffer);
715 if(length >= i->position && regexec(&i->reg,field,(size_t) 0, NULL, 0) == 0) vote++;
716 } else
717 #endif
718 {
719 if(strncmp(i->key,&read_buffer[i->position -1],i->length) == 0) vote++;
720 }
721 break;
722 case SEPARATED:
723 if(header && current_file_lineno == 1) // forgive header lines
724 {
725 vote++;
726 } else
727 {
728 #ifdef HAVE_REGEX
729 if(i->regexp)
730 {
731 if(regexec(&i->reg,get_separated_field(i->position,quote,type,read_buffer),(size_t) 0, NULL, 0) == 0) vote++;
732 } else
733 #endif
734 {
735 if(strcmp(i->key,get_separated_field(i->position,quote,type,read_buffer)) == 0) vote++;
736 }
737 }
738 break;
739 case BINARY:
740 #ifdef HAVE_REGEX
741 if(i->regexp)
742 {
743 uint8_t *field;
744 int flen = 64;
745
746 flen = length - i->position + 1 < flen ? length - i->position + 1 : flen;
747 field = get_fixed_field(i->position,flen,length,read_buffer);
748
749 if(length >= i->position && regexec(&i->reg,field,(size_t) 0, NULL, 0) == 0) vote++;
750 } else
751 #endif
752 {
753 if(memcmp(i->key,&read_buffer[i->position - 1],i->length) == 0) vote++;
754 }
755 break;
756 }
757 i = i->next;
758 }
759 if(vote || record->i == NULL) /* if keys are ok or missing, then check line length */
760 {
761 switch(type[0])
762 {
763 case FIXED_LENGTH:
764 if(((record->arb_length == RL_MIN || record->length_field != NULL) && record->length <= length) ||
765 (record->arb_length == RL_STRICT && record->length == length)) vote++;
766 break;
767 case SEPARATED:
768 len = get_field_count(quote,type,read_buffer);
769 if((record->arb_length == RL_STRICT && record->length == len) ||
770 (record->arb_length == RL_MIN && record->length <= len)) vote++;
771 break;
772 case BINARY:
773 if(((vote == ids && ids) || !ids) && record->length <= length) vote++; /* exact binary length cannot be checked */
774 break;
775 }
776 }
777
778 if(vote == ids + 1) /* every id and line length must match to get a vote */
779 {
780 return 1;
781 } else
782 {
783 return 0;
784 }
785 }
786
787
788 /* calculates votes for one input line */
789 void
vote(int bindex,int line_length)790 vote(int bindex,int line_length)
791 {
792 struct structure *s = structure;
793 struct record *r;
794 int votes,total_votes = 0;
795
796 while(s != NULL)
797 {
798 r = s->r;
799 votes = 0;
800 if(s->vote == bindex && s->type[0] != BINARY) // check only structures having all records matched so far
801 {
802 while(r != NULL && !votes)
803 {
804 votes = vote_record(s->quote,s->type,s->header,r,line_length);
805 s->vote += votes; /* only one vote per line */
806 r = r->next;
807 }
808 }
809 total_votes += votes;
810 s = s->next;
811 }
812 if(!total_votes && no_matching_lines < NO_MATCH_LINES)
813 {
814 no_matching_lines++;
815 fprintf(stderr,"%s: Line %ld in \'%s\' does not match, line length = %d\n",program,current_file->lineno,current_file->name,line_length);
816 }
817 }
818
819 void
vote_binary(int buffer_size)820 vote_binary(int buffer_size)
821 {
822 struct structure *s = structure;
823 struct record *r;
824
825 while(s != NULL)
826 {
827 r = s->r;
828 if(s->type[0] == BINARY) // check only structures having all records matched so far
829 {
830 while(r != NULL && !s->vote)
831 {
832 if(r->i != NULL)
833 {
834 if(vote_record(s->quote,s->type,s->header,r,buffer_size)) s->vote = 1;
835 }
836 r = r->next;
837 }
838 }
839 s = s->next;
840 }
841 }
842
843
844 /* calculates votes for structures */
845 /* returns pointer for structure name having all lines/blocks matched, in other case NULL */
846 /* votes has the vote count hat must mach */
847 char *
check_votes(int votes)848 check_votes(int votes)
849 {
850 struct structure *s = structure;
851 char *winner = NULL;
852 int errors = 0;
853
854 while(s != NULL && votes)
855 {
856 if(s->vote == votes)
857 {
858 if(winner == NULL)
859 {
860 winner = s->name;
861 } else
862 {
863 if(!errors)
864 {
865 fprintf(stderr,"%s: Input data matches several structures: \'%s\'",program,winner);
866 }
867 fprintf(stderr," \'%s\'",s->name);
868 errors++;
869 }
870 }
871 s->vote = 0;
872 s = s->next;
873 }
874 if(errors)
875 {
876 fprintf(stderr,"\n");
877 winner = NULL;
878 }
879 return winner;
880 }
881
882
883 /* guesses using binary only
884 */
885 char *
guess_binary_structure()886 guess_binary_structure()
887 {
888 int buffer_size;
889 char *ret = NULL;
890
891 if(!max_binary_record_length) // no binary structs
892 {
893 //file_to_text(input_fp);
894 return NULL;
895 }
896
897 buffer_size = init_guessing();
898
899 if(buffer_size > 0)
900 {
901 vote_binary(buffer_size);
902 ret = check_votes(1);
903 reset_guessing();
904 }
905
906 return ret;
907 }
908
909
910
911 /* tries guess the input file structure */
912 /* returns pointer to the name of guessed structure */
913
914 char *
guess_structure()915 guess_structure()
916 {
917 int memory_used = 0;
918 int len;
919
920 len = init_guessing();
921
922 if(len <= 0) return NULL;
923
924 do
925 {
926 len = read_guess_line();
927 if(len != -1)
928 {
929 memory_used += last_consumed;
930 vote(guess_lines,len);
931 guess_lines++;
932 }
933 } while(len != -1 && guess_lines < GUESS_LINES && memory_used < GUESS_BUFFER);
934
935 reset_guessing();
936
937 return check_votes(guess_lines);
938 }
939
940 /* start write to write buffer */
941 void
start_write()942 start_write()
943 {
944 write_pos = write_buffer;
945 }
946
947 /* write uint8_t to write buffer */
948 inline void
writec(uint8_t c)949 writec(uint8_t c)
950 {
951 *write_pos = c;
952
953 if(write_pos == write_buffer_end)
954 {
955 int written = write_buffer_end - write_buffer;
956
957 write_buffer_size = write_buffer_size * 2;
958 write_buffer = xrealloc(write_buffer,write_buffer_size);
959 write_pos = write_buffer + written;
960 write_buffer_end = write_buffer + (write_buffer_size - 1);
961 }
962
963 write_pos++;
964 }
965
966
967 /* write string to write buffer */
968 inline void
writes(uint8_t * string)969 writes(uint8_t *string)
970 {
971 register uint8_t *s = string;
972
973 if(s != NULL)
974 {
975 while(*s)
976 {
977 writec(*s);
978 s++;
979 }
980 }
981 }
982
983 void
flush_write()984 flush_write()
985 {
986 size_t bytes;
987
988 bytes = write_pos - write_buffer;
989
990 if(fwrite(write_buffer,1,bytes,output_fp) != bytes)
991 {
992 panic("Error writing to",output_file,NULL);
993 }
994 }
995
996 void
print_raw(int size,uint8_t * buffer,int stype)997 print_raw(int size, uint8_t *buffer,int stype)
998 {
999 if(fwrite(buffer,1,size,default_output_fp) != size)
1000 {
1001 panic("Error writing to",default_output_file,NULL);
1002 }
1003 if(stype != BINARY) fputc('\n',default_output_fp);
1004 }
1005
1006
1007
1008 /* prints arbitrary text */
1009 /* text can contain %-directives (no %d,%D, or %n) */
1010 void
print_text(struct structure * s,struct record * r,uint8_t * buffer)1011 print_text(struct structure *s, struct record *r,uint8_t *buffer)
1012 {
1013 register uint8_t *text = buffer;
1014 char num[64];
1015
1016 if(text == NULL) return;
1017 if(r != NULL && (r->o == no_output || r->o == raw)) return;
1018 if(s->o == no_output || s->o == raw) return;
1019
1020 start_write();
1021
1022 while(*text)
1023 {
1024 if(*text == '%' && text[1])
1025 {
1026 text++;
1027 switch(*text)
1028 {
1029 case 'f':
1030 writes(current_file_name);
1031 break;
1032 case 's':
1033 writes(s->name);
1034 break;
1035 case 'r':
1036 if(r != NULL) writes(r->name);
1037 break;
1038 case 'o':
1039 sprintf(num,"%ld",current_file_lineno);
1040 writes(num);
1041 break;
1042 case 'O':
1043 sprintf(num,"%ld",current_total_lineno);
1044 writes(num);
1045 break;
1046 case 'I':
1047 sprintf(num,"%lld",current_offset);
1048 writes(num);
1049 break;
1050 case 'i':
1051 sprintf(num,"%lld",current_file_offset);
1052 writes(num);
1053 break;
1054 case 'g':
1055 if(r->level && r->level->group_name) writes(r->level->group_name);
1056 break;
1057 case 'n':
1058 if(r->level && r->level->element_name) writes(r->level->element_name);
1059 break;
1060 case '%':
1061 writec('%');
1062 break;
1063 default:
1064 writec('%');
1065 writec(*text);
1066 break;
1067 }
1068 } else
1069 {
1070 writec(*text);
1071 }
1072 text++;
1073 }
1074 flush_write();
1075 }
1076
1077
1078 /* prints a header text */
1079 /* text can contain only one %-directive %n */
1080 /* return values: */
1081 /* 1 - Header is printed or header should not be printed */
1082 /* 0 - Header is not printed and should be printed */
1083 int
print_header(struct structure * s,struct record * r)1084 print_header(struct structure *s, struct record *r)
1085 {
1086 struct print_field *pf = r->pf;
1087
1088 if(r->o == no_output || r->o == raw || r->o->header == NULL) return 1; /* no header for this run */
1089
1090 if(pf == NULL) return 0; /* no printable fields for this record */
1091
1092 start_write();
1093
1094 while(pf != NULL)
1095 {
1096 char *text = r->o->header;
1097 while(*text)
1098 {
1099 if(*text == '%' && text[1])
1100 {
1101 text++;
1102 switch(*text)
1103 {
1104 case 'n':
1105 writes(pf->f->name);
1106 break;
1107 default:
1108 writec('%');
1109 writec(*text);
1110 break;
1111 }
1112 } else
1113 {
1114 writec(*text);
1115 }
1116 text++;
1117 }
1118 if(pf->next != NULL && r->o->separator != NULL) writes(r->o->separator);
1119 pf = pf->next;
1120 }
1121 writes(r->o->record_trailer);
1122 flush_write();
1123
1124 return 1; /* header printed */
1125 }
1126
1127 /* returns pointer to next input line
1128
1129 returns NULL if no more lines
1130 length will be written to len
1131 */
1132 uint8_t *
get_input_line(int * len,int stype)1133 get_input_line(int *len,int stype)
1134 {
1135 uint8_t *ret = NULL;
1136
1137 *len = -1;
1138
1139 do
1140 {
1141 if(current_file != NULL)
1142 {
1143 *len = read_input_line(stype);
1144 ret = read_buffer;
1145 }
1146 if(*len == -1)
1147 {
1148 ret = NULL;
1149 } else
1150 {
1151 current_total_lineno++;
1152 }
1153 } while(current_file_lineno == 1 && headers == HEADER_ALL && current_total_lineno > 1);
1154
1155 return ret;
1156 }
1157
1158 /* bpositions will be updated according current input buffer
1159 * the real length of bytes consumed returned
1160 */
1161
update_field_positions(char * type,uint8_t quote,struct record * r,int len,uint8_t * buffer)1162 size_t update_field_positions(char *type,uint8_t quote,struct record *r,int len,uint8_t *buffer)
1163 {
1164 register uint8_t *p = buffer;
1165 uint8_t *field_start;
1166 register int inside_quote = 0;
1167 struct field *f = r->f;
1168 int var_record_length;
1169 int var_field_length = 0;
1170 int cur_pos,var_field_passed;
1171 size_t retval;
1172
1173 retval = last_consumed;
1174
1175 if(r->length_field) // If dynamic length
1176 {
1177 start_write();
1178 field_start = write_pos;
1179 switch(type[0])
1180 {
1181 case BINARY:
1182 print_binary_field('d',r->length_field,buffer);
1183 break;
1184 case FIXED_LENGTH:
1185 print_fixed_field('d',r->length_field,buffer);
1186 break;
1187 }
1188 writec(0); // end of string
1189 sscanf(field_start,"%d",&var_record_length);
1190 var_record_length += r->var_length_adjust;
1191 if(var_record_length >= len) var_record_length = len - 1;
1192 var_field_length = var_record_length - r->length;
1193 if(var_field_length < 0) var_field_length = 0;
1194 }
1195
1196 switch(type[0])
1197 {
1198 case FIXED_LENGTH:
1199 if(r->arb_length == RL_MIN)
1200 {
1201 while(f != NULL)
1202 {
1203 if(f->next == NULL) // check last field existense
1204 {
1205 if(len > f->position)
1206 {
1207 f->bposition = f->position;
1208 } else
1209 {
1210 f->bposition = -1;
1211 }
1212 }
1213 f = f->next;
1214 }
1215 }
1216 case BINARY: // no break here
1217 if(type[0] == BINARY)
1218 {
1219 current_file->lineno++;
1220 current_file_lineno = current_file->lineno;
1221 }
1222
1223 if(r->length_field) // If dynamic length
1224 {
1225 if (type[0] == BINARY) retval = var_record_length;
1226 cur_pos = 0;
1227 var_field_passed = 0;
1228 while(f != NULL)
1229 {
1230 if(var_field_passed && f->const_data == NULL)
1231 {
1232 f->bposition = cur_pos;
1233 cur_pos+=f->length;
1234 }
1235
1236 if(f->var_length)
1237 {
1238 f->length = var_field_length;
1239 cur_pos = f->bposition + f->length;
1240 var_field_passed = 1;
1241 }
1242 f = f->next;
1243 }
1244 } else
1245 {
1246 if (type[0] == BINARY) retval = r->length;
1247 }
1248 break;
1249 case SEPARATED:
1250 while(f != NULL)
1251 {
1252 f->bposition = -1;
1253 f = f->next;
1254 }
1255
1256 f = r->f;
1257
1258 #ifdef WIN32
1259 while(*p != '\n' && *p != '\r' && f != NULL)
1260 #else
1261 while(*p != '\n' && f != NULL)
1262 #endif
1263 {
1264 if(p == buffer) // first
1265 {
1266 if(*p != type[1]) f->bposition = 0;
1267 f = f->next;
1268 }
1269 if((*p == type[1] && !inside_quote))
1270 {
1271 p++;
1272 if(type[2] == '*') while(*p == type[1]) p++;
1273 if(*p != type[1])
1274 {
1275 f->bposition = (int) (p - buffer);
1276 } else
1277 {
1278 p--;
1279 }
1280 f = f->next;
1281 }
1282 if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && inside_quote && quote)
1283 {
1284 p++;
1285 } else if(*p == quote && quote)
1286 {
1287 inside_quote = !inside_quote;
1288 }
1289 #ifdef WIN32
1290 if(*p != '\n' && *p != '\r') p++;
1291 #else
1292 if(*p != '\n') p++;
1293 #endif
1294 }
1295 break;
1296 }
1297 return retval;
1298 }
1299
1300 /* check which record applies to current line */
1301 struct record *
select_record(struct structure * s,int length,uint8_t * buffer)1302 select_record(struct structure *s,int length,uint8_t *buffer)
1303 {
1304 register struct record *r;
1305
1306 r = s->r;
1307
1308 while(r != NULL)
1309 {
1310 if(vote_record(s->quote,s->type,s->header,r,length)) return r;
1311 r = r->next;
1312 }
1313
1314 return NULL;
1315 }
1316
1317 /* write field content to pipe and read the output, returns bytes read. Output is written to pipe_output */
execute_pipe(uint8_t * input,int input_length,struct pipe * p)1318 int execute_pipe(uint8_t *input,int input_length,struct pipe *p)
1319 {
1320 int in_fds[2];
1321 int out_fds[2];
1322 pid_t pid;
1323 FILE *rfd,*wfd;
1324 int ret=0;
1325
1326 if(!input_length) return 0; // dont pipe with no data
1327
1328 #if defined(HAVE_WORKING_FORK) && defined(HAVE_DUP2) && defined(HAVE_PIPE)
1329 if (pipe(in_fds) != 0 || pipe(out_fds) != 0) panic("Cannot create pipe",strerror(errno),NULL);
1330 pid = fork();
1331 if(pid == (pid_t) 0) /* Child */
1332 {
1333 close(in_fds[1]);
1334 close(out_fds[0]);
1335 if(dup2(in_fds[0],STDIN_FILENO) == -1) panic("dup2 error",strerror(errno),NULL);
1336 close(in_fds[0]);
1337 if(dup2(out_fds[1],STDOUT_FILENO) == -1) panic("dup2 error",strerror(errno),NULL);
1338 close(out_fds[1]);
1339 if(execl(SHELL_CMD, "sh", "-c", p->command, NULL) == -1) panic("Starting a shell with execl failed",p->command,strerror(errno));
1340 _exit(EXIT_SUCCESS);
1341 } else if(pid > (pid_t) 0)
1342 {
1343 close(in_fds[0]);
1344 close(out_fds[1]);
1345 wfd = fdopen(in_fds[1],"w");
1346 if(wfd == NULL) panic("Cannot write to command",p->command,strerror(errno));
1347 rfd = fdopen(out_fds[0],"r");
1348 if(rfd == NULL) panic("Cannot read from command",p->command,strerror(errno));
1349
1350 if(fwrite(input,input_length,1,wfd) != 1) panic("Cannot write to command",p->command,strerror(errno));
1351 fclose(wfd);
1352 ret = (int) fread(pipe_output,1,PIPE_OUTPUT_LEN,rfd);
1353 fclose(rfd);
1354 if(ret && pipe_output[ret - 1] == '\n') pipe_output[ret - 1] = '\000'; // remove last linefeed
1355 } else
1356 {
1357 panic("Cannot fork",strerror(errno),NULL);
1358 }
1359 #else
1360 panic("pipe is not supported in this system",NULL,NULL);
1361 #endif
1362 return ret;
1363 }
1364
1365
1366 /* print a single fixed field */
1367 void
print_fixed_field(uint8_t format,struct field * f,uint8_t * buffer)1368 print_fixed_field(uint8_t format,struct field *f,uint8_t *buffer)
1369 {
1370 register int i = 0;
1371 register uint8_t *data;
1372 uint8_t *start = write_pos;
1373 int len = f->length;
1374
1375 if(!f->length && f->var_length) return;
1376
1377 if(f->const_data != NULL)
1378 {
1379 data = f->const_data;
1380 if (format == 't') format = 'd'; // Dont't trim consts
1381 } else
1382 {
1383 if(f->bposition < 0) return; /* last variable length field is missing */
1384 data = &buffer[f->bposition];
1385
1386 if(f->p != NULL)
1387 {
1388 len = execute_pipe(data,f->length,f->p);
1389 data = pipe_output;
1390 }
1391 }
1392
1393 switch(format)
1394 {
1395 case 'd':
1396 case 'D':
1397 case 'C':
1398 case 'e':
1399 case 'x':
1400 if(data == pipe_output)
1401 {
1402 while(i < len) writec(data[i++]);
1403 } else
1404 {
1405 if(f->length)
1406 {
1407 #ifdef WIN32
1408 while(i < len && data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
1409 #else
1410 while(i < len && data[i] != '\n' && data[i]) writec(data[i++]);
1411 #endif
1412 } else
1413 {
1414 #ifdef WIN32
1415 while(data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
1416 #else
1417 while(data[i] != '\n' && data[i]) writec(data[i++]);
1418 #endif
1419 }
1420 }
1421 break;
1422 case 't':
1423 if(data == pipe_output)
1424 {
1425 while(isblank(data[i])) i++;
1426 while(i < len) writec(data[i++]);
1427 } else
1428 {
1429 while(isblank(data[i])) i++;
1430 if(f->length)
1431 {
1432 #ifdef WIN32
1433 while(i < len && data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
1434 #else
1435 while(i < len && data[i] != '\n' && data[i]) writec(data[i++]);
1436 #endif
1437 } else
1438 {
1439 #ifdef WIN32
1440 while(data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
1441 #else
1442 while(data[i] != '\n' && data[i]) writec(data[i++]);
1443 #endif
1444 }
1445 }
1446
1447 if(write_pos > start && isspace(write_pos[-1]))
1448 {
1449 write_pos--;
1450 while(write_pos > start && (isspace(*write_pos))) write_pos--;
1451 write_pos++;
1452 }
1453 break;
1454 }
1455 }
1456
1457 /* print a single binary field
1458 if field type is ASC, fixed field printing is used
1459 */
1460 void
print_binary_field(uint8_t format,struct field * f,uint8_t * buffer)1461 print_binary_field(uint8_t format,struct field *f,uint8_t *buffer)
1462 {
1463 register uint8_t *p,*data_end;
1464 uint8_t *data,c;
1465 char *pf;
1466 static uint8_t pb[2 * FIELD_SIZE];
1467
1468 if(!f->length && f->var_length) return;
1469
1470 if(f->const_data != NULL || (f->type == F_ASC && format != 'h'))
1471 {
1472 print_fixed_field(format,f,buffer);
1473 return;
1474 }
1475
1476 if(f->p != NULL)
1477 {
1478 int i=0,len;
1479 len = execute_pipe(&buffer[f->bposition],f->length,f->p);
1480 data = pipe_output;
1481 while(i < len && data[i]) writec(data[i++]);
1482 return;
1483 }
1484
1485
1486 switch(f->type)
1487 {
1488 case F_INT:
1489 case F_UINT:
1490 case F_FLOAT:
1491 case F_DOUBLE:
1492 data = endian_and_align(&buffer[f->bposition],system_endianess,f->endianess,f->length);
1493 break;
1494 default:
1495 data = &buffer[f->bposition];
1496 break;
1497 }
1498
1499 pb[0] = 0;
1500
1501 switch(format)
1502 {
1503 case 'h':
1504 p = &buffer[f->bposition]; // original data before any endian things
1505 data_end = p + f->length;
1506 while(p < data_end)
1507 {
1508 writec('x');
1509 writec(htocb(*p));
1510 writec(htocl(*p));
1511 p++;
1512 }
1513 break;
1514 case 'd':
1515 case 't':
1516 case 'D':
1517 case 'C':
1518 case 'x':
1519 switch(f->type)
1520 {
1521 case F_CHAR:
1522 writec(*data);
1523 break;
1524 case F_INT:
1525 switch(f->length)
1526 {
1527 case 1:
1528 pf = format == 'x' ? "%x" : "%i";
1529 sprintf(pb,pf,(int) *(int8_t *) data);
1530 break;
1531 case 2:
1532 pf = format == 'x' ? "%x" : "%i";
1533 sprintf(pb,pf,(int) *(int16_t *) data);
1534 break;
1535 case 4:
1536 pf = format == 'x' ? "%lx" : "%li";
1537 sprintf(pb,pf,(long int) *(int32_t *) data);
1538 break;
1539 case 8:
1540 pf = format == 'x' ? "%llx" : "%lli";
1541 sprintf(pb,pf,(long long int) *(int64_t *) data);
1542 break;
1543 }
1544 writes(pb);
1545 break;
1546 case F_UINT:
1547 switch(f->length)
1548 {
1549 case 1:
1550 pf = format == 'x' ? "%x" : "%u";
1551 sprintf(pb,pf,(unsigned int) *(uint8_t *) data);
1552 break;
1553 case 2:
1554 pf = format == 'x' ? "%x" : "%u";
1555 sprintf(pb,pf,(unsigned int) *(uint16_t *) data);
1556 break;
1557 case 4:
1558 pf = format == 'x' ? "%lx" : "%lu";
1559 sprintf(pb,pf,(unsigned long int) *(uint32_t *) data);
1560 break;
1561 case 8:
1562 pf = format == 'x' ? "%llx" : "%llu";
1563 sprintf(pb,pf,(unsigned long long int) *(uint64_t *) data);
1564 break;
1565 }
1566 writes(pb);
1567 break;
1568 case F_FLOAT:
1569 sprintf(pb,"%f",(double) *(float *) data);
1570 writes(pb);
1571 break;
1572 case F_DOUBLE:
1573 sprintf(pb,"%f",(double) *(double *) data);
1574 writes(pb);
1575 break;
1576 case F_BCD:
1577 p = data;
1578 data_end = p + f->length;
1579 switch(f->endianess)
1580 {
1581 case F_BIG_ENDIAN:
1582 do
1583 {
1584 c = bcdtocb(*p);
1585 if(c)
1586 {
1587 writec(c);
1588 c = bcdtocl(*p);
1589 if(c) writec(c);
1590 }
1591 p++;
1592 } while(p < data_end && c);
1593 break;
1594 case F_LITTLE_ENDIAN:
1595 do
1596 {
1597 c = bcdtocl(*p);
1598 if(c)
1599 {
1600 writec(c);
1601 c = bcdtocb(*p);
1602 if(c) writec(c);
1603 }
1604 p++;
1605 } while(p < data_end && c);
1606 break;
1607 }
1608 break;
1609 case F_HEX:
1610 p = data;
1611 data_end = p + f->length;
1612 switch(f->endianess)
1613 {
1614 case F_BIG_ENDIAN:
1615 while(p < data_end)
1616 {
1617 writec(htocb(*p));
1618 writec(htocl(*p));
1619 p++;
1620 }
1621 break;
1622 case F_LITTLE_ENDIAN:
1623 p += f->length - 1;
1624 while(p >= data)
1625 {
1626 writec(htocb(*p));
1627 writec(htocl(*p));
1628 p--;
1629 }
1630 break;
1631 }
1632 break;
1633 }
1634 }
1635 }
1636
1637
1638
1639 /* print a single separated field */
1640 void
print_separated_field(uint8_t format,uint8_t quote,uint8_t separator,struct field * f,uint8_t * buffer)1641 print_separated_field(uint8_t format,uint8_t quote,uint8_t separator,struct field *f,uint8_t *buffer)
1642 {
1643 register uint8_t *p;
1644 uint8_t *start;
1645 int inside_quote = 0;
1646
1647 start = write_pos;
1648
1649 if(f->const_data != NULL) // use fixed field printing for const data
1650 {
1651 print_fixed_field(format,f,buffer);
1652 } else
1653 {
1654 if(f->bposition < 0)
1655 {
1656 if(format == 'D' || format == 'C') while(write_pos - start < f->length) writec(' ');
1657 return;
1658 }
1659
1660 if(f->p != NULL)
1661 {
1662 p = &buffer[f->bposition];
1663 if(*p == quote && quote) {
1664 p++;
1665 inside_quote = 1;
1666 }
1667 #ifdef WIN32
1668 while((*p != separator || inside_quote) && *p != '\n' && *p != '\r')
1669 #else
1670 while((*p != separator || inside_quote) && *p != '\n')
1671 #endif
1672 {
1673 if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && quote)
1674 {
1675 p++;
1676 } else if(*p == quote)
1677 {
1678 if(inside_quote) inside_quote=0;
1679 }
1680 #ifdef WIN32
1681 if(*p != '\n' && *p != '\r') p++;
1682 #else
1683 if(*p != '\n') p++;
1684 #endif
1685 }
1686 int i=0,len;
1687 len = execute_pipe(&buffer[f->bposition],p - &buffer[f->bposition],f->p);
1688 while(i < len && pipe_output[i]) writec(pipe_output[i++]);
1689 return;
1690 }
1691
1692
1693 switch(format)
1694 {
1695 case 'd':
1696 case 't':
1697 case 'D':
1698 case 'C':
1699 case 'e':
1700 case 'x':
1701 p = &buffer[f->bposition];
1702 if(quote || format == 't') while(*p != separator && isblank(*p)) p++;
1703 if(*p == quote && quote)
1704 {
1705 p++;
1706 inside_quote = 1;
1707 if(format == 't') while(isblank(*p)) p++;
1708 }
1709 #ifdef WIN32
1710 while((*p != separator || inside_quote) && *p != '\n' && *p != '\r')
1711 #else
1712 while((*p != separator || inside_quote) && *p != '\n')
1713 #endif
1714 {
1715 if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && quote)
1716 {
1717 p++;
1718 } else if(*p == quote)
1719 {
1720 if(inside_quote)
1721 {
1722 if(format == 't' && isblank(write_pos[-1]))
1723 {
1724 write_pos--;
1725 while(write_pos > start && isblank(*write_pos)) write_pos--;
1726 write_pos++;
1727 }
1728 if(format == 'D' || format == 'C') while(write_pos - start == f->length) writec(' ');
1729 return;
1730 }
1731 }
1732
1733 if(format == 'C' && (write_pos - start) == f->length) return;
1734 #ifdef WIN32
1735 if(*p != '\n' && *p != '\r') {
1736 writec(*p);
1737 p++;
1738 }
1739 #else
1740 if(*p != '\n') {
1741 writec(*p);
1742 p++;
1743 }
1744 #endif
1745
1746
1747 }
1748 if(format == 't' && isspace(write_pos[-1]))
1749 {
1750 write_pos--;
1751 while(write_pos > start && isspace(*write_pos)) write_pos--;
1752 write_pos++;
1753 }
1754 if(format == 'D' || format == 'C') while(write_pos - start < f->length) writec(' ');
1755 break;
1756 }
1757 }
1758 }
1759
1760 /* search the lookup table, return the found value */
1761 uint8_t *
make_lookup(struct lookup * l,uint8_t * search)1762 make_lookup(struct lookup *l,uint8_t *search)
1763 {
1764 register uint8_t *k,*s;
1765 int search_len = strlen(search);
1766 uint8_t *ret_val = NULL;
1767 struct lookup_data *d;
1768
1769 switch(l->type)
1770 {
1771 case EXACT:
1772
1773 d = l->data[hash(search,0)];
1774
1775 while(d != NULL && ret_val == NULL)
1776 {
1777 k = d->key;
1778 s = search;
1779 while(*k == *s && *k)
1780 {
1781 k++;
1782 s++;
1783 }
1784 if(!*s && !*k) ret_val = d->value;
1785 d = d->next;
1786 }
1787 break;
1788 case LONGEST:
1789 while(search_len && ret_val == NULL)
1790 {
1791 d = l->data[hash(search,0)];
1792
1793 while(d != NULL && ret_val == NULL)
1794 {
1795 k = d->key;
1796 s = search;
1797 while(*k == *s && *k)
1798 {
1799 k++;
1800 s++;
1801 }
1802 if(!*s && !*k) ret_val = d->value;
1803 d = d->next;
1804 }
1805 search_len--;
1806 search[search_len] = 0;
1807 }
1808 break;
1809 }
1810
1811 if(ret_val == NULL) ret_val = l->default_value;
1812
1813 return ret_val;
1814 }
1815
1816 void
print_indent(uint8_t * buffer,int times)1817 print_indent(uint8_t *buffer,int times)
1818 {
1819 start_write();
1820 while(times--) writes(buffer);
1821 flush_write();
1822 }
1823
1824 /* format field with printf
1825 */
1826 #define CONV_BUF_SIZE 1048576
1827 void
make_conversion(struct format * f,uint8_t * start)1828 make_conversion(struct format *f,uint8_t *start)
1829 {
1830 static uint8_t conv_buffer[CONV_BUF_SIZE];
1831
1832 *write_pos = 0;
1833
1834 conv_buffer[0] = 0;
1835
1836 #ifdef HAVE_PRINTF_H
1837 switch(f->type)
1838 {
1839 case PA_INT:
1840 case PA_INT|PA_FLAG_SHORT:
1841 case PA_CHAR:
1842 case PA_WCHAR:
1843 snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atoi(start));
1844 break;
1845 case PA_INT|PA_FLAG_LONG:
1846 snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atol(start));
1847 break;
1848 case PA_INT|PA_FLAG_LONG_LONG:
1849 snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atoll(start));
1850 break;
1851 case PA_FLOAT:
1852 case PA_DOUBLE:
1853 snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atof(start));
1854 break;
1855 case PA_DOUBLE|PA_FLAG_LONG_DOUBLE:
1856 snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,strtold(start,NULL));
1857 break;
1858 case PA_WSTRING:
1859 case PA_STRING:
1860 case PA_POINTER:
1861 snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,start);
1862 break;
1863 default:
1864 return;
1865 }
1866 write_pos = start;
1867 writes(conv_buffer);
1868 #endif
1869 }
1870
1871
1872
1873 /* print fields */
1874 /* returns the count of fields actually printed */
1875 int
print_fields(struct structure * s,struct record * r,uint8_t * buffer)1876 print_fields(struct structure *s, struct record *r,uint8_t *buffer)
1877 {
1878 uint8_t num[64];
1879 int max_justify_len = 0;
1880 register uint8_t *d;
1881 uint8_t *f;
1882 int i;
1883 uint8_t justify = LEFT_JUSTIFY;
1884 uint8_t *indent,*separator;
1885 uint8_t *data_start,*rep_start = NULL,*rep_pos = NULL;
1886 uint8_t *field_start;
1887 uint8_t *lookup_value;
1888 int retval = 0;
1889 int replacing,just_replaced;
1890 int lookup_len;
1891 struct print_field *pf = r->pf;
1892 struct output *o;
1893
1894 while(pf != NULL) {
1895 pf->justify_length = -1;
1896 pf = pf->next;
1897 }
1898
1899 pf = r->pf;
1900
1901 start_write();
1902
1903 while(pf != NULL)
1904 {
1905 o = pf->f->o ? pf->f->o : r->o;
1906
1907 if(o != no_output)
1908 {
1909 justify = o->justify;
1910 d = o->data;
1911 data_start = write_pos;
1912 pf->data = data_start;
1913 pf->empty = 1;
1914 replacing = 0;
1915 just_replaced = 0;
1916 lookup_value = NULL;
1917
1918 if(pf->f->lookup != NULL) d = o->lookup;
1919
1920 if(o->hex_cap)
1921 {
1922 bcd_to_ascii = bcd_to_ascii_cap;
1923 hex_to_ascii = hex_to_ascii_cap;
1924 } else
1925 {
1926 bcd_to_ascii = bcd_to_ascii_low;
1927 hex_to_ascii = hex_to_ascii_low;
1928 }
1929
1930 while(*d)
1931 {
1932 if(justify == *d)
1933 {
1934 if(justify != LEFT_JUSTIFY && justify != RIGHT_JUSTIFY && pf->justify_length == -1 && !replacing)
1935 {
1936 pf->justify_length = (int) (write_pos - data_start);
1937 if(pf->justify_length > max_justify_len)
1938 {
1939 max_justify_len = pf->justify_length;
1940 }
1941 }
1942 }
1943
1944 if(*d == '%')
1945 {
1946 d++;
1947 switch(*d)
1948 {
1949 case 'f':
1950 writes(current_file_name);
1951 break;
1952 case 's':
1953 writes(s->name);
1954 break;
1955 case 'r':
1956 if(r != NULL) writes(r->name);
1957 break;
1958 case 'o':
1959 sprintf(num,"%ld",current_file_lineno);
1960 writes(num);
1961 break;
1962 case 'O':
1963 sprintf(num,"%ld",current_total_lineno);
1964 writes(num);
1965 break;
1966 case 'I':
1967 sprintf(num,"%lld",current_offset);
1968 writes(num);
1969 break;
1970 case 'i':
1971 sprintf(num,"%lld",current_file_offset);
1972 writes(num);
1973 break;
1974 case 'p':
1975 if(pf->f->const_data == NULL)
1976 {
1977 sprintf(num,"%d",pf->f->position + (s->type[0] != SEPARATED ? 1 : 0));
1978 writes(num);
1979 }
1980 break;
1981 case '%':
1982 writec('%');
1983 break;
1984 case 'n':
1985 writes(pf->f->name);
1986 break;
1987 case 'l':
1988 case 'L':
1989 if(pf->f->lookup != NULL)
1990 {
1991 if(lookup_value == NULL)
1992 {
1993 field_start = write_pos; // misuse write buffer for temp space for field value
1994 switch(s->type[0]) // write trimmed data for search key
1995 {
1996 case FIXED_LENGTH:
1997 print_fixed_field('t',pf->f,buffer);
1998 break;
1999 case SEPARATED:
2000 print_separated_field('t',s->quote,s->type[1],pf->f,buffer);
2001 break;
2002 case BINARY:
2003 print_binary_field('t',pf->f,buffer);
2004 break;
2005 }
2006 writec(0);
2007 lookup_value = make_lookup(pf->f->lookup,field_start);
2008 write_pos = field_start; // restore write buffer
2009 }
2010 } else if(lookup_value == NULL)
2011 {
2012 lookup_value = "";
2013 }
2014
2015 writes(lookup_value);
2016
2017 if(*d == 'L')
2018 {
2019 lookup_len = strlen(lookup_value);
2020 while(lookup_len++ < pf->f->length) writec(' ');
2021 }
2022 break;
2023 case 'h':
2024 if(s->type[0] == BINARY) print_binary_field(*d,pf->f,buffer);
2025 break;
2026 case 'd':
2027 case 't':
2028 case 'D':
2029 case 'C':
2030 case 'e':
2031 case 'x':
2032 field_start = write_pos;
2033
2034 if(pf->f->rep != NULL && !replacing) // start possible replacing
2035 {
2036 replacing = 1;
2037 just_replaced = 1;
2038 rep_pos = d;
2039 rep_start = write_pos;
2040 d = pf->f->rep->value;
2041 break; // end case
2042 }
2043
2044 switch(s->type[0])
2045 {
2046 case FIXED_LENGTH:
2047 print_fixed_field(*d,pf->f,buffer);
2048 break;
2049 case SEPARATED:
2050 print_separated_field(*d,s->quote,s->type[1],pf->f,buffer);
2051 break;
2052 case BINARY:
2053 print_binary_field(*d,pf->f,buffer);
2054 break;
2055 }
2056
2057 if(pf->f->f != NULL) make_conversion(pf->f->f,field_start);
2058
2059 if(!o->print_empty)
2060 {
2061 f = field_start;
2062 while(f < write_pos)
2063 {
2064 if(strchr(o->empty_chars,*f) == NULL)
2065 {
2066 pf->empty = 0;
2067 f = write_pos; // to stop while loop
2068 }
2069 f++;
2070 }
2071 }
2072 if(*d == 'e') write_pos = field_start;
2073 break;
2074 default:
2075 writec('%');
2076 writec(*d);
2077 break;
2078 }
2079 if(!just_replaced) d++;
2080 } else
2081 {
2082 writec(*d);
2083 d++;
2084 }
2085
2086
2087 if(replacing && !*d)
2088 {
2089 d = rep_pos;
2090 d++;
2091 replacing = 0;
2092 if(*rep_pos == 'D' || *rep_pos == 'C')
2093 {
2094 if(write_pos - rep_start < pf->f->length)
2095 {
2096 while(write_pos - rep_start < pf->f->length) writec(' ');
2097 } else if(write_pos - rep_start > pf->f->length)
2098 {
2099 write_pos = rep_start + pf->f->length;
2100 }
2101 }
2102 }
2103
2104 if(just_replaced) just_replaced = 0;
2105 }
2106
2107 if(justify == RIGHT_JUSTIFY)
2108 {
2109 pf->justify_length = (int) (write_pos - data_start);
2110 if (pf->justify_length > max_justify_len)
2111 {
2112 max_justify_len = pf->justify_length;
2113 }
2114 }
2115 writec(0); // end of data marker
2116 }
2117 pf = pf->next;
2118 }
2119
2120 pf = r->pf;
2121
2122 /* count the number of fields to be printed */
2123 /* we need this before hand, because we must know if the is att least */
2124 /* one field to be printed, then all separators must be printed */
2125 while(pf != NULL)
2126 {
2127 o = pf->f->o ? pf->f->o : r->o;
2128
2129 if(o != no_output)
2130 {
2131 if(o->print_empty || !pf->empty)
2132 {
2133 retval++;
2134 }
2135 }
2136 pf = pf->next;
2137 }
2138
2139 pf = r->pf;
2140
2141 while(pf != NULL && retval)
2142 {
2143 o = pf->f->o ? pf->f->o : r->o;
2144
2145 if(o != no_output)
2146 {
2147 separator = o->separator;
2148 indent = o->indent;
2149 if(o->print_empty || !pf->empty)
2150 {
2151 if(indent != NULL)
2152 {
2153 if(r->level)
2154 {
2155 int i;
2156 i = get_indent_depth(r->level->level);
2157 while(i--) fputs(indent,output_fp);
2158 } else
2159 {
2160 fputs(indent,output_fp);
2161 fputs(indent,output_fp);
2162 }
2163 }
2164 if((justify != LEFT_JUSTIFY && justify != RIGHT_JUSTIFY && max_justify_len) || justify == RIGHT_JUSTIFY)
2165 {
2166 i = max_justify_len - pf->justify_length;
2167 if(pf->justify_length > -1 && i)
2168 {
2169 while(i > JUSTIFY_STRING - 1)
2170 {
2171 justify_string[JUSTIFY_STRING - 1] = (uint8_t) 0;
2172 fputs(justify_string,output_fp);
2173 i -= JUSTIFY_STRING - 1;
2174 }
2175 justify_string[i] = (uint8_t) 0;
2176 fputs(justify_string,output_fp);
2177 justify_string[i] = ' ';
2178 }
2179 }
2180 fputs(pf->data,output_fp);
2181 }
2182 if(pf->next != NULL && separator != NULL) fputs(separator,output_fp);
2183 }
2184 pf = pf->next;
2185 }
2186 return retval;
2187 }
2188
2189
2190
2191 /* make a list of printable fields
2192 if include list is non empty use names from it
2193 else use the whole field list in f */
2194 struct print_field *
make_print_list(struct include_field * fl,struct field * f)2195 make_print_list(struct include_field *fl,struct field *f)
2196 {
2197 struct print_field *ret = NULL,*c = NULL;
2198 struct field *n;
2199
2200 if(fl == NULL)
2201 {
2202 while(f != NULL)
2203 {
2204 if(strcmp(f->name,"FILLER") != 0)
2205 {
2206 if(ret == NULL)
2207 {
2208 ret = xmalloc(sizeof(struct print_field));
2209 c = ret;
2210 } else
2211 {
2212 c->next = xmalloc(sizeof(struct print_field));
2213 c = c->next;
2214 }
2215 c->f = f;
2216 c->next = NULL;
2217 }
2218 f = f->next;
2219 }
2220 } else
2221 {
2222 while(fl != NULL)
2223 {
2224 n = const_field; // first check const list
2225
2226 while(n != NULL && (strcasecmp(fl->name,n->name) != 0))
2227 {
2228 n = n->next;
2229 }
2230
2231 if(n == NULL) // if not found in const list check input fields
2232 {
2233 n = f;
2234 while(n != NULL && (strcasecmp(fl->name,n->name) != 0))
2235 {
2236 n = n->next;
2237 }
2238 }
2239
2240 if(n != NULL)
2241 {
2242 if(ret == NULL)
2243 {
2244 ret = xmalloc(sizeof(struct print_field));
2245 c = ret;
2246 } else
2247 {
2248 c->next = xmalloc(sizeof(struct print_field));
2249 c = c->next;
2250 }
2251 c->f = n;
2252 c->next = NULL;
2253 fl->found = 1;
2254 }
2255 fl = fl->next;
2256 }
2257 }
2258 return ret;
2259 }
2260
2261 /* check that all fields we found from current structure */
2262 /* return the count of unmatched fields */
2263 int
check_field_list(struct include_field * fl)2264 check_field_list(struct include_field *fl)
2265 {
2266 int ret = 0;
2267
2268 while(fl != NULL)
2269 {
2270 if(!fl->found)
2271 {
2272 if(!fl->reported) fprintf(stderr,"%s: Field '%s' was not found in the current structure\n",program,fl->name);
2273 fl->reported = 1;
2274 ret++;
2275 }
2276 fl = fl->next;
2277 }
2278 return ret;
2279 }
2280
2281 void
init_structure(struct structure * s,struct record * current_record,int length,uint8_t * buffer)2282 init_structure(struct structure *s,struct record *current_record,int length, uint8_t *buffer)
2283 {
2284 struct record *r;
2285 struct field *f;
2286 struct replace *rep;
2287 struct expression *e;
2288 int list_errors = 0;
2289
2290 r = s->r;
2291
2292 while(r != NULL)
2293 {
2294 f = r->f;
2295 while(f != NULL)
2296 {
2297 if(s->header && f->name == NULL)
2298 f->name = xstrdup(get_separated_field(f->position,s->quote,s->type,buffer));
2299
2300 rep = replace;
2301 while(rep != NULL)
2302 {
2303 if(strcasecmp(f->name,rep->field) == 0)
2304 {
2305 f->rep = rep;
2306 rep->found = 1;
2307 }
2308 rep = rep->next;
2309 }
2310
2311 e = expression;
2312 while(e != NULL)
2313 {
2314 if(strcasecmp(f->name,e->field) == 0)
2315 {
2316 e->found = 1;
2317 }
2318 e = e->next;
2319 }
2320
2321 f = f->next;
2322 }
2323 r = r->next;
2324 }
2325
2326 r = s->r;
2327
2328 while(r != NULL)
2329 {
2330 if(r->o != no_output) r->pf = make_print_list(r->o->fl,r->f);
2331 r = r->next;
2332 }
2333
2334 r = s->r;
2335
2336 while(r != NULL)
2337 {
2338 if(check_field_list(r->o->fl)) list_errors++;
2339 r = r->next;
2340 }
2341 if(list_errors) panic("Some fields from field list were not found in the current structure or constant values",s->name,NULL);
2342
2343 rep = replace;
2344 while(rep != NULL)
2345 {
2346 if(!rep->found) panic("Field to be replaced was not found in the current structure",rep->field,NULL);
2347 rep = rep->next;
2348 }
2349
2350 e = expression;
2351 while(e != NULL)
2352 {
2353 if(!e->found) panic("Field in expression was not found in the current structure",e->field,NULL);
2354 e = e->next;
2355 }
2356 }
2357
2358 void
invalid_input(char * file,long int lineno,int strict,int length,int stype)2359 invalid_input(char *file, long int lineno,int strict,int length,int stype)
2360 {
2361 if(stype == BINARY)
2362 {
2363 if(strict)
2364 {
2365 fprintf(stderr,"%s: Invalid input block in file \'%s\', offset %lld",program,file,current_file_offset);
2366 if(debug_lineno) fprintf(stderr,". Block was written to debug file");
2367 fprintf(stderr,"\n");
2368 }
2369 } else
2370 {
2371 fprintf(stderr,"%s: Invalid input line in file \'%s\', line %ld, line length = %d",program,file,lineno,length);
2372 if(debug_lineno) fprintf(stderr,", line %ld in debug file",debug_lineno);
2373 fprintf(stderr,"\n");
2374 }
2375 if(strict) panic("Using option -l does not cause program to abort in case of invalid input",NULL,NULL);
2376 }
2377
2378
2379 static int
hash_scan_expression(struct expr_list ** list,char * value,int casecmp)2380 hash_scan_expression(struct expr_list **list,char *value,int casecmp)
2381 {
2382 register struct expr_list *l = list[hash(value,0)];
2383
2384 while(l != NULL)
2385 {
2386 if(casecmp)
2387 {
2388 if(strcasecmp(l->value,value) == 0) return 1;
2389 } else
2390 {
2391 if(strcmp(l->value,value) == 0) return 1;
2392 }
2393 l = l->next;
2394 }
2395 return 0;
2396 }
2397
2398 static int
hash_scan_expression_len(struct expr_list ** list,char * value,int casecmp,size_t comp_len)2399 hash_scan_expression_len(struct expr_list **list,char *value,int casecmp,size_t comp_len)
2400 {
2401 register struct expr_list *l = list[hash(value,comp_len)];
2402
2403 while(l != NULL)
2404 {
2405 if(casecmp)
2406 {
2407 if(strncasecmp(l->value,value,l->value_len) == 0) return 1;
2408 } else
2409 {
2410 if(strncmp(l->value,value,l->value_len) == 0) return 1;
2411 }
2412 l = l->next;
2413 }
2414 return 0;
2415 }
2416
2417 static inline int
scan_expression_list(struct expr_list * l,char * value,char op,int casecmp)2418 scan_expression_list(struct expr_list *l,char *value,char op,int casecmp)
2419 {
2420 while(l != NULL)
2421 {
2422 switch(op)
2423 {
2424 case OP_START:
2425 if(casecmp)
2426 {
2427 if(strncasecmp(l->value,value,l->value_len) == 0) return 1;
2428 } else
2429 {
2430 if(strncmp(l->value,value,l->value_len) == 0) return 1;
2431 }
2432 break;
2433 case OP_EQUAL:
2434 case OP_NOT_EQUAL:
2435 if(casecmp)
2436 {
2437 if(strcasecmp(l->value,value) == 0) return 1;
2438 } else
2439 {
2440 if(strcmp(l->value,value) == 0) return 1;
2441 }
2442 break;
2443 case OP_CONTAINS:
2444 if(casecmp)
2445 {
2446 if(strcasestr(value,l->value) != NULL) return 1;
2447 } else
2448 {
2449 if(strstr(value,l->value) != NULL) return 1;
2450 }
2451 break;
2452 #ifdef HAVE_REGEX
2453 case OP_REQEXP:
2454 if(regexec(&l->reg,value,(size_t) 0, NULL, 0) == 0) return 1;
2455 break;
2456 #endif
2457 }
2458 l = l->next;
2459 }
2460 return 0;
2461 }
2462
2463 static int
full_scan_expression(struct expression * e,char * value,int casecmp)2464 full_scan_expression(struct expression *e,char *value,int casecmp)
2465 {
2466 register int i = 0;
2467
2468
2469 if(e->fast_entries)
2470 {
2471 while(i < e->fast_entries)
2472 {
2473 if(scan_expression_list(e->expr_hash[e->fast_expr_hash[i]],value,e->op,casecmp)) return 1;
2474 i++;
2475 }
2476 } else
2477 {
2478 while(i < MAX_EXPR_HASH)
2479 {
2480 if(scan_expression_list(e->expr_hash[i],value,e->op,casecmp)) return 1;
2481 i++;
2482 }
2483 }
2484 return 0;
2485 }
2486
2487 /* returns true if and = 0 and attleast one expression is true
2488 * or and = 1 and all expressions are true
2489 */
2490 int
eval_expression(struct structure * s,struct record * r,int and,int invert,int casecmp,uint8_t * buffer)2491 eval_expression(struct structure *s,struct record *r, int and,int invert, int casecmp, uint8_t *buffer)
2492 {
2493 struct expression *e = expression;
2494 int retval = 0;
2495 int prev_retval;
2496 int loop_break = 0;
2497 int expression_count = 0;
2498 struct output *o;
2499 size_t len;
2500
2501 if(e == NULL) return 0;
2502
2503 while(e != NULL && !loop_break)
2504 {
2505 if(e->f != NULL)
2506 {
2507
2508 o = e->f->o ? e->f->o : r->o;
2509 if(o != no_output && o->hex_cap)
2510 {
2511 bcd_to_ascii = bcd_to_ascii_cap;
2512 hex_to_ascii = hex_to_ascii_cap;
2513 } else
2514 {
2515 bcd_to_ascii = bcd_to_ascii_low;
2516 hex_to_ascii = hex_to_ascii_low;
2517 }
2518
2519
2520 start_write();
2521 switch(s->type[0])
2522 {
2523 case FIXED_LENGTH:
2524 print_fixed_field('d',e->f,buffer);
2525 break;
2526 case SEPARATED:
2527 print_separated_field('d',s->quote,s->type[1],e->f,buffer);
2528 break;
2529 case BINARY:
2530 print_binary_field('d',e->f,buffer);
2531 break;
2532 }
2533 writec(0); // end of string
2534 prev_retval = retval;
2535
2536 switch(e->op)
2537 {
2538 case OP_START:
2539 len = e->exp_max_len;
2540 prev_retval = retval;
2541 while(len >= e->exp_min_len && prev_retval == retval)
2542 {
2543 retval += hash_scan_expression_len(e->expr_hash,write_buffer,casecmp,len);
2544 len--;
2545 }
2546 break;
2547 case OP_CONTAINS:
2548 case OP_REQEXP:
2549 retval += full_scan_expression(e,write_buffer,casecmp);
2550 break;
2551 case OP_EQUAL:
2552 retval += hash_scan_expression(e->expr_hash,write_buffer,casecmp);
2553 break;
2554 case OP_NOT_EQUAL:
2555 if(hash_scan_expression(e->expr_hash,write_buffer,casecmp) == 0) retval++;
2556 break;
2557 }
2558 }
2559 if(!and && retval)
2560 {
2561 loop_break = 1;
2562 }
2563 expression_count++;
2564 if(and && retval != expression_count)
2565 {
2566 loop_break = 1;
2567 }
2568 e = e->next;
2569 }
2570 if(and)
2571 {
2572 if (retval == expression_count)
2573 {
2574 retval = 1;
2575 } else
2576 {
2577 retval = 0;
2578 }
2579 }
2580 if(invert) retval = !retval;
2581 return retval;
2582 }
2583 /* initialize expressions field pointers */
2584 /* pointers are preinitialized for faster execution */
2585 void
init_expression_list(struct record * r)2586 init_expression_list(struct record *r)
2587 {
2588 struct expression *e;
2589 struct field *f = r->f;
2590 int exp_count = 0;
2591
2592 e = expression;
2593
2594 while(e != NULL)
2595 {
2596 e->f = NULL;
2597 e = e->next;
2598 exp_count++;
2599 }
2600
2601 e = expression;
2602
2603 while(f != NULL && exp_count)
2604 {
2605 e = expression;
2606 while(e != NULL && exp_count)
2607 {
2608 if(e->f == NULL && strcasecmp(f->name,e->field) == 0)
2609 {
2610 e->f = f;
2611 exp_count--;
2612 }
2613 e = e->next;
2614 }
2615 f = f->next;
2616 }
2617 }
2618
2619 void
open_debug_file(int stype)2620 open_debug_file(int stype)
2621 {
2622 sprintf(debug_file,"ffe_error_%d.log",(int) getpid());
2623 if(stype == BINARY)
2624 {
2625 debug_fp = xfopenb(debug_file,"w");
2626 } else
2627 {
2628 debug_fp = xfopen(debug_file,"w");
2629 }
2630 }
2631
2632
2633 void
write_debug_file(uint8_t * line,int len,int stype)2634 write_debug_file(uint8_t *line,int len,int stype)
2635 {
2636 if(debug_fp == NULL) open_debug_file(stype);
2637
2638 if(stype == BINARY)
2639 {
2640 fwrite(line,len,1,debug_fp);
2641 } else if(line != NULL)
2642 {
2643 while(*line != '\n') fputc(*line++,debug_fp);
2644 fputs("\n",debug_fp);
2645 }
2646 debug_lineno++;
2647 }
2648
2649 static void
select_output(struct output * o)2650 select_output(struct output *o)
2651 {
2652 if(o->ofp != NULL)
2653 {
2654 output_fp = o->ofp;
2655 output_file = o->output_file;
2656 } else
2657 {
2658 output_fp = default_output_fp;
2659 output_file = default_output_file;
2660 }
2661 }
2662
2663
2664 /* main loop for execution */
2665 void
execute(struct structure * s,int strict,int expression_and,int expression_invert,int expression_case,int debug,char * anon_to_use)2666 execute(struct structure *s,int strict, int expression_and,int expression_invert,int expression_case, int debug,char *anon_to_use)
2667 {
2668 uint8_t *input_line;
2669 struct record *r = NULL;
2670 struct record *prev_record = NULL;
2671 int length;
2672 int header_printed = 0;
2673 int fields_printed;
2674 int first_line = 1;
2675 int i;
2676 int anon_field_count=0;
2677
2678 current_file_lineno = 0;
2679 current_total_lineno = 0;
2680 current_file_name = files->name;
2681 headers = s->header;
2682
2683 i = 0;
2684 while(i < JUSTIFY_STRING) justify_string[i++] = ' ';
2685
2686 reset_levels(0,MAXLEVEL);
2687
2688 write_buffer = xmalloc(write_buffer_size);
2689 write_buffer_end = write_buffer + (write_buffer_size - 1);
2690
2691 select_output(s->o);
2692 print_text(s,NULL,s->o->file_header);
2693 while((input_line = get_input_line(&length,s->type[0])) != NULL)
2694 {
2695 prev_record = r;
2696 r = select_record(s,length,input_line);
2697 if(r == NULL)
2698 {
2699 if(debug) write_debug_file(input_line,length,s->type[0]);
2700 invalid_input(current_file_name,current_file_lineno,strict,length,s->type[0]);
2701 if(s->type[0] == BINARY)
2702 {
2703 last_consumed = 1; /* advance one byte and check next block, this is a rather expensive way to read ahead... */
2704 }
2705
2706 } else
2707 {
2708 if(first_line)
2709 {
2710 init_structure(s,r,length,input_line);
2711 anon_field_count = update_anon_info(s,anon_to_use);
2712 if(anon_field_count) init_libgcrypt();
2713
2714 }
2715
2716 if(expression != NULL && (prev_record != r || prev_record == NULL))
2717 {
2718 init_expression_list(r);
2719 }
2720
2721 last_consumed = update_field_positions(s->type,s->quote,r,length,input_line);
2722
2723 if((!first_line || !headers) && r->o != no_output)
2724 {
2725 if((r->pf == NULL && r->o->no_data == 1) || r->pf != NULL || r->o == raw)
2726 {
2727 if(expression == NULL || (eval_expression(s,r,expression_and,expression_invert,expression_case,input_line)))
2728 {
2729 if(anon_field_count) anonymize_fields(s->type,s->quote,r,length,input_line); // anonymize after exp. evaluation
2730 if(r->o == raw)
2731 {
2732 print_raw(s->type[0] == BINARY ? last_consumed : length,input_line,s->type[0]);
2733 } else
2734 {
2735 select_output(r->o);
2736 print_level_before(prev_record,r);
2737 if(r->o->header != NULL && !header_printed)
2738 {
2739 header_printed = print_header(s,r);
2740 }
2741 if(r->o->indent != NULL && r->o->record_header != NULL)
2742 print_indent(r->o->indent,r->level != NULL ? r->level->level : 1);
2743 print_text(s,r,r->o->record_header);
2744 fields_printed = print_fields(s,r,input_line);
2745 if(fields_printed || r->o->print_empty)
2746 {
2747 if(r->o->indent != NULL && r->o->record_trailer != NULL)
2748 print_indent(r->o->indent,r->level != NULL ? r->level->level : 1);
2749 print_text(s,r,r->o->record_trailer);
2750 }
2751 }
2752 }
2753 }
2754 }
2755
2756 if(first_line) {
2757 first_line = 0;
2758 if(headers) r = NULL;
2759 }
2760 }
2761 }
2762 print_level_end(r);
2763 select_output(s->o);
2764 print_text(s,r,s->o->file_trailer);
2765 free(write_buffer);
2766 if(debug_fp != NULL) fclose(debug_fp);
2767 }
2768
2769
2770
2771