1 /*
2 ** Modular Logfile Analyzer
3 ** Copyright 2000 Jan Kneschke <jan@kneschke.de>
4 **
5 ** Homepage: http://www.modlogan.org
6 **
7
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version, and provided that the above
12 copyright and permission notice is included with all distributed
13 copies of this or derived software.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
23
24 **
25 ** $Id: main.c,v 1.115 2004/08/27 20:07:37 ostborn Exp $
26 */
27
28 /**
29 * \mainpage ModLogAn API Documenation
30 *
31 * \section Introduction
32 * Add docs.
33 *
34 */
35
36 #include <stdlib.h>
37 #include <stdio.h>
38 #include <time.h>
39 #include <sys/time.h>
40 #include <unistd.h>
41 #include <dirent.h>
42 #include <errno.h>
43 #include <string.h>
44 #include <sys/stat.h>
45 #include <sys/time.h>
46 #include <assert.h>
47
48 #include <zlib.h>
49
50 #include "config.h"
51
52 #include "gettext.h"
53
54 #ifdef HAVE_GETOPT_H
55 #include <getopt.h>
56 #endif
57
58 #ifdef HAVE_PWD_H
59 #include <pwd.h>
60 #ifdef HAVE_GRP_H
61 #include <grp.h>
62 #define RUN_AS_USER
63 #endif
64 #endif
65
66 #include "mrecord.h"
67 #include "mhistory.h"
68 #include "mstate.h"
69 #include "mlocale.h"
70 #include "mconfig.h"
71 #include "mplugins.h"
72 #include "mdatatypes.h"
73 #include "misc.h"
74 #include "datatypes/webhist/datatype.h"
75 #include "datatypes/mailhist/datatype.h"
76 #include "datatypes/state/datatype.h"
77 #include "datatypes/query/datatype.h"
78 #include "datatypes/string/datatype.h"
79 #include "datatypes/count/datatype.h"
80
81 #include "mresolver.h"
82
83 #define MSTATE_WRITE 1
84
85 size_t mem_mhash_size = 0;
86 size_t mem_mhash_count = 0;
87 size_t mem_mlist_size = 0;
88 size_t mem_mlist_count = 0;
89 size_t mem_mdata_size = 0;
90 size_t mem_mdata_count = 0;
91
92 size_t mem_mdata_type_count[M_DATA_TYPE_IPPLWATCH] = { 0, };
93
show_version()94 void show_version() {
95 printf("%s %s\n", PACKAGE, VERSION);
96 fflush(stdout);
97 }
98
show_header()99 void show_header() {
100 printf("%s %s\n", PACKAGE, VERSION);
101 fflush(stdout);
102 }
103
show_usage()104 void show_usage() {
105 show_header();
106
107 printf("Options:\n");
108 printf(" -c <configfile> filename of the configfile\n");
109 printf(" -r force execution as root\n");
110 printf(" -o <section>:<key>=<value>\n");
111 printf(" set a config-option in section <section>\n");
112 #ifdef RUN_AS_USER
113 printf(" -u <username> force execution as username (need -r)\n");
114 #endif
115 #ifdef HAVE_GETOPT_LONG
116 printf(" -h --help this help screen\n");
117 printf(" -v --version display version\n");
118 #else
119 printf(" -h this help screen\n");
120 printf(" -v display version\n");
121 #endif
122 };
123
restore_internal_state(mconfig * conf,mlist * state_list)124 int restore_internal_state (mconfig *conf, mlist *state_list) {
125 char *fn;
126 char buf[255];
127 FILE *split_file;
128 int empty = 1;
129
130 if (conf->statedir == NULL) {
131 fprintf(stderr, "%s.%d: global:statdir is empty\n", __FILE__, __LINE__);
132 return -1;
133 }
134
135 fn = malloc(strlen(conf->statedir) + strlen("/modlogan.statefiles") + 1);
136 strcpy(fn, conf->statedir);
137 strcat(fn, "/modlogan.statefiles");
138
139 if ((split_file = fopen(fn, "r")) == NULL) {
140 M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_ERRORS,
141 "NOTICE: can't open %s: %s, <- ignore this message on the first run\n", fn, strerror(errno));
142
143 free(fn);
144 return 0;
145 }
146
147 while (fgets(buf, sizeof(buf), split_file)) {
148 mstate *state = NULL;
149 mlist *hist = mlist_init();
150
151 empty = 0;
152
153 /* remove the \n */
154 buf[strlen(buf)-1] = '\0';
155
156 M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
157 "reading statefile from %s/%s/mla.state.xml\n", conf->statedir, buf);
158
159 if (history_read(conf, hist, buf)) {
160 fprintf(stderr, "%s.%d: hist failed ?\n", __FILE__, __LINE__);
161 mlist_free(hist);
162 hist = NULL;
163 }
164
165 if (conf->incremental) {
166 state = mstate_init();
167
168 if (mstate_read(conf, state, 0, 0, buf)) {
169 M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
170 "reading state failed");
171
172 mstate_free(state);
173 state = NULL;
174 }
175 }
176
177 if (!hist || (conf->incremental && !state)) {
178 if (hist) {
179 mlist_free(hist);
180 } else {
181 M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
182 "reading history failed");
183 }
184
185 if (state) {
186 mstate_free(state);
187 } else if (conf->incremental) {
188 M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
189 "reading state failed");
190 }
191
192 state = NULL;
193 hist = NULL;
194 } else {
195 mdata *data;
196 const char *key = splaytree_insert(conf->strings, buf);
197
198 data = mdata_State_create(key, state, hist);
199 mlist_insert(state_list, data);
200
201 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
202 "state restored for '%s'\n", buf);
203 }
204 }
205
206 fclose(split_file);
207
208 if (empty) {
209 fprintf(stderr, "%s.%d: \n"
210 " modlogan.statefiles (%s) was found, but the file is empty\n"
211 " looks like modlogan died in the previous run while writing the output\n",
212 __FILE__, __LINE__, fn);
213 free(fn);
214 return -1;
215 }
216
217 free(fn);
218
219 return 0;
220 }
221
222
223
224 typedef struct {
225 mlogrec *record;
226 int rec_state;
227 } mgetrec;
228
229
get_next_record(mconfig * conf,mlogrec * rec)230 int get_next_record (mconfig *conf, mlogrec *rec) {
231 mplugin *func;
232 int i, j;
233 int state = M_RECORD_EOF;
234 int last_rec_state = M_RECORD_NO_ERROR;
235 static mgetrec **recs = NULL;
236 static mgetrec **readahead = NULL;
237 time_t t = -1;
238 int ndx = -1;
239 time_t rt = -1;
240 int rndx = -1;
241
242 if (recs == NULL && conf->plugin_count) {
243 recs = malloc(sizeof(mgetrec *) * conf->plugin_count);
244 for (i = 0; i < conf->plugin_count; i++) {
245 recs[i] = malloc(sizeof(mgetrec));
246 recs[i]->record = mrecord_init();
247 recs[i]->rec_state = M_RECORD_EOF;
248 }
249
250 }
251
252 if (readahead == NULL) {
253 readahead = malloc(sizeof(mgetrec *) * conf->read_ahead_limit);
254 for (i = 0; i < conf->read_ahead_limit; i++) {
255 readahead[i] = malloc(sizeof(mgetrec));
256 readahead[i]->record = mrecord_init();
257 readahead[i]->rec_state = M_RECORD_EOF;
258 }
259
260 }
261
262 /* fill the read-ahead-buffer */
263 for (j = 0; j < conf->read_ahead_limit; j++) {
264 if (last_rec_state != M_RECORD_EOF &&
265 readahead[j]->record->timestamp == 0) {
266 /* slot is free */
267
268 ndx = -1;
269 t = -1;
270 /* check the plugins for new records */
271 for (i = 0; i < conf->plugin_count; i++) {
272 func = ((mplugin **)(conf->plugins))[i];
273 conf->plugin_conf = func->config;
274
275 if (func->get_next_record) {
276 if (recs[i]->record->timestamp == 0) {
277 recs[i]->rec_state = func->get_next_record(conf, recs[i]->record);
278
279 if (recs[i]->record->timestamp < 0) {
280 fprintf(stderr, "%s.%d: %ld %ld\n",
281 __FILE__, __LINE__,
282 recs[i]->record->timestamp,
283 t);
284 }
285
286 last_rec_state = recs[i]->rec_state;
287
288 switch(recs[i]->rec_state) {
289 case M_RECORD_NO_ERROR:
290 /* start the resolver for this record */
291 resolver_start(conf, recs[i]->record);
292 break;
293 case M_RECORD_SKIPPED:
294 case M_RECORD_CORRUPT:
295 case M_RECORD_IGNORED:
296 case M_RECORD_HARD_ERROR:
297 #if 0
298 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_PARSING, M_DEBUG_LEVEL_WARNINGS,
299 "got %d\n",
300 recs[i]->rec_state);
301 #endif
302 return recs[i]->rec_state;
303 case M_RECORD_EOF:
304 break;
305 }
306 }
307
308 #if 0
309 fprintf(stderr, "%s.%d: %d %d\n",
310 __FILE__, __LINE__,
311 recs[i]->record->timestamp,
312 t);
313 #endif
314 if ((recs[i]->record->timestamp > 0) && /* valid timestamp */
315 ((t == -1) || /* first result */
316 (recs[i]->record->timestamp < t))) { /* oldest timestamp */
317 t = recs[i]->record->timestamp;
318 ndx = i;
319 } else {
320 #if 0
321 fprintf(stderr, "%s.%d: %ld %ld %d\n",
322 __FILE__, __LINE__,
323 recs[i]->record->timestamp,
324 t, i);
325 #endif
326 }
327 }
328 }
329
330 /* got a new record */
331 if (ndx != -1) {
332 mrecord_move(readahead[j]->record, recs[ndx]->record);
333 readahead[j]->rec_state = recs[ndx]->rec_state;
334
335 if (recs[ndx]->record->timestamp < 0) {
336 fprintf(stderr, "%s.%d: %ld %ld %d\n",
337 __FILE__, __LINE__,
338 recs[ndx]->record->timestamp,
339 t, i);
340 }
341 #if 0
342 fprintf(stderr, "%s.%d: readahead: filled slot %d, %d\n",
343 __FILE__, __LINE__,
344 j, readahead[j]->record->ext_type);
345 #endif
346 } else {
347 #if 0
348 fprintf(stderr, "%s.%d: no record for me\n", __FILE__, __LINE__);
349 #endif
350 }
351 }
352
353 /* select the oldest record from our readahead storage */
354 if (readahead[j]->record->timestamp != 0) {
355 if (rt == -1 || readahead[j]->record->timestamp < rt) {
356 rt = readahead[j]->record->timestamp;
357 rndx = j;
358 } else {
359 #if 0
360 fprintf(stderr, "%s.%d: %ld - %ld\n", __FILE__, __LINE__, rt, readahead[j]->record->timestamp);
361 #endif
362 }
363 } else {
364 #if 0
365 fprintf(stderr, "%s.%d - no record\n", __FILE__, __LINE__);
366 #endif
367 }
368 }
369
370 if (rndx != -1) {
371 mrecord_move(rec, readahead[rndx]->record);
372 state = readahead[rndx]->rec_state;
373 if (state == M_RECORD_EOF) {
374 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_PARSING, M_DEBUG_LEVEL_WARNINGS,
375 "got %d\n",
376 state);
377 }
378 } else {
379 #if 0
380 M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_PARSING, M_DEBUG_LEVEL_WARNINGS,
381 "got - not records\n");
382 #endif
383
384
385 /* got no record, kill myself */
386 for (i = 0; i < conf->plugin_count; i++) {
387 mrecord_free(recs[i]->record);
388 free(recs[i]);
389 }
390
391 free(recs);
392 recs = NULL;
393
394 for (i = 0; i < conf->read_ahead_limit; i++) {
395 mrecord_free(readahead[i]->record);
396 free(readahead[i]);
397 }
398
399 free(readahead);
400 readahead = NULL;
401 }
402
403 return state;
404 }
405
generate_monthly_output(mconfig * conf,mstate * state,const char * subpath)406 int generate_monthly_output(mconfig *conf, mstate *state, const char *subpath) {
407 int i;
408 mplugin *func;
409
410 if (subpath) {
411 /* set the splitter */
412 const char *key = splaytree_insert(conf->strings, "splitby");
413 mdata *data = mdata_String_create(key, subpath);
414
415 if (data) {
416 mhash_insert_replace(conf->variables, data);
417 }
418 }
419
420 /* call all output-generators */
421 for (i = 0; i < conf->plugin_count; i++) {
422 func = ((mplugin **)(conf->plugins))[i];
423 conf->plugin_conf = func->config;
424
425 if (func->gen_report) {
426 if (func->gen_report(conf, state, subpath))
427 return -1;
428 }
429 }
430
431 return 0;
432 }
generate_history_output(mconfig * conf,mlist * history,const char * subpath)433 int generate_history_output(mconfig *conf, mlist *history, const char *subpath) {
434 int i;
435 mplugin *func;
436
437 if (subpath) {
438 const char *key = splaytree_insert(conf->strings, "splitby");
439 mdata *data = mdata_String_create(key, subpath);
440
441 if (data) {
442 mhash_insert_replace(conf->variables, data);
443 }
444 }
445
446 for (i = 0; i < conf->plugin_count; i++) {
447 func = ((mplugin **)(conf->plugins))[i];
448 conf->plugin_conf = func->config;
449
450 if (func->gen_history && func->gen_history(conf, history, subpath)) return -1;
451 }
452
453 return 0;
454 }
455
dump(mlist * state_list)456 void dump (mlist *state_list) {
457 mlist *s;
458 int i;
459
460 for (s = state_list; s; s = s->next) {
461 mstate_web *w;
462 if (!s->data) return;
463
464 w = ((mstate *)(s->data->data.state.state))->ext;
465
466 for (i = 0; i < 31; i++) {
467 fprintf(stderr, "dump_list (%p)->'%s': %d: %ld\n", s, s->data->key, i, w->days[i].hits);
468 }
469 }
470 }
471
insert_record(mconfig * conf,mlist * state_list,mlogrec * record)472 int insert_record(mconfig *conf, mlist *state_list, mlogrec *record) {
473 int i = 0;
474 mplugin *func;
475
476
477 resolver_finish(conf, record);
478
479 for (i = 0; i < conf->plugin_count; i++) {
480 func = ((mplugin **)(conf->plugins))[i];
481 conf->plugin_conf = func->config;
482
483 if (func->insert_record)
484 if (func->insert_record(conf, state_list, record))
485 return -1;
486 }
487
488 return 0;
489 }
490
491 #ifdef RUN_AS_USER
492 /* Change to user privileges. May be only called by root.
493 * It returns 0 on success.
494 * -1 on any error.
495 */
change_to_user(char * user)496 static int change_to_user(char *user) {
497 struct passwd *pwd;
498
499 /* Get user info */
500 pwd = getpwnam (user);
501 if (!pwd) return -1;
502
503 /* We don't want to change to root user, -r is here for that. */
504 if (!pwd->pw_uid) return -1;
505
506 #ifdef HAVE_SETGROUPS
507 #ifdef HAVE_GETGROUPLIST
508 #ifndef NGROUPS
509 #define NGROUPS 32
510 #endif
511 {
512 gid_t *groups;
513 int ngroups = NGROUPS;
514
515 /* Get user group list */
516 groups = (gid_t *) malloc (ngroups * sizeof (gid_t));
517 if (!groups) return -1;
518 if (getgrouplist(user, pwd->pw_gid, groups, &ngroups) == -1) {
519 groups = (gid_t *) realloc (groups, ngroups * sizeof (gid_t));
520 if (!groups) return -1;
521 if (getgrouplist(user, pwd->pw_gid, groups, &ngroups) == -1) {
522 free(groups);
523 return -1;
524 }
525 }
526
527 /* Set additionnal groups */
528 if (setgroups(ngroups, groups) == -1) {
529 free(groups);
530 return -1;
531 }
532 free(groups);
533
534 }
535 #undef NGROUPS
536 #else
537 /* Remove any additionnal group */
538 if (setgroups(0, NULL) == -1) return -1;
539 #endif /* HAVE_GETGROUPLIST */
540 #endif /* HAVE_SETGROUPS */
541
542 /* Set main group */
543 if (setgid (pwd->pw_gid) == -1) return -1;
544 /* Set user */
545 if (setuid (pwd->pw_uid) == -1) return -1;
546
547 /* Re-check ... */
548 if (getuid() != pwd->pw_uid || geteuid() != pwd->pw_uid) return -1;
549 return 0;
550 }
551 #endif /* RUN_AS_USER */
552
main(int argc,char ** argv)553 int main (int argc, char **argv) {
554 mconfig *conf = NULL;
555 mlogrec *rec = NULL;
556 mlist *state_list = NULL;
557 mlist *l;
558
559 int ret, l_month = -1, l_year = -1, l_hour = -1, l_day = -1, i;
560 char *conf_filename = NULL;
561 long lines = 0, lines_corrupt = 0, lines_skipped = 0, lines_ignored = 0;
562 int first_valid_line = 0;
563 int root_check = 1;
564
565 struct tm *tm;
566
567 time_t l_stamp = -1;
568 mtimer process_timer, post_process_timer, parse_timer, setup_timer;
569
570 #ifdef RUN_AS_USER
571 char *run_as_user = NULL;
572 #define CMD_OPTIONS "c:hvro:u:"
573 #else
574 #define CMD_OPTIONS "c:hvro:"
575 #endif
576
577 #ifdef HAVE_GETOPT_LONG
578 int option_index = 0;
579 static struct option long_options[] = {
580 { "help", 0, NULL, 'h' },
581 { "version", 0, NULL, 'v' },
582 { "root", 0, NULL, 'r' },
583 { NULL, 0, NULL, 0 }
584 };
585 #endif
586
587 MTIMER_RESET(process_timer);
588 MTIMER_RESET(post_process_timer);
589 MTIMER_RESET(parse_timer);
590 MTIMER_RESET(setup_timer);
591
592 if (!(conf = mconfig_init())) {
593 fprintf(stderr, "%s.%d: init config failed\n", __FILE__, __LINE__);
594 return -1;
595 }
596
597 /*
598 * command line options
599 *
600 * -h/--help help
601 * -r/--root allow root
602 * -u <uid> set new uid
603 * -v/--version version
604 * -c <file> configfile
605 *
606 * -o <section>:<key>=<value>
607 */
608
609 opterr = 0;
610
611 while (
612 #ifdef HAVE_GETOPT_LONG
613 (i = getopt_long(argc, argv, CMD_OPTIONS,
614 long_options, &option_index)) != -1
615
616 #else
617 (i = getopt(argc,argv,CMD_OPTIONS)) != EOF
618 #endif
619 ) {
620 switch(i) {
621 case 'c':
622 conf_filename = optarg;
623 break;
624 case 'h':
625 show_usage();
626 exit( EXIT_SUCCESS);
627 case 'v':
628 show_version();
629 exit( EXIT_SUCCESS);
630 case 'r':
631 root_check = 0;
632 break;
633 #ifdef RUN_AS_USER
634 case 'u':
635 run_as_user = optarg;
636 break;
637 #endif
638 case 'o': {
639 char *key = splaytree_insert(conf->strings, optarg);
640 mdata *data = mdata_Count_create(key, 1, 0);
641 /* add option to the config-file handling */
642
643 mlist_insert_sorted(conf->cmdlineoptions, data);
644
645 break;
646 }
647 default:
648 show_usage();
649 exit( EXIT_FAILURE);
650 }
651 }
652
653 show_header();
654
655 if (getuid() == 0 || geteuid() == 0) {
656 if (root_check) {
657 fprintf(stderr, "ModLogAn detected that it is running with root permissions.\n"
658 "As root permissions are not directly required to run modlogan, \n"
659 "modlogan decided do stop working with root permission.\n\n"
660 "The reason for this decision is quite simple:\n"
661 "ModLogAn doesn't want to be announced at BugTraq.\n"
662 "Although this might be a little bit too defensive it is still better\n"
663 "then nothing.\n\n"
664 "If you still have to run modlogan with root permissions \n"
665 "use the -r/--root switch which will disable this test\n");
666 exit(-1);
667 }
668 #ifdef RUN_AS_USER
669 if (run_as_user) {
670 if (change_to_user(run_as_user)) {
671 fprintf(stderr, "ModLogAn failed to change to user '%s'. Exiting.\n", run_as_user);
672 exit(-1);
673 } else {
674 fprintf(stderr, "ModLogAn is running as user '%s'.\n", run_as_user);
675 }
676 } else
677 #endif
678 fprintf(stderr, "WARNING: ModLogAn is running as user 'root'.\n");
679
680 }
681
682 init_locale(conf);
683
684 MTIMER_START(setup_timer);
685
686 if (mconfig_read(conf, conf_filename)) {
687 mconfig_free(conf);
688
689 fprintf(stderr, "%s.%d: reading configfile failed - going down hard\n", __FILE__, __LINE__);
690 return -1;
691 }
692
693 if (conf->show_options) {
694 /* we only have to show some options */
695
696 mconfig_free(conf);
697
698 return 0;
699 }
700
701 state_list = mlist_init();
702
703 if (restore_internal_state(conf, state_list)) {
704
705 mconfig_free(conf);
706 mlist_free(state_list);
707
708 return -1;
709 }
710
711 rec = mrecord_init();
712
713 if (conf->incremental) {
714 l = state_list;
715
716 while (l) {
717 mdata *data = l->data;
718 mstate *int_state;
719 if (!data) break;
720
721 int_state = data->data.state.state;
722 if (int_state->timestamp > l_stamp) {
723 l_stamp = int_state->timestamp;
724 l_year = int_state->year-1900;
725 l_month = int_state->month-1;
726 }
727
728 l = l->next;
729 }
730 }
731
732 if (l_stamp != -1) {
733 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
734 "Ignoring all records before %s", ctime(&l_stamp));
735 }
736
737 M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_NONE,
738 "NOTICE: startup - finished\n");
739
740 MTIMER_STOP(setup_timer);
741 MTIMER_CALC(setup_timer);
742
743 printf("[");
744
745 /* mainloop */
746 MTIMER_START(parse_timer);
747 while ((ret = get_next_record(conf, rec)) != M_RECORD_EOF) {
748 MTIMER_STOP(parse_timer);
749 MTIMER_CALC(parse_timer);
750
751 MTIMER_START(process_timer);
752
753 lines++;
754 if (ret == M_RECORD_NO_ERROR) {
755 /* HACK */
756 if (rec->timestamp == 0) {
757 fprintf(stderr, "%s.%d: line %ld returned no timestamp !! something strange will happen. Going down hard\n",
758 __FILE__, __LINE__,
759 lines);
760 return -1;
761 }
762
763 /* set the first timestamp */
764 if (l_month == -1) {
765 tm = localtime(&(rec->timestamp));
766 l_month = tm->tm_mon;
767 l_year = tm->tm_year;
768 l_stamp = rec->timestamp;
769 l_hour = tm->tm_hour;
770 }
771
772 /* is the current timestamp below the timestamp
773 * of the last successfully parsed record ?
774 */
775 if (rec->timestamp < l_stamp) {
776 /* skip the record if we are in incremental mode 1 and no
777 * record has been parsed successfully yet
778 */
779 if (first_valid_line == 0 && conf->incremental == 1) {
780 lines_skipped++;
781 MTIMER_STOP(process_timer);
782 MTIMER_CALC(process_timer);
783 MTIMER_START(parse_timer);
784 continue;
785 }
786 M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_VERBOSE,
787 "Out of Sequence - ignoring timestamp: %ld > %ld\n", l_stamp, rec->timestamp);
788
789 rec->timestamp = l_stamp;
790 }
791
792 tm = localtime(&(rec->timestamp));
793
794 /* set last day of month */
795 l_day = tm->tm_mday;
796
797 /* generate report if we have reached a threshold (semi-online mode) */
798 if (conf->gen_report_threshold > 0 &&
799 (lines - lines_corrupt + 1) % conf->gen_report_threshold == 0) {
800
801 MTIMER_STOP(process_timer);
802 MTIMER_CALC(process_timer);
803
804 printf("]\n");
805
806 MTIMER_START(post_process_timer);
807 M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_WARNINGS,
808 "o(t) writing month %02i - %04i\n", l_month+1, l_year+1900);
809
810
811 for (l = state_list; l; l = l->next) {
812 mdata *data = l->data;
813 mdata *histdata = NULL;
814 mstate *state;
815
816 if (!data) break;
817
818 state = data->data.state.state;
819
820 switch (state->ext_type) {
821 case M_STATE_TYPE_WEB:
822 histdata = mdata_WebHist_create_by_state(state);
823 break;
824 case M_STATE_TYPE_MAIL:
825 histdata = mdata_Mailhist_create_by_state(state);
826 break;
827 case M_STATE_TYPE_UNSET:
828 continue;
829 default:
830 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
831 "Unknown state-type: %d\n", state->ext_type);
832 return (-1);
833 }
834
835 mlist_insert_replace(data->data.state.history, histdata);
836
837 /* sort the history list */
838 data->data.state.history = mlist_sort_full_by_string(data->data.state.history);
839
840 #if MSTATE_WRITE
841
842 if (0 != mstate_write(conf, data->data.state.state,
843 M_STATE_WRITE_BY_MONTH,
844 *(data->key) ? data->key : NULL)) {
845 fprintf(stderr, "%s.%d: writing state file failed\n", __FILE__, __LINE__);
846 return -1;
847 }
848 history_write(conf, data->data.state.history, *(data->key) ? data->key : NULL);
849 #endif
850 if (0 != generate_monthly_output(conf, data->data.state.state, *(data->key) ? data->key : NULL)) {
851 fprintf(stderr, "%s.%d: output generation failed - no data this month?\n", __FILE__, __LINE__);
852 /* return -1; */
853 }
854 if (0 != generate_history_output(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
855 fprintf(stderr, "%s.%d: history generation failed - going down hard\n", __FILE__, __LINE__);
856 return -1;
857 }
858
859 }
860 MTIMER_STOP(post_process_timer);
861 MTIMER_CALC(post_process_timer);
862
863
864 printf("[");
865 MTIMER_START(process_timer);
866 }
867
868 /* we have left a month, generate the reports */
869 if (tm->tm_mon != l_month || tm->tm_year != l_year) {
870
871 MTIMER_STOP(process_timer);
872 MTIMER_CALC(process_timer);
873
874 printf("]\n");
875
876 MTIMER_START(post_process_timer);
877
878 M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_WARNINGS,
879 "o writing month %02i - %04i\n", l_month+1, l_year+1900);
880
881
882 for (l = state_list; l; l = l->next) {
883 mdata *data = l->data;
884 mdata *histdata = NULL;
885 mstate *state;
886
887 if (!data) break;
888
889 state = data->data.state.state;
890
891 switch (state->ext_type) {
892 case M_STATE_TYPE_WEB:
893 histdata = mdata_WebHist_create_by_state(state);
894 break;
895 case M_STATE_TYPE_MAIL:
896 histdata = mdata_Mailhist_create_by_state(state);
897 break;
898 case M_STATE_TYPE_UNSET:
899 if (state->ext) {
900 M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
901 "no ext_type, but ext is set\n");
902 return -1;
903 }
904 continue;
905 default:
906 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
907 "Unknown state-type: %d\n", state->ext_type);
908 return (-1);
909 }
910
911 mlist_insert_replace(data->data.state.history, histdata);
912
913 /* sort the history list */
914 data->data.state.history = mlist_sort_full_by_string(data->data.state.history);
915
916 if (conf->debug_level > 3)
917 fprintf(stderr, "o(t) -- state written: (...)/%s/\n", *(data->key) ? data->key : ".");
918 #if MSTATE_WRITE
919 if (0 != mstate_write(conf, data->data.state.state,
920 M_STATE_WRITE_BY_MONTH,
921 *(data->key) ? data->key : NULL)) {
922 fprintf(stderr, "%s.%d: writing state file failed\n", __FILE__, __LINE__);
923 return -1;
924 }
925 history_write(conf, data->data.state.history, *(data->key) ? data->key : NULL);
926 #endif
927 if (0 != generate_monthly_output(conf, data->data.state.state, *(data->key) ? data->key : NULL)) {
928 fprintf(stderr, "%s.%d: output generation failed - no data this month?\n", __FILE__, __LINE__);
929 /* return -1; */
930 }
931 if (0 != generate_history_output(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
932 fprintf(stderr, "%s.%d: history generation failed - going down hard\n", __FILE__, __LINE__);
933 return -1;
934 }
935
936 mstate_free(data->data.state.state);
937 data->data.state.state = mstate_init();
938 }
939
940 tm = localtime(&(rec->timestamp));
941
942 l_month = tm->tm_mon;
943 l_year = tm->tm_year;
944
945 MTIMER_STOP(post_process_timer);
946 MTIMER_CALC(post_process_timer);
947
948 for (i = 0; i < ((lines % 50000) / 1000); i++) printf(" ");
949 printf("[");
950
951 MTIMER_START(process_timer);
952 }
953
954 if (insert_record(conf, state_list, rec) != 0) {
955 fprintf(stderr, "%s.%d: inserting record failed - going down hard\n", __FILE__, __LINE__);
956 return -1;
957 }
958
959 first_valid_line = 1;
960
961 l_stamp = rec->timestamp;
962 } else if (ret == M_RECORD_SKIPPED) {
963 lines_skipped++;
964 } else if (ret == M_RECORD_IGNORED) {
965 lines_ignored++;
966 } else if (ret == M_RECORD_HARD_ERROR) {
967 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
968 "a hard error occured in line %ld - going down hard\n", lines);
969
970 exit(-1);
971 } else {
972 lines_corrupt++;
973
974 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
975 "parser reported an error in line %ld\n", lines);
976 }
977 mrecord_reset(rec);
978
979 /* cosmetics */
980 if (lines % 1000 == 0) {
981 printf(".");
982 if (lines % (1000 * 50)== 0) {
983 printf(" %8ld", lines);
984 if (conf->debug_level > 1) {
985 printf(" - %10.2f (%4ld, %4ld)",
986 (lines)/((MTIMER_GET_USER_MSEC(parse_timer)+MTIMER_GET_USER_MSEC(process_timer))/1000.0),
987 lines_corrupt,
988 lines_skipped);
989 }
990 printf("\n ");
991 }
992 fflush(stdout);
993 }
994
995 MTIMER_STOP(process_timer);
996 MTIMER_CALC(process_timer);
997
998 MTIMER_START(parse_timer);
999 }
1000
1001 MTIMER_STOP(parse_timer);
1002 MTIMER_CALC(parse_timer);
1003
1004 printf("]\n");
1005
1006 MTIMER_START(post_process_timer);
1007
1008 if (l_month != -1) {
1009 FILE *split_file;
1010 char *fn;
1011 printf(" ");
1012
1013 if (conf->debug_level > 0)
1014 printf("writing month %02i - %04i\n",l_month+1, l_year+1900);
1015
1016 if (conf->debug_level > 1)
1017 fprintf(stderr, "%s.%d: writing the last month\n", __FILE__, __LINE__ );
1018 #if MSTATE_WRITE
1019 /* create filename */
1020 fn = malloc(strlen(conf->statedir) + strlen("/modlogan.statefiles") + 1);
1021 strcpy(fn, conf->statedir);
1022 strcat(fn, "/modlogan.statefiles");
1023 if (NULL == (split_file = fopen(fn, "w"))) {
1024 fprintf(stderr, "%s.%d: %s\n",
1025 __FILE__, __LINE__,
1026 strerror(errno));
1027 } else {
1028 fclose(split_file);
1029 }
1030 #endif
1031
1032 for (l = state_list; l; l = l->next) {
1033 mdata *data = l->data;
1034 mdata *histdata = NULL;
1035 int faulty = 0;
1036 mstate *state;
1037
1038 if (!data) break;
1039
1040 state = data->data.state.state;
1041
1042 if (conf->debug_level > 1)
1043 fprintf(stderr, "%s.%d: subpath = '%s'\n",
1044 __FILE__, __LINE__,
1045 data->key );
1046
1047 switch (state->ext_type) {
1048 case M_STATE_TYPE_WEB:
1049 if (NULL == (histdata = mdata_WebHist_create_by_state(state))) {
1050 M_WP();
1051
1052 return -1;
1053 }
1054 break;
1055 case M_STATE_TYPE_MAIL:
1056 histdata = mdata_Mailhist_create_by_state(state);
1057 break;
1058 case M_STATE_TYPE_TRAFFIC:
1059 break;
1060 case M_STATE_TYPE_IPPL:
1061 //histdata = mdata_Ipplhist_create_by_state(state);
1062 break;
1063 case M_STATE_TYPE_UNSET:
1064 if (state->ext) {
1065 M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
1066 "no ext_type, but ext is set\n");
1067 return -1;
1068 }
1069 break;
1070 default:
1071 M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
1072 "Unknown state-type: %d\n", state->ext_type);
1073 return -1;
1074 }
1075
1076 if (histdata) mlist_insert_replace(data->data.state.history, histdata);
1077
1078 #if MSTATE_WRITE
1079 if (!faulty && mstate_write(conf, data->data.state.state, M_STATE_WRITE_DEFAULT, *(data->key) ? data->key : NULL)) {
1080 faulty = 1;
1081 }
1082 if (!faulty && history_write(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
1083 faulty = 1;
1084 }
1085 if (!faulty) {
1086 /* write the current directoy to the split file */
1087 if (NULL != (split_file = fopen(fn, "a+"))) {
1088 fprintf(split_file, "%s\n", data->key);
1089 fclose(split_file);
1090 }
1091 }
1092 #endif
1093
1094 if (state->ext_type != M_STATE_TYPE_UNSET) {
1095 /* generate reports */
1096 if (!faulty && 0 != generate_monthly_output(conf, data->data.state.state, *(data->key) ? data->key : NULL)) {
1097 fprintf(stderr, "%s.%d: output generation failed for %s: no data this month?\n",
1098 __FILE__, __LINE__, data->key);
1099 faulty = 1;
1100 }
1101
1102 /* generate history */
1103 if (!faulty && 0 != generate_history_output(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
1104 fprintf(stderr, "%s.%d: history generation failed for %s\n",
1105 __FILE__, __LINE__, data->key);
1106 faulty = 1;
1107 }
1108 } else {
1109 fprintf(stderr, "%s.%d: skipped monthly / history output generation for %s, no data\n",
1110 __FILE__, __LINE__, data->key);
1111 }
1112 }
1113
1114 free(fn);
1115 }
1116
1117 MTIMER_STOP(post_process_timer);
1118 MTIMER_CALC(post_process_timer);
1119
1120 mlist_free(state_list);
1121 mrecord_free(rec);
1122
1123 if (conf->debug_level > 0) {
1124 printf(" --> Setup : Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
1125 MTIMER_GET_WALL_MSEC(setup_timer)/1000.0,
1126 MTIMER_GET_USER_MSEC(setup_timer)/1000.0,
1127 MTIMER_GET_SYSTEM_MSEC(setup_timer)/1000.0);
1128 printf(" --> Parse : Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
1129 MTIMER_GET_WALL_MSEC(parse_timer)/1000.0,
1130 MTIMER_GET_USER_MSEC(parse_timer)/1000.0,
1131 MTIMER_GET_SYSTEM_MSEC(parse_timer)/1000.0);
1132 printf(" --> Process : Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
1133 MTIMER_GET_WALL_MSEC(process_timer)/1000.0,
1134 MTIMER_GET_USER_MSEC(process_timer)/1000.0,
1135 MTIMER_GET_SYSTEM_MSEC(process_timer)/1000.0);
1136 printf(" --> Post-Process: Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
1137 MTIMER_GET_WALL_MSEC(post_process_timer)/1000.0,
1138 MTIMER_GET_USER_MSEC(post_process_timer)/1000.0,
1139 MTIMER_GET_SYSTEM_MSEC(post_process_timer)/1000.0);
1140
1141 printf("%s: %.2f %s (%ld %s, %ld %s, %ld %s, %ld %s)\n",
1142 "Throughput", (lines)/((MTIMER_GET_USER_MSEC(parse_timer)+MTIMER_GET_USER_MSEC(process_timer))/1000.0),
1143 "rec/s",
1144 lines, "records",
1145 lines_corrupt, "corrupt records",
1146 lines_skipped, "skipped records",
1147 lines_ignored, "ignored records"
1148 );
1149 }
1150 #if 0
1151 fprintf(stdout, "mhash: c: %d, s: %d\n", mem_mhash_count, mem_mhash_size);
1152 fprintf(stdout, "mlist: c: %d, s: %d\n", mem_mlist_count, mem_mlist_size);
1153 fprintf(stdout, "mdata: c: %d, s: %d\n", mem_mdata_count, mem_mdata_size);
1154
1155 for (i = 0; i < M_DATA_TYPE_IPPLWATCH; i++) {
1156 fprintf(stdout, "mdata %d: c: %d\n", i, mem_mdata_type_count[i]);
1157 }
1158 #endif
1159 mconfig_free(conf);
1160 return 0;
1161 }
1162