1 /***************************************************************************
2 * Copyright (C) 2004, 2005, 2006 by Stephen McInerney *
3 * spm@stedee.id.au *
4 * *
5 * $Id: dnshistory.c 66 2006-06-24 23:50:26Z steve $
6 * *
7 * This program is free software; you can redistribute it and/or modify *
8 * it under the terms of the GNU General Public License as published by *
9 * the Free Software Foundation; either version 2 of the License, or *
10 * (at your option) any later version. *
11 * *
12 * This program is distributed in the hope that it will be useful, *
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
15 * GNU General Public License for more details. *
16 * *
17 * You should have received a copy of the GNU General Public License *
18 * along with this program; if not, write to the *
19 * Free Software Foundation, Inc., *
20 * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
21 * *
22 ***************************************************************************/
23
24 /***************************************************************************
25 * dnshistory
26 *
27 * a web logfile preparser for extracting IP addresses and doing lookups
28 *
29 ***************************************************************************/
30
31 /***************************************************************************
32 * PURPOSE:
33 *
34 * Given a log file (stdin or cmdline) parse the address field and record
35 * a history of DNS information and changes over time
36 * Time used is _current_ time NOT log time
37 *
38 * Expects addresses to be solely in IP format (xxx.yyy.zzz.aaa)
39 * Stores the results in a Berkely DB
40 * Requires PCRE to extract the address - a bit over kill, but more
41 * flexible over the longer term
42 *
43 * Files specified as arguments can be in .gz format
44 *
45 * Operates in one of SIX modes:
46 * 1. Given a logfile, lookup IP addresses and store FQD names in a Berkeley DB
47 * 2. Given two log files:
48 * One with IP Address
49 * Other being same log but with IP Address replaced with FQD Names
50 * Store the IP Address/FQDN pairs in a Berkeley DB
51 * This option assumes #1 was done some time in the past, and we now
52 * wish to store the values in this Berkeley DB. eg via dnstran
53 * 3. Given a logfile, retrieve FQD names from a Berkely DB
54 * Replacing the IP Address from the Log with the FQDN;
55 * display the now altered log line
56 * 4. Dump the DB out as a simple text file
57 * a "NONAME" fqdn implies a failure to lookup a name, conversely
58 * the IP address as the fqdn implies a successful failure to get a name
59 * 5. Read same back in
60 * 6. Do history retrievals from CMD line IP Addresses
61 *
62 ***************************************************************************/
63
64 /***************************************************************************
65 ***************************************************************************
66 * Logic:
67 *
68 * Case 1:
69 * Grab and Hold the current time
70 * Read a log file
71 * Extract the address from each line
72 * Check for last lookup time on this record
73 * - use the 'current time' as the basis for comparison
74 * If appropriate, fire off a new thread to deal with the name resolution
75 * Each individual thread will store the updated result in a Berkeley DB
76 *
77 * Case 2:
78 * Read the RAW Log (has no lookups yet done)
79 * Read the converted Log
80 * Grab the time from the log entry
81 * Check for last lookup time on this record
82 * - use the log time as the basis for comparison
83 * If appropriate, store a new record with a new time
84 *
85 * Case 3:
86 * Read the supplied log
87 * Lookup the IP Address from a previous DB
88 * Replace the IP Address with the retrieved FQDN and print to STDOUT
89 * If no value, simply send the IP Address
90 * Print to STDOUT the rest of the logline
91 *
92 * Case 4:
93 * Cursor thru the supplied DB (or use default)
94 * Display the IP Address and each time/fqdn stored value
95 * for this IP Address on a single line
96 * tab separated list between time/fqdn pairs
97 * comma separated between time and fqdn values
98 *
99 * Case 5:
100 * Open the import file
101 * Parse the import file, record chunk by chunk
102 * Create a normal "dnshistory" structure/linked list
103 * Store via the normal method
104 *
105 * Case 6:
106 * Given one or more IP Address on the command line
107 * Display the IP Address and each time/fqdn stored value
108 * for this IP Address on a single line
109 * tab separated list between time/fqdn pairs
110 * comma separated between time and fqdn values
111 *
112 *
113 * IP Addresses are stored as in_addr - see netinet/in.h (not char[15])
114 *
115 ***************************************************************************
116 ***************************************************************************/
117
118 /***************************************************************************
119 ***************************************************************************
120 * TODO:
121 *
122 ***************************************************************************
123 ***************************************************************************/
124
125 /***************************************************************************
126 ***************************************************************************
127 * BUGS:
128 * + Expects addresses to be solely in IP4 format (xxx.yyy.zzz.aaa)
129 * + Doesn't check for already converted logs
130 * + Can't deal with log lines with escaped quotes: \"
131 *
132 ***************************************************************************
133 ***************************************************************************/
134
135 /***************************************************************************
136 ***************************************************************************
137 * ModificationHistory:
138 **********************
139 * 11-Jun-2004 steve Initial Creation; modified from visitors.c
140 * 18-Mar-2005 steve Modularise and add output ability
141 * 12-Aug-2005 steve flag getnameinfo to error on no resolution
142 * 10-Oct-2005 steve Add showhistory
143 * Refactor for display output
144 * 24-Dec-2005 steve Open DB's as RO or RW as appropriate
145 * 02-Jan-2006 steve Add import (Case 5)
146 *
147 ***************************************************************************
148 * CODING STYLE:
149 * indent -kr -bad -bap -bbo -nbc -br -brs -c49 -cd49 -i4 -ce -cp17 -l180 -nut -npcs -psl
150 ***************************************************************************
151 ***************************************************************************/
152
153
154 #include "dnshistory.h"
155
156 /************************************************************************
157 * GLOBALS *
158 ************************************************************************/
159 /*-- Date/Time --*/
160 time_t current_day; /* The current day, check to see if we need to update a record in this run
161 Also see SAME_RUN */
162
163 /*-- DB Setup --*/
164 DB *dnshistory_db_ptr; /* Database Pointer */
165 char g_db_dirfilename[MAX_FILENAME_LENGTH] = DATABASE; /* File name for the Database */
166
167 int rtn_db; /* Value from DB calls; Used for program exit if necessary */
168 unsigned long int number_added_addresses = 0; /* How many Addresses did we save/store? */
169 unsigned long int number_name_lookups = 0; /* How many Name Lookups did we perform? */
170 unsigned long int number_retries = 0; /* How many name lookup retries */
171 unsigned long int number_successful_retries = 0; /* How many name lookup retries */
172 unsigned long int number_successful_fqdns = 0; /* How many FQDNs did we store, vs just IP's */
173 unsigned long int total_lines = 0; /* How many log lines in total */
174 unsigned long int bad_lines = 0; /* How many dud log lines did we get? */
175 unsigned long int bad_recom_lines = 0; /* How many dud recombined log lines did we get? */
176 unsigned long int bad_mismatched_lines = 0; /* How many mismatched lines did we get? */
177
178 pcre *cmp_log_regexp = NULL; /* Main compiled RE - use as pointer only to one of the below */
179 pcre *cmp_log_regexp_clf = NULL; /* CLF compiled RE */
180 pcre *cmp_log_regexp_xferlog = NULL; /* FTP, xferlog format compiled RE */
181 pcre *cmp_log_regexp_squid = NULL; /* SQUID format compiled RE */
182 pcre *cmp_log_regexp_combined_enhanced = NULL; /* Enhanced Combined compiled RE */
183 pcre *cmp_log_regexp_iptables = NULL; /* syslog/linux-iptables compiled RE */
184 pcre *cmp_log_regexp_syslog = NULL; /* syslog compiled RE - used to test for possible iptables*/
185
186 int g_log_type = 0; /* What type of log file is this? LOG_???? */
187
188 /************************************************************************
189 * FUNCTIONS *
190 ************************************************************************/
191
192 /* These two are the main worker functions - call as new threads */
193 void *add_new_address(void *); /* Haven't seen this address before. Do the name lookup; Add a new entry. */
194 void *add_old_address(void *); /* Have! seen this address before. Do the name lookup; Update the old entry */
195
196 void store_dns_records(dns_record_t *); /* Store a dns record in our DB */
197
198 void add_address(char *); /* Given an IP address; create a new thread to do a lookup/DB Update */
199 void retrieve_address(char *, char *, char *); /* Retrieve the FQDN from the DB */
200 void add_recombined_address(char *, char *, char *);
201
202 void check_n_fix_fqdn(char *); /* strip any funky chars from a FQDN */
203 int name_lookup(struct in_addr ipaddr, char *); /* Do the actual name lookup */
204 int address_exists(dns_record_t *); /* Does this IP address already exist, looked up? */
205 void name_lookup_errors(int); /* Deal with name lookup errors */
206 void drop_long_lines(gzFile *, char *, buffer_position *); /* Strip Log Lines that are too big */
207 char *get_log_line(char *, int, gzFile *, buffer_position *); /* Faster version of gzgets */
208
209 void dump_dns_historydb(DB **); /* Dump the DB out */
210 void showhistory(DB **, int argc, char *[]);
211 void import_dns_historydb(void);
212
213 void increase_thread_counter(void);
214 void decrease_thread_counter(void);
215
216 void close_exit(int);
217
218 void re_compile_all_regexes(void);
219 int identify_log_format(char *);
220
221 /************************************************************************
222 * MUTEXS *
223 ************************************************************************/
224 pthread_mutex_t mutex_thread_count = PTHREAD_MUTEX_INITIALIZER; /* Lock access to thread_count */
225 unsigned int thread_count = 0; /* How many outstanding threads - to know when to exit */
226 unsigned int thread_count_max = 0; /* How many maximum number of concurrent threads */
227 pthread_cond_t cond_thread_count = PTHREAD_COND_INITIALIZER; /* Wait for another thread to signal termination */
228
229 pthread_mutex_t mutex_db_access = PTHREAD_MUTEX_INITIALIZER; /* Lock access to DB activites */
230 pthread_mutex_t mutex_malloc_dns_rec = PTHREAD_MUTEX_INITIALIZER;
231 int malloc_dns_rec = 0;
232 pthread_mutex_t mutex_malloc_dns_list = PTHREAD_MUTEX_INITIALIZER;
233 int malloc_dns_list = 0;
234
235 pthread_mutex_t mutex_other_counters = PTHREAD_MUTEX_INITIALIZER; /* Lock access to general counters */
236
237 /************************************************************************
238 ************************************************************************
239 *** MAIN ***
240 ************************************************************************
241 ************************************************************************/
242 /* MAIN */
243 int
main(int argc,char * argv[])244 main(int argc, char *argv[])
245 {
246 /**********************************************************************/
247 pcre_struct main_pcre;
248 pcre_struct recombine_pcre;
249
250 char buffer_primary[BUFSIZE]; /* Primary log buffer */
251 int buffer_length; /* How many char's currently in the Primary Buffer */
252 char buffer_recombine[BUFSIZE]; /* Recombine log buffer */
253 int buffer_recombine_length; /* How many char's currently in the Recombine Buffer */
254 char buffer_tmp_output[BUFSIZE]; /* Temporary buffer for output displaying */
255
256 bool have_set_logtype_flag = false; /* Set to true when we've set the log type */
257
258 /* Log File handlers */
259 gzFile *file_input = NULL; /* Input File descriptor */
260 gzFile *file_recombine_input = NULL; /* Recombine Input File descriptor */
261 char *fgets_rtn = NULL; /* Return value from doing fgets. Check for end of file */
262
263 char str_address[NI_MAXHOST]; /* IP Addresses */
264 char str_address2[NI_MAXHOST]; /* IP Addresses for iptables lookups */
265 char str_previous_address[NI_MAXHOST]; /* Previous IP Addresses */
266 char str_fqdn[NI_MAXHOST] = ""; /* Retrieved FQDN Address */
267 char str_fqdn2[NI_MAXHOST] = ""; /* Retrieved FQDN Address for iptables lookups */
268 char *buf_ptr; /* Offset pointer into the file's buffer - used to print from other than the start */
269
270 char str_time[SIZE_DATE_TIME]; /* String to hold the current time out of the current log line */
271 char str_time_raw[SIZE_DATE_TIME];
272 char access_size_raw[25];
273 char access_size_recombine[25];
274 int comp_ret = 0;
275
276 /* Recombine Variables */
277 bool badlogline_flag = false; /* Used to indictate we need to "continue" the main loop due to a long line error */
278 char str_address_recombine[NI_MAXHOST]; /* Recombined Addresses - should be FQDN or raw IP Address */
279 char *bufer_recombine_ptr; /* Offset pointer into the recombine file's buffer - used to print from other than the start */
280
281 buffer_position buf_posn;
282 buffer_position buf_recombine_posn;
283
284 unsigned int position_address = 0;
285 unsigned int position_datetime = 0;
286
287 /**********************************************************************/
288
289 /*****************************
290 * BEGIN
291 *****************************/
292 process_options(argc, argv);
293
294 /* Dumping the Database is easy
295 * Do so and exit */
296 if (g_dumpdnsdb == true) {
297 open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_RDONLY);
298 dump_dns_historydb(&dnshistory_db_ptr);
299 close_exit(V_EXIT_OK);
300 }
301 /* Likewise with individual IP lookups */
302 if (g_showhistory == true) {
303 open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_RDONLY);
304 showhistory(&dnshistory_db_ptr, argc, argv);
305 close_exit(V_EXIT_OK);
306 }
307 /* Do the import and exit if chosen */
308 if (g_doimport == true) {
309 open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_CREATE | DB_EXCL);
310 import_dns_historydb();
311 close_exit(V_EXIT_OK);
312 }
313 if (g_dotranslate == true) {
314 open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_RDONLY);
315 } else {
316 open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_CREATE);
317 }
318
319
320 /* From here on, we're looping over log files */
321 VPRINT(VERBOSE4, "Setting pthread stacksize to: %d\n", THREAD_STACK_SIZE);
322
323 re_compile_all_regexes();
324
325 buf_posn.current_pos_ptr = buf_posn.decomp_buf + DECOMP_BUFSIZE;
326 buf_posn.end_decompbuf_ptr = NULL;
327
328 /* Open Input Log File if appropriate */
329 if (g_filename != NULL) {
330 VPRINT(VERBOSE4, "Using file: %s\n", g_filename);
331 /* use_file = true; */
332 file_input = gzopen(g_filename, "rb");
333 ERR_NULL_EXIT(file_input, V_EXIT_BAD_FILE_OPEN, msg_F_file_open, g_filename);
334 }
335
336 /* Compile additional, detailed PCRE for line checking if doing recombining */
337 /* Open Input Recombine Log File if appropriate */
338 if (g_dorecombine == true) {
339 memset(&recombine_pcre, 0, sizeof(recombine_pcre));
340 strncpy(recombine_pcre.regular_expression, PATTERN_COMBINED_ENHANCED, MAX_RE_LENGTH);
341 VPRINT(VERBOSE3, "Recombine Reg Ex Pattern: %s\n", recombine_pcre.regular_expression);
342 recombine_pcre.re_pcre = pcre_compile(recombine_pcre.regular_expression, 0, &recombine_pcre.error, &recombine_pcre.erroffset, NULL);
343 if (main_pcre.re_pcre == NULL) {
344 re_compile_failed(recombine_pcre.erroffset, recombine_pcre.error, recombine_pcre.regular_expression);
345 }
346
347 buf_recombine_posn.current_pos_ptr = buf_recombine_posn.decomp_buf + DECOMP_BUFSIZE;
348 buf_recombine_posn.end_decompbuf_ptr = NULL;
349
350 VPRINT(VERBOSE4, "Using Recombine file: %s\n", g_recombine_filename);
351 file_recombine_input = gzopen(g_recombine_filename, "rb");
352 ERR_NULL_EXIT(file_recombine_input, V_EXIT_BAD_FILE_OPEN, msg_F_file_open, g_recombine_filename);
353 }
354
355
356 current_day = time(NULL); /* set the current time - use for checking if a record is current to this run or not */
357 str_previous_address[0] = '\0';
358
359
360 /*************************************************************
361 * The Primary Loop!
362 *************************************************************/
363 VPRINT(VERBOSE4, "Beginning Main Loop.%s\n", "");
364 while (1) {
365 /* Read a line from file or stdin */
366 if (file_input != NULL) {
367 fgets_rtn = get_log_line(buffer_primary, BUFSIZE, file_input, &buf_posn);
368 } else {
369 fgets_rtn = fgets(buffer_primary, BUFSIZE, stdin);
370 }
371 if (fgets_rtn == NULL) {
372 /* At file end! */
373 break;
374 }
375
376 total_lines++;
377
378 /* check for log line too long */
379 buffer_length = (int) strlen(buffer_primary);
380 if (strpbrk("\n", buffer_primary) == NULL) {
381 if (file_input == NULL) {
382 drop_long_lines(NULL, buffer_primary, NULL);
383 } else {
384 drop_long_lines(file_input, buffer_primary, &buf_posn);
385 }
386 bad_lines++;
387 if (g_dorecombine == true) {
388 badlogline_flag = true;
389 } else {
390 continue;
391 }
392
393 }
394
395 /* Identify the log type on first run thru */
396 if (! have_set_logtype_flag) {
397 if (g_log_type == LOG_AUTO) {
398 g_log_type = identify_log_format(buffer_primary);
399 if (g_log_type <= 0) {
400 ERRVPRINT(VERBOSE0, "Cannot recognise log format.%s", "\n");
401 exit(1);
402 }
403 }
404 memset(&main_pcre, 0, sizeof(main_pcre));
405 switch (g_log_type) {
406 case LOG_FTP:
407 main_pcre.re_pcre = cmp_log_regexp_xferlog;
408 strncpy(main_pcre.regular_expression, PATTERN_XFERLOG, MAX_RE_LENGTH);
409 position_address = LF_XFERLOG_ADDRESS;
410 position_datetime = LF_XFERLOG_DATE_TIME;
411 break;
412 case LOG_SQUID:
413 main_pcre.re_pcre = cmp_log_regexp_squid;
414 strncpy(main_pcre.regular_expression, PATTERN_SQUID, MAX_RE_LENGTH);
415 position_address = LF_SQUID_ADDRESS;
416 position_datetime = LF_SQUID_DATE_TIME;
417 break;
418 case LOG_CLF:
419 main_pcre.re_pcre = cmp_log_regexp_clf;
420 strncpy(main_pcre.regular_expression, PATTERN_CLF, MAX_RE_LENGTH);
421 position_address = LF_NCSA_ADDRESS;
422 position_datetime = LF_NCSA_DATE_TIME;
423 break;
424 case LOG_IPTABLES:
425 main_pcre.re_pcre = cmp_log_regexp_iptables;
426 strncpy(main_pcre.regular_expression, PATTERN_IPTABLES, MAX_RE_LENGTH);
427 position_address = LF_IPTABLES_ADDRESS_SRC;
428 position_datetime = LF_IPTABLES_DATE_TIME;
429 break;
430 default:
431 ERRVPRINT(VERBOSE0, "Unknown LOG Type Setting. Sorry.... : %d\n", g_log_type);
432 exit(1);
433 }
434 VPRINT(VERBOSE3, "Main Reg Ex Pattern: %s\n", main_pcre.regular_expression);
435 }
436
437
438 if (badlogline_flag != true) {
439 VPRINT(VERBOSE3, "-%lu-%s", total_lines, buffer_primary);
440
441 /* Apply the pattern match. */
442 main_pcre.ret = pcre_exec(main_pcre.re_pcre, NULL, buffer_primary, buffer_length, 0, 0, main_pcre.ovector, OVECCOUNT);
443 /* check for RE matching errors */
444 if (main_pcre.ret <= 0) {
445 if (g_log_type != LOG_IPTABLES) {
446 re_check_errors(main_pcre.ret, total_lines, buffer_primary);
447 if (g_dorecombine == true) {
448 badlogline_flag = true;
449 } else {
450 continue;
451 }
452 } else {
453 /* Assume no RegEx failures with iptables. Ha! Idea is to skip over other syslog messages */
454 continue;
455 }
456 }
457
458 main_pcre.cp_substr_ret = pcre_copy_substring(buffer_primary, main_pcre.ovector, main_pcre.ret, position_address, str_address, BUFSIZE);
459 if (main_pcre.cp_substr_ret < 0) {
460 error_substring_extract(main_pcre.regular_expression, buffer_primary, position_address, main_pcre.cp_substr_ret, total_lines);
461 if (g_dorecombine == true) {
462 badlogline_flag = true;
463 } else {
464 continue;
465 }
466 }
467 if (g_log_type == LOG_IPTABLES) {
468 main_pcre.cp_substr_ret = pcre_copy_substring(buffer_primary, main_pcre.ovector, main_pcre.ret,LF_IPTABLES_ADDRESS_DST , str_address2, BUFSIZE);
469 if (main_pcre.cp_substr_ret < 0) {
470 error_substring_extract(main_pcre.regular_expression, buffer_primary, position_address, main_pcre.cp_substr_ret, total_lines);
471 if (g_dorecombine == true) {
472 badlogline_flag = true;
473 } else {
474 continue;
475 }
476 }
477 }
478 }
479
480 if (g_dotranslate == true) {
481 /*************************************************************
482 * Do the translating
483 *************************************************************/
484 /* Retrieve correct fqdn from DB by date match.
485 * Spit out full line with replaced IP Address */
486 comp_ret = strncmp(str_previous_address, str_address, SIZE_ADDRESS);
487 if (comp_ret != 0) {
488 /* Only update the FQDN if the most recent address has changed */
489
490 /* Extract Date/Time */
491 main_pcre.cp_substr_ret = pcre_copy_substring(buffer_primary, main_pcre.ovector, main_pcre.ret, position_datetime, str_time, SIZE_DATE_TIME);
492 if (main_pcre.cp_substr_ret < 0) {
493 ERRVPRINT(VERBOSE0, msg_F_vital_substring, position_datetime);
494 close_exit(V_EXIT_PCRE_NO_SUBSTR);
495 }
496
497 retrieve_address(str_address, str_fqdn, str_time);
498 strncpy(str_previous_address, str_address, SIZE_ADDRESS);
499
500 if (g_log_type == LOG_IPTABLES) {
501 retrieve_address(str_address2, str_fqdn2, str_time);
502 }
503 }
504 if (g_log_type == LOG_IPTABLES) {
505 if ((str_fqdn[0] == '\0') && (str_fqdn2[0] == '\0')) {
506 /* No change, display line-in as line-out */
507 printf("%s", buffer_primary);
508 } else {
509 if (str_fqdn[0] == '\0') {
510 buffer_tmp_output[0] = '\0';
511 strncat(buffer_tmp_output, buffer_primary, main_pcre.ovector[((LF_IPTABLES_ADDRESS_DST * 2) - 1) + 1]);
512 printf("%s", buffer_tmp_output);
513 printf("%s", str_fqdn2);
514 buffer_tmp_output[0] = '\0';
515 strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[((LF_IPTABLES_ADDRESS_DST * 2) + 1)]);
516 printf("%s", buffer_tmp_output);
517 } else {
518 buffer_tmp_output[0] = '\0';
519 strncat(buffer_tmp_output, buffer_primary, main_pcre.ovector[((position_address * 2) - 1) + 1]);
520 printf("%s", buffer_tmp_output);
521 printf("%s", str_fqdn);
522 printf(" DST=");
523 buffer_tmp_output[0] = '\0';
524 if (str_fqdn2[0] == '\0') {
525 strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[(LF_IPTABLES_ADDRESS_DST * 2)]);
526 } else {
527 printf("%s", str_fqdn2);
528 strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[((LF_IPTABLES_ADDRESS_DST * 2) + 1)]);
529 }
530 printf("%s", buffer_tmp_output);
531 }
532 }
533 } else {
534 if (str_fqdn[0] == '\0') {
535 /* No change, display line-in as line-out */
536 printf("%s", buffer_primary);
537 } else {
538 /* Have a returned value. Displayed FQDN and rest of line minus IP Address. */
539 /* print before */
540 if (position_address > 1) {
541 buffer_tmp_output[0] = '\0';
542 strncat(buffer_tmp_output, buffer_primary, main_pcre.ovector[((position_address * 2) - 1) + 1]);
543 printf("%s", buffer_tmp_output);
544 }
545 printf("%s", str_fqdn);
546 buffer_tmp_output[0] = '\0';
547 strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[((position_address * 2) + 1)]);
548 printf("%s", buffer_tmp_output);
549 }
550 }
551 } else if (g_dorecombine == true) {
552 if (g_log_type != LOG_CLF) {
553 ERRVPRINT(VERBOSE0, "Sorry, can't recombine non CLF logs. Yet. Please email author!%s", "\n");
554 exit(1);
555 }
556 /*************************************************************
557 * Do the recombining
558 *************************************************************/
559 /* Grab part of the recombine log file */
560 fgets_rtn = get_log_line(buffer_recombine, BUFSIZE, file_recombine_input, &buf_recombine_posn);
561 ERR_NULL_EXIT(fgets_rtn, V_EXIT_EARLY_LOG_CLOSE, msg_F_early_log_termination, total_lines);
562
563 buffer_recombine_length = (int) strlen(buffer_recombine);
564 /* check for log line too long */
565 if ((strpbrk("\n", buffer_recombine) == NULL) || (badlogline_flag == true)) {
566 drop_long_lines(file_recombine_input, buffer_recombine, &buf_recombine_posn);
567 if (badlogline_flag == false) {
568 bad_lines++;
569 }
570 badlogline_flag = false;
571 continue;
572 }
573
574 VPRINT(VERBOSE3, "+%lu+%s", total_lines, buffer_recombine);
575 VPRINT(VERBOSE5, " New Address: %s Old Address: %s\n", str_address, str_previous_address);
576 if (strncmp(str_previous_address, str_address, SIZE_ADDRESS) != 0) {
577 /* Only update the FQDN if the most recent address has changed */
578
579 strncpy(str_previous_address, str_address, SIZE_ADDRESS);
580
581 main_pcre.ret = pcre_exec(main_pcre.re_pcre, NULL, buffer_recombine, buffer_recombine_length, 0, 0, main_pcre.ovector, OVECCOUNT);
582 if (main_pcre.ret <= 0) {
583 re_check_errors(main_pcre.ret, total_lines, buffer_recombine);
584 bad_lines++;
585 continue;
586 }
587
588 main_pcre.cp_substr_ret =
589 pcre_copy_substring(buffer_recombine, main_pcre.ovector, main_pcre.ret, LF_NCSA_ADDRESS, str_address_recombine, sizeof(str_address_recombine));
590 if (recombine_pcre.cp_substr_ret < 0) {
591 error_substring_extract(recombine_pcre.regular_expression, buffer_recombine, LF_NCSA_ADDRESS, recombine_pcre.cp_substr_ret, total_lines);
592 bad_lines++;
593 continue;
594 }
595
596 /* Extract Date/Time */
597 main_pcre.cp_substr_ret = pcre_copy_substring(buffer_recombine, main_pcre.ovector, main_pcre.ret, LF_NCSA_DATE_TIME, str_time, sizeof(str_time));
598 if (recombine_pcre.cp_substr_ret < 0) {
599 /* Fatal Error, as we've successfully matched elsewhere. We really shouldn't ever get here... */
600 ERRVPRINT(VERBOSE0, "%s\n", msg_F_vital_substring);
601 close_exit(V_EXIT_PCRE_NO_SUBSTR);
602 }
603
604 buf_ptr = strpbrk(buffer_primary, " ");
605 bufer_recombine_ptr = strpbrk(buffer_recombine, " ");
606
607 comp_ret = strncmp(buf_ptr, bufer_recombine_ptr, BUFSIZE);
608 if (comp_ret != 0) {
609 /* We have a mismatched line.
610 * Now verify if truely borked file(s) or simply an IP Address substitution elsewhere in a line */
611
612 /* First off, we will compare the date/time stamps */
613 /* We already have the recombine buffer's date time, so grab from the "raw" logfile */
614 recombine_pcre.ret = pcre_exec(recombine_pcre.re_pcre, NULL, buffer_primary, buffer_length, 0, 0, recombine_pcre.ovector, OVECCOUNT);
615 if (recombine_pcre.ret <= 0) {
616 re_check_errors(recombine_pcre.ret, total_lines, buffer_primary);
617 bad_lines++;
618 continue;
619 }
620 recombine_pcre.cp_substr_ret =
621 pcre_copy_substring(buffer_primary, recombine_pcre.ovector, recombine_pcre.ret, LF_NCSA_DATE_TIME, str_time_raw, sizeof(str_time_raw));
622 if (recombine_pcre.cp_substr_ret < 0) {
623 error_substring_extract(recombine_pcre.regular_expression, buffer_primary, LF_NCSA_DATE_TIME, recombine_pcre.cp_substr_ret, total_lines);
624 bad_lines++;
625 continue;
626 }
627 comp_ret = strncmp(str_time_raw, str_time, strlen(str_time));
628 ERR_NONZERO_EXIT(comp_ret, V_EXIT_MISMATCHED_LINES, msg_F_mismatched_lines, total_lines, buffer_primary, buffer_recombine);
629
630 /* Date/Time ok. Lets look at size in bytes. Would use URL, but they get replaced as well unf. */
631 /* Copy out the size in bytes of the access - already looking at the main log */
632 recombine_pcre.cp_substr_ret =
633 pcre_copy_substring(buffer_primary, recombine_pcre.ovector, recombine_pcre.ret, LF_NCSA_BYTES, access_size_raw, sizeof(access_size_raw));
634 if (recombine_pcre.cp_substr_ret < 0) {
635 error_substring_extract(recombine_pcre.regular_expression, buffer_primary, LF_NCSA_BYTES, recombine_pcre.cp_substr_ret, total_lines);
636 bad_lines++;
637 continue;
638 }
639 /* Now from the recombine file/buffer */
640 recombine_pcre.ret = pcre_exec(recombine_pcre.re_pcre, NULL, buffer_recombine, buffer_recombine_length, 0, 0, recombine_pcre.ovector, OVECCOUNT);
641 if (recombine_pcre.ret <= 0) {
642 re_check_errors(recombine_pcre.ret, total_lines, buffer_recombine);
643 bad_lines++;
644 continue;
645 }
646 recombine_pcre.cp_substr_ret =
647 pcre_copy_substring(buffer_recombine, recombine_pcre.ovector, recombine_pcre.ret, LF_NCSA_BYTES, access_size_recombine, sizeof(access_size_recombine));
648 if (recombine_pcre.cp_substr_ret < 0) {
649 error_substring_extract(recombine_pcre.regular_expression, buffer_primary, LF_NCSA_DATE_TIME, recombine_pcre.cp_substr_ret, total_lines);
650 bad_lines++;
651 continue;
652 }
653 comp_ret = strncmp(access_size_raw, access_size_recombine, strlen(access_size_raw));
654 ERR_NONZERO_EXIT(comp_ret, V_EXIT_MISMATCHED_LINES, msg_F_mismatched_lines, total_lines, buffer_primary, buffer_recombine);
655
656 ERRVPRINT(VERBOSE2, msg_W_mismatched_lines, total_lines, buffer_primary, buffer_recombine);
657 bad_mismatched_lines++;
658 }
659
660 add_recombined_address(str_address, str_address_recombine, str_time);
661 }
662 } else { /* Default ==> (g_dolookups == true) */
663 /*************************************************************
664 * Default Action - do the lookups and store results in DB
665 *************************************************************/
666
667 /* Rapid check: have we seen this IP before?
668 * Typically we expect to see multiple entries to the same address, one after the other
669 * If not, add the new one, and update the old address */
670 comp_ret = strncmp(str_previous_address, str_address, SIZE_ADDRESS);
671 if (comp_ret != 0) {
672 /* Don't exceed Maximum Number of threads! Stop and hold, once we get too many */
673 pthread_mutex_lock(&mutex_thread_count);
674 while (thread_count >= g_max_threads) {
675 VPRINT(VERBOSE2, "HOLDING: Exceeding Maximum Thread Count!: %d <= %d\n", g_max_threads, thread_count);
676 pthread_cond_wait(&cond_thread_count, &mutex_thread_count);
677 }
678 pthread_mutex_unlock(&mutex_thread_count);
679 add_address(str_address);
680 strncpy(str_previous_address, str_address, SIZE_ADDRESS);
681 }
682 if (g_log_type == LOG_IPTABLES) {
683 /* Don't exceed Maximum Number of threads! Stop and hold, once we get too many */
684 pthread_mutex_lock(&mutex_thread_count);
685 while (thread_count >= g_max_threads) {
686 VPRINT(VERBOSE2, "HOLDING: Exceeding Maximum Thread Count!: %d <= %d\n", g_max_threads, thread_count);
687 pthread_cond_wait(&cond_thread_count, &mutex_thread_count);
688 }
689 pthread_mutex_unlock(&mutex_thread_count);
690 add_address(str_address2);
691 }
692 }
693 }
694
695
696 if (g_dolookups == true) {
697 pthread_mutex_lock(&mutex_thread_count);
698 while (thread_count > 0) {
699 VPRINT(VERBOSE5, "FINISHING: Countdown thread count: %d\n", thread_count);
700 pthread_cond_wait(&cond_thread_count, &mutex_thread_count);
701 }
702
703 VPRINT(VERBOSE1, "Maximum Concurrent Threads: %d\n", thread_count_max);
704 VPRINT(VERBOSE5, "Final Remaining DNS Records: %d\n", malloc_dns_rec);
705 VPRINT(VERBOSE5, "Final Remaining DNS Lists: %d\n", malloc_dns_list);
706 }
707
708 VPRINT(VERBOSE1, "Stored %lu Addresses from %lu Log Lines\n", number_added_addresses, total_lines);
709 if (number_name_lookups > 0) {
710 VPRINT(VERBOSE1, "Number of Name Lookups Performed: %lu of %lu Log Lines\n", number_name_lookups, total_lines);
711 }
712 if (bad_lines > 0) {
713 VPRINT(VERBOSE0, "%s%lu of %lu\n", msg_I_number_bad_lines, bad_lines, total_lines);
714 }
715 if (bad_recom_lines > 0) {
716 VPRINT(VERBOSE0, "%s%lu of %lu\n", "Number of Bad Recombined Lines: ", bad_recom_lines, total_lines);
717 }
718 if (bad_mismatched_lines > 0) {
719 VPRINT(VERBOSE0, "%s%lu of %lu\n", "Number of Possibly Mismatched Lines: ", bad_mismatched_lines, total_lines);
720 }
721
722 VPRINT(VERBOSE1, "%s%lu of %lu\n", "Number of Stored FQDNs: ", number_successful_fqdns, number_name_lookups);
723 VPRINT(VERBOSE1, "%s%lu of %lu\n", "Number of Successful Retries: ", number_successful_retries, number_retries);
724
725 close_dnshistory_db(&dnshistory_db_ptr);
726 return (V_EXIT_OK);
727 }
728
729 /************************************************************************
730 ************************************************************************
731 * END *
732 ************************************************************************
733 ************************************************************************/
734
735
736 /************************************************************************
737 * add_address *
738 * *
739 * Given an IP Address, do the lookup, and add the FQDN to our DNS DB *
740 ************************************************************************/
741 void
add_address(char * str_ipaddr)742 add_address(char *str_ipaddr)
743 {
744 /* Logic:
745 * Convert from char to ip_addr
746 * Seen this IP address today?
747 * Yes:
748 * exit ok
749 * No:
750 * Lookup this IP Address (using char form!)
751 * New record?
752 * Yes:
753 * Initialize record and add to db
754 * date_set and date_last to same time
755 * Create new record
756 * No:
757 * Has the fqdn changed?
758 * Yes:
759 * Create a new record with date_set and date_last same time
760 * No:
761 * Update date_last to current day
762 * Update existing record
763 */
764
765 /**********************************************************************/
766 dns_record_t *dnsrec; /* Current DNS Record to work with */
767 pthread_t pthrd; /* Thread id - ignored */
768 pthread_attr_t pthrd_attrs; /* Thread Attributes */
769 int ret_thrd; /* Thread function results */
770 int ret_pton; /* Results: inet_pton */
771 struct dns_record_lists_t *list, *list_next; /* list cleanup pointers */
772
773 /**********************************************************************/
774
775 VPRINT(VERBOSE5, " INIT. Add Address; Do Lookup on: %s\n", str_ipaddr);
776
777 dnsrec = XMALLOC(dns_record_t, 1);
778 pthread_mutex_lock(&mutex_malloc_dns_rec);
779 malloc_dns_rec++;
780 pthread_mutex_unlock(&mutex_malloc_dns_rec);
781 dnsrec->list = NULL; /* Probably unnecessary, but can't hurt */
782
783 /* Convert from char string to in_addr_t */
784 ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec->ipaddress);
785 if (ret_pton == 0) {
786 /* XXX What happens if already converted??? */
787 ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
788 return;
789 }
790
791 /* Set the stack size for threads */
792 pthread_attr_init(&pthrd_attrs);
793 ret_thrd = pthread_attr_setstacksize(&pthrd_attrs, THREAD_STACK_SIZE);
794 if (ret_thrd != 0) {
795 ERRVPRINT(VERBOSE0, msg_E_thread_stack_resize, ret_thrd);
796 }
797
798 if (!address_exists(dnsrec)) {
799 /* Is a new record. So, create threads with reduced stack and fire off */
800 VPRINT(VERBOSE5, " New Record%s", "\n");
801 /* First lets store a copy of this record, so we don't do repeated name lookups */
802 dnsrec->date_last = current_day;
803 number_added_addresses++;
804 store_dns_records(dnsrec);
805 increase_thread_counter();
806 ret_thrd = pthread_create(&pthrd, &pthrd_attrs, add_new_address, (void *) dnsrec);
807 if (ret_thrd != 0) {
808 ERRVPRINT(VERBOSE0, msg_E_thread_creation, ret_thrd);
809 }
810 } else {
811 /* We have seen this record before */
812 VPRINT(VERBOSE5, " Old Record: %lu --> %lu\n", dnsrec->date_last, current_day);
813 if (dnsrec->date_last < (current_day - g_dns_timeout)) {
814 /* But not in our timeout period - so need to update with a new record! */
815 VPRINT(VERBOSE5, " Outside timeout period. New Lookup.%s", "\n");
816 /* First store an updated "date_last".
817 * This way later requests will ignore this record if the name lookup takes a while */
818 dnsrec->date_last = current_day;
819 number_added_addresses++;
820 store_dns_records(dnsrec);
821 increase_thread_counter();
822 ret_thrd = pthread_create(&pthrd, &pthrd_attrs, add_old_address, (void *) dnsrec);
823 if (ret_thrd != 0) {
824 ERRVPRINT(VERBOSE0, msg_E_thread_creation, ret_thrd);
825 }
826 } else {
827 /* Can ignore this record, deallocate it and all it's links */
828 list = dnsrec->list;
829 VPRINT(VERBOSE5, " Inside timeout period. Free Record(s).%s", "\n");
830 while (list != NULL) {
831 VPRINT(VERBOSE5, " Freeing List Entry: %s\n", list->fqdn);
832 list_next = list->next;
833 XFREE(list);
834 list = list_next;
835 pthread_mutex_lock(&mutex_malloc_dns_list);
836 malloc_dns_list--;
837 pthread_mutex_unlock(&mutex_malloc_dns_list);
838 }
839 XFREE(dnsrec);
840 pthread_mutex_lock(&mutex_malloc_dns_rec);
841 malloc_dns_rec--;
842 pthread_mutex_unlock(&mutex_malloc_dns_rec);
843 }
844 }
845 }
846
847
848 /************************************************************************
849 * retrieve_address *
850 * *
851 * Given an IP Address, do the lookup, *
852 * and return the FQDN from the DNS DB *
853 * *
854 * str_fqdn[0] is set to '\0' on a "no valid answer" answer. *
855 ************************************************************************/
856 void
retrieve_address(char * str_ipaddr,char * str_fqdn,char * str_time)857 retrieve_address(char *str_ipaddr, char *str_fqdn, char *str_time)
858 {
859 /**********************************************************************/
860 dns_record_t *dnsrec; /* Current DNS Record to work with */
861 struct dns_record_lists_t *list, *list_next; /* list cleanup/traversing pointers */
862 int ret_pton; /* Results: inet_pton */
863 struct tm time_rec; /* Gotta convert that string'ed time into a timerec first */
864 time_t time_logentry = 0; /* The current log time */
865 int ret_addrexists = 0;
866 time_t temp_time_squid; /* For pulling in squid times */
867
868 /**********************************************************************/
869
870 VPRINT(VERBOSE5, " INIT. Retrieve Address; Looking for: %s\n", str_ipaddr);
871
872 dnsrec = XMALLOC(dns_record_t, 1);
873 dnsrec->list = NULL; /* Probably unnecessary, but can't hurt */
874
875 /* Convert from char string to in_addr_t */
876 ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec->ipaddress);
877 if (ret_pton == 0) {
878 /* XXX What happens if already converted??? */
879 ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
880 str_fqdn[0] = '\0';
881 } else {
882
883 ret_addrexists = address_exists(dnsrec);
884 if ((ret_addrexists = 0) || (dnsrec->list == NULL)) {
885 /* No answer to give */
886 str_fqdn[0] = '\0';
887 } else {
888 /* Find and return the correct answer by timestamp */
889 /******************
890 * Use this entry if no older entry
891 * time_logentry newer or same as date_set use this entry
892 * time_logentry older than date_set, try next older;
893 ******************/
894 list = dnsrec->list;
895
896 if (list->next != NULL) {
897 memset(&time_rec, 0, sizeof(time_rec)); /* reset time_rec - failing to do so causes time comparison problems. Yuk */
898 switch (g_log_type) {
899 case LOG_CLF:
900 strptime(str_time, DATE_TIME_FORMAT, &time_rec);
901 break;
902 case LOG_FTP:
903 strptime(str_time, DATE_TIME_XFERLOG_FORMAT, &time_rec);
904 break;
905 case LOG_IPTABLES:
906 strptime(str_time, DATE_TIME_IPTABLES_FORMAT, &time_rec);
907 break;
908 case LOG_SQUID:
909 temp_time_squid = strtoul(str_time, NULL, 10);
910 localtime_r(&temp_time_squid, &time_rec);
911 break;
912 }
913
914 time_logentry = mktime(&time_rec);
915 VPRINT(VERBOSE5, " **Date/Time: %s -> %ld\n", str_time, time_logentry);
916 VPRINT(VERBOSE5, " **Date/Time: DB START: %ld -> %ld\n", time_logentry, dnsrec->date_last);
917
918 while (list != NULL) {
919 VPRINT(VERBOSE5, " **Date/Time: DB List: %ld -> %ld\n", time_logentry, list->date_set);
920 if ((list->next == NULL) || (difftime(time_logentry, list->date_set) >= 0)) {
921 strncpy(str_fqdn, list->fqdn, NI_MAXHOST);
922 }
923 list = list->next;
924 }
925 } else {
926 /* We only have one entry, so no need to do a time compare */
927 strncpy(str_fqdn, list->fqdn, NI_MAXHOST);
928 }
929 }
930 }
931
932 /* Can ignore this record, deallocate it and all it's links */
933 list = dnsrec->list;
934 VPRINT(VERBOSE5, " Inside timeout period. Free Record(s).%s", "\n");
935 while (list != NULL) {
936 VPRINT(VERBOSE5, " Freeing List Entry: %s\n", list->fqdn);
937 list_next = list->next;
938 XFREE(list);
939 list = list_next;
940 }
941 XFREE(dnsrec);
942
943 return;
944 }
945
946
947 /************************************************************************
948 * add_recombined_address *
949 * *
950 * Given an IP Address and FQDN , add to our DNS DB *
951 ************************************************************************/
952 void
add_recombined_address(char * str_ipaddr,char * str_fqdn,char * str_time)953 add_recombined_address(char *str_ipaddr, char *str_fqdn, char *str_time)
954 {
955 /* Logic:
956 */
957
958 /**********************************************************************/
959 dns_record_t *dnsrec; /* Current DNS Record to work with */
960 int ret_pton; /* Results: inet_pton */
961 struct dns_record_lists_t *list, *list_next; /* list cleanup pointers */
962 struct dns_record_lists_t dnsrec_list; /* Single "list" entry for this record */
963 struct tm time_rec; /* Gotta convert that string'ed time into a timerec first */
964 time_t time_logentry = 0; /* The current log time */
965
966 /**********************************************************************/
967
968 VPRINT(VERBOSE3, " **INIT: Add Recombined Address; Given: %s Add: %s\n", str_ipaddr, str_fqdn);
969
970 dnsrec = XMALLOC(dns_record_t, 1);
971 dnsrec->list = NULL; /* Probably unnecessary, but can't hurt */
972
973 /* Convert from char string to in_addr_t */
974 ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec->ipaddress);
975 if (ret_pton == 0) {
976 /* XXX What happens if already converted??? */
977 ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
978 return;
979 }
980
981 /* Convert the Log Date/Time to time_t */
982 memset(&time_rec, 0, sizeof(time_rec)); /* reset time_rec - failing to do so causes time comparison problems. Yuk */
983 strptime(str_time, DATE_TIME_FORMAT, &time_rec);
984 time_logentry = mktime(&time_rec);
985 VPRINT(VERBOSE5, " ARA: Date/Time: %s -> %ld\n", str_time, time_logentry);
986
987 /************************************************
988 * New Record?
989 ************************************************/
990 if (!address_exists(dnsrec)) {
991 /* This is a new record. */
992 VPRINT(VERBOSE4, " ARA: New Record%s", "\n");
993
994 memset(&dnsrec_list, 0, sizeof(dnsrec_list));
995 dnsrec->list = &dnsrec_list;
996
997 strncpy(dnsrec->list->fqdn, str_fqdn, NI_MAXHOST);
998 dnsrec->date_last = time_logentry;
999 dnsrec->list->date_set = time_logentry;
1000 dnsrec->list->next = NULL;
1001
1002 /* Store this newly updated new record */
1003 VPRINT(VERBOSE2, " ARA: Storing New: %15s %s\n", str_ipaddr, str_fqdn);
1004 number_added_addresses++;
1005 store_dns_records(dnsrec);
1006 XFREE(dnsrec);
1007 } else {
1008 /************************************************
1009 * Not New Record!
1010 ************************************************/
1011 VPRINT(VERBOSE5, " ARA: Old Record: %lu --> %lu\n", dnsrec->date_last, time_logentry);
1012 if (dnsrec->date_last < (time_logentry - g_dns_timeout)) {
1013 /* But not in our timeout period - so need to update with a new record! */
1014 VPRINT(VERBOSE5, " ARA: Outside timeout period. Update Entry.%s", "\n");
1015 /* First store an updated "date_last".
1016 * This way later requests will ignore this record if the name lookup takes a while */
1017 dnsrec->date_last = time_logentry;
1018 if (strncmp(dnsrec->list->fqdn, str_fqdn, NI_MAXHOST) != 0) {
1019 /* This is a new and updated FQDN, insert */
1020 VPRINT(VERBOSE4, " ARA: Add Recombined Address: Changed FQDN: %s --> %s\n", dnsrec->list->fqdn, str_fqdn);
1021 list_next = XMALLOC(struct dns_record_lists_t, 1);
1022
1023 list_next->date_set = time_logentry;
1024 strncpy(list_next->fqdn, str_fqdn, NI_MAXHOST);
1025 list_next->next = dnsrec->list;
1026 dnsrec->list = list_next;
1027 VPRINT(VERBOSE2, " ARA: Storing Update: %15s %s\n", str_ipaddr, str_fqdn);
1028 }
1029 number_added_addresses++;
1030 store_dns_records(dnsrec); /* We only need to store an updated value, we've already stored a new time */
1031 }
1032 /* Deallocate it and all it's links */
1033 list = dnsrec->list;
1034 VPRINT(VERBOSE5, " ARA: Inside timeout period. Free Record(s).%s", "\n");
1035 while (list != NULL) {
1036 VPRINT(VERBOSE5, " ARA: Freeing List Entry: %s\n", list->fqdn);
1037 list_next = list->next;
1038 XFREE(list);
1039 list = list_next;
1040 }
1041 XFREE(dnsrec);
1042 }
1043 }
1044
1045
1046 /************************************************************************
1047 * name_lookup *
1048 * *
1049 * Do the name lookup and error handling *
1050 * Requires a string[NI_MAXHOST] to dump the results into *
1051 * -> host_name *
1052 ************************************************************************/
1053 int
name_lookup(struct in_addr ipaddr,char * host_name)1054 name_lookup(struct in_addr ipaddr, char *host_name)
1055 {
1056 /**********************************************************************/
1057 int ret_gni = 0; /* getnameinfo Return value */
1058 struct sockaddr_in sa; /* Structure to send to getnameinfo */
1059
1060 sa.sin_family = AF_INET; /* IPv4 lookup */
1061 sa.sin_port = 0; /* Don't care about the port */
1062 sa.sin_addr.s_addr = ipaddr.s_addr; /* Set the IP address to lookup */
1063 int retries = g_dns_lookups;
1064 int err; /* Error Checking */
1065 bool doing_retry = 0; /* Flag to help count number of retries */
1066
1067 /**********************************************************************/
1068
1069 err = pthread_mutex_lock(&mutex_other_counters);
1070 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1071 number_name_lookups++; /* Increase number of lookups counter */
1072 err = pthread_mutex_unlock(&mutex_other_counters);
1073 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1074
1075 /* Do the lookup */
1076 while (retries > 0) {
1077 ret_gni = getnameinfo((struct sockaddr *) &sa, sizeof(sa), host_name, NI_MAXHOST, NULL, 0, NI_NAMEREQD);
1078
1079 /* Error/Success checking */
1080 if (ret_gni != 0) {
1081 if ((ret_gni == EAI_AGAIN) || (ret_gni == EAI_NONAME)) {
1082 VPRINT(VERBOSE3, "NameLookup: TryAgain: %s\n", host_name);
1083 retries--;
1084 if (retries > 0) {
1085
1086 err = pthread_mutex_lock(&mutex_other_counters);
1087 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1088 number_retries++;
1089 err = pthread_mutex_unlock(&mutex_other_counters);
1090 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1091
1092 if (doing_retry == false) {
1093 doing_retry = true;
1094 }
1095 sleep(g_dns_retry_delay);
1096 }
1097 } else {
1098 name_lookup_errors(ret_gni);
1099 break;
1100 }
1101 } else {
1102 VPRINT(VERBOSE2, "NameLookup: Success: %s\n", host_name);
1103 break;
1104 }
1105 }
1106
1107 check_n_fix_fqdn(host_name);
1108
1109 if (ret_gni == 0) {
1110 err = pthread_mutex_lock(&mutex_other_counters);
1111 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1112 number_successful_fqdns++;
1113 err = pthread_mutex_unlock(&mutex_other_counters);
1114 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1115 }
1116
1117 if ((ret_gni == 0) && (doing_retry == true)) {
1118 err = pthread_mutex_lock(&mutex_other_counters);
1119 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1120 number_successful_retries++;
1121 err = pthread_mutex_unlock(&mutex_other_counters);
1122 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1123 }
1124 /* Even tho we have a technical failure, we want to return a success for ipaddy == ipaddy lookups */
1125 if (ret_gni == EAI_NONAME) {
1126 ret_gni = 0;
1127 }
1128 return (ret_gni);
1129 }
1130
1131
1132 /************************************************************************
1133 * increase_thread_counter *
1134 * *
1135 * Do as the name says. Lock; increase; Unlock; exit. Duh. *
1136 ************************************************************************/
1137 void
increase_thread_counter()1138 increase_thread_counter()
1139 {
1140 /**********************************************************************/
1141 int err; /* Error Checking */
1142
1143 /**********************************************************************/
1144
1145 /* Increase the count of the number of threads running */
1146 err = pthread_mutex_lock(&mutex_thread_count);
1147 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1148 /* thread_count is a global */
1149 thread_count++;
1150 if (thread_count > thread_count_max) {
1151 thread_count_max = thread_count;
1152 }
1153 VPRINT(VERBOSE4, "THRD START: Thread count: %d\n", thread_count);
1154 err = pthread_mutex_unlock(&mutex_thread_count);
1155 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1156 }
1157
1158
1159 /************************************************************************
1160 * decrease_thread_counter *
1161 * *
1162 * Do as the name says. Lock; decrease; Unlock; Signal; exit. *
1163 * We do a thread cond signal as well - just to alert for file end *
1164 * processing. *
1165 ************************************************************************/
1166 void
decrease_thread_counter()1167 decrease_thread_counter()
1168 {
1169 /**********************************************************************/
1170 int err; /* Error Checking */
1171
1172 /**********************************************************************/
1173
1174 /* Decrease # of thread's
1175 * Signal that this thread has finished - used for end of file processing */
1176 err = pthread_mutex_lock(&mutex_thread_count);
1177 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1178 /* thread_count is a global */
1179 thread_count--;
1180 VPRINT(VERBOSE4, "THRD EXIT: Thread count: %d\n", thread_count);
1181 pthread_cond_signal(&cond_thread_count);
1182 err = pthread_mutex_unlock(&mutex_thread_count);
1183 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1184 }
1185
1186
1187 /************************************************************************
1188 * add_new_address *
1189 * *
1190 * Do the name lookup and error handling *
1191 * Requires a string[NI_MAXHOST] to dump the results into *
1192 * This is a threaded function *
1193 ************************************************************************/
1194 void *
add_new_address(void * arg)1195 add_new_address(void *arg)
1196 /* ip_address : is of type "struct in_addr" */
1197 {
1198 /**********************************************************************/
1199 int ret_gni; /* getnameinfo Return value */
1200 dns_record_t *dnsrec = arg; /* Current DNS Record to work with */
1201 struct dns_record_lists_t dnsrec_list; /* Single "list" entry for this record */
1202 int err; /* Error Checking */
1203
1204 /**********************************************************************/
1205
1206 VPRINT(VERBOSE5, "ADD_NEW_ADDR: START!:%s", "\n");
1207
1208 /* Initialize the records */
1209 memset(&dnsrec_list, 0, sizeof(dnsrec_list)); /* Not strictly necessary, but just in case */
1210 dnsrec->list = &dnsrec_list;
1211
1212 /* Do the lookup */
1213 ret_gni = name_lookup(dnsrec->ipaddress, dnsrec->list->fqdn);
1214
1215 /* Error/Success checking */
1216 if (ret_gni != 0) {
1217 name_lookup_errors(ret_gni);
1218 } else {
1219 dnsrec->date_last = current_day;
1220 dnsrec->list->date_set = current_day;
1221 dnsrec->list->next = NULL;
1222 }
1223
1224 /* Store this newly updated new record */
1225 store_dns_records(dnsrec);
1226 XFREE(dnsrec);
1227
1228 err = pthread_mutex_lock(&mutex_malloc_dns_rec);
1229 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1230 malloc_dns_rec--;
1231 err = pthread_mutex_unlock(&mutex_malloc_dns_rec);
1232 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1233
1234 decrease_thread_counter();
1235 /* Detach this thread - returning memory etc
1236 * Exit. */
1237 pthread_detach(pthread_self());
1238 pthread_exit(0);
1239 }
1240
1241
1242 /************************************************************************
1243 * add_old_address *
1244 * *
1245 * Do the name lookup for an old record *
1246 * Wants a dns_record_t argument passed in. *
1247 * Assumes this has been malloc'd and will then free it *
1248 * This is a threaded function *
1249 ************************************************************************/
1250 void *
add_old_address(void * arg)1251 add_old_address(void *arg)
1252 /* dns_record is a pointer to struct dns_record_t */
1253 {
1254 /**********************************************************************/
1255 int ret_gni; /* getnameinfo Return value */
1256 dns_record_t *dnsrec = arg; /* Current DNS Record to work with */
1257 struct dns_record_lists_t dnsrec_list; /* Single "list" entry for this record */
1258 struct dns_record_lists_t *list, *list_next; /* list cleanup pointers */
1259 int err; /* Error Checking */
1260
1261 /**********************************************************************/
1262
1263 /* increase_thread_counter (); */
1264
1265 /* Initialize the records */
1266 memset(&dnsrec_list, 0, sizeof(dnsrec_list)); /* Not strictly necessary, but just in case */
1267
1268 /* Do the lookup */
1269 ret_gni = name_lookup(dnsrec->ipaddress, dnsrec_list.fqdn);
1270
1271 /* Error/Success checking */
1272 if (ret_gni != 0) {
1273 name_lookup_errors(ret_gni);
1274 } else {
1275 /* Check the most recent record for this IP Address */
1276 if (strncmp(dnsrec->list->fqdn, dnsrec_list.fqdn, NI_MAXHOST) != 0) {
1277 /* This is a new and updated FQDN, insert */
1278 VPRINT(VERBOSE2, "ADD_OLD_ADDR: Changed FQDN: %s --> %s\n", dnsrec->list->fqdn, dnsrec_list.fqdn);
1279 dnsrec->date_last = current_day;
1280 dnsrec_list.date_set = current_day;
1281 dnsrec_list.next = dnsrec->list;
1282 dnsrec->list = &dnsrec_list;
1283 store_dns_records(dnsrec); /* We only need to store an updated value, we've already stored a new time */
1284 }
1285 }
1286
1287 /* Final tidy up - free the list of FQDN's */
1288 list = dnsrec->list;
1289 if (dnsrec->list == &dnsrec_list) { /* Don't try and free our local variable dnsrec_list */
1290 list = dnsrec->list->next;
1291 }
1292 while (list != NULL) {
1293 VPRINT(VERBOSE5, "ADD_OLD_ADDR: Freeing List Entry: %s\n", list->fqdn);
1294 list_next = list->next;
1295 XFREE(list);
1296 list = list_next;
1297
1298 err = pthread_mutex_lock(&mutex_malloc_dns_list);
1299 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1300 malloc_dns_list--;
1301 err = pthread_mutex_unlock(&mutex_malloc_dns_list);
1302 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1303 }
1304 XFREE(dnsrec);
1305 err = pthread_mutex_lock(&mutex_malloc_dns_rec);
1306 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1307 malloc_dns_rec--;
1308 err = pthread_mutex_unlock(&mutex_malloc_dns_rec);
1309 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1310 decrease_thread_counter();
1311
1312 VPRINT(VERBOSE5, "ADD_OLD_ADDR: EXITING!: %s\n", dnsrec_list.fqdn);
1313 /* Detach this thread - returning memory etc
1314 * Exit. */
1315 pthread_detach(pthread_self());
1316 pthread_exit(0);
1317 }
1318
1319
1320 /************************************************************************
1321 * name_lookup_errors *
1322 * *
1323 * Deal with Name Lookup errors (getnameinfo) *
1324 ************************************************************************/
1325 void
name_lookup_errors(int error_code)1326 name_lookup_errors(int error_code)
1327 {
1328 if (g_verbosity >= VERBOSE1) {
1329 switch (error_code) {
1330 case EAI_AGAIN:
1331 ERRVPRINT(VERBOSE0, "NameLookup: %s\n", msg_namelookup_try_agin);
1332 break;
1333 case EAI_FAIL:
1334 ERRVPRINT(VERBOSE0, "NameLookup: %s\n", "Fatal Error");
1335 break;
1336 default:
1337 ERRVPRINT(VERBOSE0, "NameLookup: %d %s\n", error_code, msg_namelookup_default);
1338 break;
1339 }
1340 }
1341 }
1342
1343
1344 /************************************************************************
1345 * drop_long_lines *
1346 * *
1347 * Strip Log Lines that are too big *
1348 ************************************************************************/
1349 void
drop_long_lines(gzFile * file_input,char * buffer_ptr,buffer_position * buf_posn)1350 drop_long_lines(gzFile * file_input, char *buffer_ptr, buffer_position * buf_posn)
1351 {
1352 /**********************************************************************/
1353 char *fgets_rtn = NULL;
1354
1355 /**********************************************************************/
1356
1357 /* Strip Log Lines that are too big */
1358 buffer_ptr[BUFCUTOFF] = '\0';
1359
1360 ERRVPRINT(VERBOSE1, "#%lu %s :%s...\n", total_lines, msg_W_line_too_big, buffer_ptr);
1361 while (1) {
1362 if (file_input == NULL) {
1363 fgets_rtn = fgets(buffer_ptr, BUFSIZE, stdin);
1364 } else {
1365 fgets_rtn = get_log_line(buffer_ptr, BUFSIZE, file_input, buf_posn);
1366 /* ERRVPRINT (VERBOSE2, " %s\n", buffer_ptr); */
1367 }
1368 if (strpbrk("\n", buffer_ptr) != NULL) {
1369 break;
1370 }
1371 }
1372 }
1373
1374
1375 /************************************************************************
1376 * get_log_line *
1377 * *
1378 * Get the next log line (\n terminated) *
1379 * Is designed to deal with gz'ed logs, but will also do plain text *
1380 * The buffer_position structure is for tracking the raw de-gziped *
1381 * buffer and where we are up to within it. *
1382 ************************************************************************/
1383 char *
get_log_line(char * buf,int size,gzFile * file_ptr,buffer_position * buf_posn)1384 get_log_line(char *buf, int size, gzFile * file_ptr, buffer_position * buf_posn)
1385 {
1386 /* Faster version of gzgets.
1387 * Original code and algorithim taken from mergelog-4.5
1388 * by Bertrand Demiddelaer
1389 *
1390 * Has been quite hacked around to fix some issues in the re-implementation.
1391 * Lack of a terminating '\n' in the original being the biggest problem.
1392 */
1393
1394 /**********************************************************************/
1395 char *out_copy = buf;
1396 int bytes_returned;
1397 int size_ctr = 0;
1398
1399 /**********************************************************************/
1400
1401 VPRINT(VERBOSE5, "\n##GLL: Start: %-10p Cur: %-10p End: %-10p\n", buf_posn->decomp_buf, buf_posn->current_pos_ptr, buf_posn->end_decompbuf_ptr);
1402 while (1) {
1403 if (buf_posn->current_pos_ptr > buf_posn->end_decompbuf_ptr) {
1404 bytes_returned = gzread(file_ptr, buf_posn->decomp_buf, DECOMP_BUFSIZE);
1405 if (bytes_returned <= 0) {
1406 return (NULL);
1407 }
1408 buf_posn->end_decompbuf_ptr = buf_posn->decomp_buf + ((bytes_returned - 1) * sizeof(char));
1409 buf_posn->current_pos_ptr = buf_posn->decomp_buf;
1410 }
1411
1412 if (size_ctr >= size - 1) {
1413 /* This buffer is now full - we've exceeded line size */
1414 buf[size - 1] = '\0';
1415 return (buf);
1416 } else {
1417 /* Copy each char from the ungzipped buffer to the line request buffer */
1418 *out_copy = *buf_posn->current_pos_ptr;
1419 if (*buf_posn->current_pos_ptr == '\n') {
1420 out_copy++;
1421 *out_copy = '\0';
1422 buf_posn->current_pos_ptr++; /* We will be leaving shortly, so increment to next character */
1423 return (buf);
1424 }
1425 }
1426 size_ctr++;
1427 out_copy++;
1428 buf_posn->current_pos_ptr++;
1429 }
1430 }
1431
1432
1433 /************************************************************************
1434 * close_exit *
1435 * *
1436 * Close the open Database and Exit *
1437 * *
1438 * Arguments: *
1439 * FILE * file_input The input File. If NULL, use STDIN. *
1440 * int exit_code The exit code to exit with. *
1441 * Should be defined in error.h *
1442 ************************************************************************/
1443 void
close_exit(int exit_code)1444 close_exit(int exit_code)
1445 {
1446 close_dnshistory_db(&dnshistory_db_ptr);
1447 exit(exit_code);
1448 }
1449
1450
1451 /************************************************************************
1452 * address_exists *
1453 * *
1454 * Have we seen this address? *
1455 * If yes, fill in remaining data fields and return 1 *
1456 * If no, return 0 *
1457 ************************************************************************/
1458 int
address_exists(dns_record_t * dns_rec_ptr)1459 address_exists(dns_record_t * dns_rec_ptr)
1460 {
1461 /**********************************************************************/
1462 int rtn = 0; /* Return value */
1463 DBT dbt_key, dbt_data; /* Key and Data Pointers for results from BDB */
1464 int db_rtn; /* return value from DBD Get */
1465
1466 struct dns_record_lists_t *new_dns_rec = NULL;
1467 struct dns_record_lists_t *tail_dns_rec = NULL;
1468 void *idx_ptr = NULL; /* Counter. Where are we up to in the new memory block */
1469
1470 int size = 0; /* Counter. How big is each data item for the memory block */
1471 int nbr_items = 0; /* Counter. How many data loops */
1472 int err; /* Error Checking */
1473
1474 char str_address[INET_ADDRSTRLEN]; /* Temp holder for displaying IP Address to lookup */
1475
1476 /**********************************************************************/
1477
1478 VPRINT(VERBOSE5, " Address Exists?: %16s\n", inet_ntop(AF_INET, &dns_rec_ptr->ipaddress, str_address, INET_ADDRSTRLEN));
1479
1480 memset(&dbt_key, 0, sizeof(dbt_key));
1481 memset(&dbt_data, 0, sizeof(dbt_data));
1482
1483 dbt_key.data = (char *) &dns_rec_ptr->ipaddress.s_addr;
1484 dbt_key.size = sizeof(dns_rec_ptr->ipaddress.s_addr);
1485
1486 err = pthread_mutex_lock(&mutex_db_access); /* Lock all DB accesses */
1487 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1488 db_rtn = dnshistory_db_ptr->get(dnshistory_db_ptr, NULL, &dbt_key, &dbt_data, 0);
1489 if (db_rtn == 0) {
1490 /* Yes! We have seen this address before. */
1491 idx_ptr = dbt_data.data;
1492 size = sizeof(dns_rec_ptr->date_last);
1493 memcpy(&dns_rec_ptr->date_last, idx_ptr, size);
1494 idx_ptr += size;
1495
1496 size = sizeof(nbr_items);
1497 memcpy(&nbr_items, idx_ptr, size);
1498 idx_ptr += size;
1499
1500 while (nbr_items > 0) {
1501 /* Build up the structure of old addresses */
1502 new_dns_rec = XMALLOC(struct dns_record_lists_t, 1);
1503
1504 err = pthread_mutex_lock(&mutex_malloc_dns_list);
1505 ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
1506 malloc_dns_list++;
1507 err = pthread_mutex_unlock(&mutex_malloc_dns_list);
1508 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1509
1510 /* Add this new rec to the end of the list */
1511 if (dns_rec_ptr->list == NULL) {
1512 dns_rec_ptr->list = new_dns_rec;
1513 } else {
1514 tail_dns_rec->next = new_dns_rec;
1515 }
1516 tail_dns_rec = new_dns_rec;
1517 new_dns_rec->next = NULL;
1518
1519 /* Store date_set for this fqdn */
1520 size = sizeof(new_dns_rec->date_set);
1521 memcpy(&new_dns_rec->date_set, idx_ptr, size);
1522 idx_ptr += size;
1523
1524 /* Store the fqdn itself */
1525 size = (strlen((char *) idx_ptr) + 1) * sizeof(char);
1526 memcpy(new_dns_rec->fqdn, idx_ptr, size);
1527 check_n_fix_fqdn(new_dns_rec->fqdn);
1528 idx_ptr += size;
1529
1530 VPRINT(VERBOSE5, " Address Exists? Yes: %s\n", new_dns_rec->fqdn);
1531 nbr_items--;
1532 }
1533 rtn = 1;
1534 err = pthread_mutex_unlock(&mutex_db_access);
1535 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1536 } else {
1537 err = pthread_mutex_unlock(&mutex_db_access);
1538 ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
1539 VPRINT(VERBOSE5, " Address Exists?: No.%s", "\n");
1540 rtn = 0;
1541 }
1542 return (rtn);
1543 }
1544
1545
1546 /************************************************************************
1547 * store_dns_records *
1548 * *
1549 * Store the dns records for this lookup into the hash DB. *
1550 * Compact any strings down to minimum levels. *
1551 ************************************************************************/
1552 void
store_dns_records(dns_record_t * dns_rec_ptr)1553 store_dns_records(dns_record_t * dns_rec_ptr)
1554 {
1555 /* dns_rec_ptr: is the dns structure, and links, to store */
1556 /**********************************************************************/
1557 void *data_ptr = NULL; /* pointer to the data store to store in the hash */
1558 void *idx_ptr = NULL; /* Counter. Where are we up to in the new memory block */
1559 struct dns_record_lists_t *list_ptr; /* pointer to index thru the dns history */
1560
1561 int data_size = 0; /* size of fixed data in the dns record */
1562 int key_size = 0; /* size of the cookie itself - key for the hash */
1563 int nbr_items = 0; /* Counter. How many data loops */
1564 int size = 0; /* Counter. How big is each data item for the memory block */
1565
1566 char str_address[INET_ADDRSTRLEN]; /* VPRINT buffer only! */
1567
1568 /**********************************************************************/
1569
1570 VPRINT(VERBOSE4, " **STORE: %16s\n", inet_ntop(AF_INET, &dns_rec_ptr->ipaddress, str_address, INET_ADDRSTRLEN));
1571
1572 key_size = sizeof(dns_rec_ptr->ipaddress.s_addr);
1573 data_size = sizeof(dns_rec_ptr->date_last);
1574 list_ptr = dns_rec_ptr->list;
1575 while (list_ptr != NULL) {
1576 data_size += sizeof(list_ptr->date_set);
1577 data_size += (strlen(list_ptr->fqdn) + 1) * sizeof(char);
1578 list_ptr = list_ptr->next;
1579 nbr_items++;
1580 }
1581 data_size += sizeof(nbr_items);
1582
1583 /* Grab the memory needed for all the dns rec's data */
1584 data_ptr = (void *) malloc(data_size);
1585 ERR_NULL_EXIT(data_ptr, V_EXIT_MEMORY_EXHAUSTION, msg_F_memory_alloc, "");
1586 memset(data_ptr, 0, data_size); /* Not strictly necessary, but just in case */
1587
1588 /* Store the most recent date for this dns record */
1589 idx_ptr = data_ptr;
1590 size = sizeof(dns_rec_ptr->date_last);
1591 memcpy(idx_ptr, &dns_rec_ptr->date_last, size);
1592 idx_ptr += size;
1593 /* Store the number of items in the dns history, for later extraction! */
1594 size = sizeof(nbr_items);
1595 memcpy(idx_ptr, &nbr_items, size);
1596 idx_ptr += size;
1597
1598 list_ptr = dns_rec_ptr->list;
1599 while (list_ptr != NULL) {
1600 /* Date this record was first seen */
1601 size = sizeof(list_ptr->date_set);
1602 memcpy(idx_ptr, &list_ptr->date_set, size);
1603 idx_ptr += size;
1604
1605 /* FQDN storage */
1606 VPRINT(VERBOSE5, " STO: fqdn: %s\n", list_ptr->fqdn);
1607 size = (strlen(list_ptr->fqdn) + 1) * sizeof(char);
1608 memcpy(idx_ptr, &list_ptr->fqdn, size);
1609 idx_ptr += size;
1610
1611 list_ptr = list_ptr->next;
1612 }
1613
1614 add_record(&dnshistory_db_ptr, &dns_rec_ptr->ipaddress.s_addr, key_size, data_ptr, data_size);
1615 XFREE(data_ptr); /* release the memory grabbed */
1616 }
1617
1618
1619 /************************************************************************
1620 * display_record *
1621 * *
1622 * Given a BDB Data Key and Data Value pair, display the stored *
1623 * address information/history *
1624 ************************************************************************/
1625 void
display_record(DBT * dbt_key,DBT * dbt_data,bool realdate)1626 display_record(DBT * dbt_key, DBT * dbt_data, bool realdate)
1627 {
1628 dns_record_t dnsrec; /* Current DNS Record to work with */
1629 struct dns_record_lists_t dnslist_rec; /* Current DNS Data to work with */
1630
1631 void *idx_ptr = NULL; /* Counter. Where are we up to in the new memory block */
1632 int size = 0; /* Counter. How big is each data item for the memory block */
1633 int nbr_items = 0; /* Counter. How many data loops */
1634 char str_address[INET_ADDRSTRLEN]; /* Print buffer for converting stored IPAddresses to Normal w.x.y.z */
1635
1636 struct tm *date_last_full;
1637
1638 VPRINT(VERBOSE5, " Display Record!%s", "\n");
1639
1640 memset(&dnsrec, 0, sizeof(dnsrec));
1641 idx_ptr = dbt_data->data;
1642 size = sizeof(dnsrec.date_last);
1643 memcpy(&dnsrec.date_last, idx_ptr, size);
1644 idx_ptr += size;
1645
1646 size = sizeof(nbr_items);
1647 memcpy(&nbr_items, idx_ptr, size);
1648 idx_ptr += size;
1649
1650 /* Copy in the raw IP Address */
1651 memcpy(&dnsrec.ipaddress.s_addr, dbt_key->data, sizeof(dnsrec.ipaddress.s_addr));
1652 inet_ntop(AF_INET, &dnsrec.ipaddress, str_address, INET_ADDRSTRLEN);
1653 printf("%-s", str_address);
1654
1655 /* Loop thru the various list items */
1656 while (nbr_items > 0) {
1657 memset(&dnslist_rec, 0, sizeof(dnsrec));
1658 /* Store date_set for this fqdn */
1659 size = sizeof(dnslist_rec.date_set);
1660 memcpy(&dnslist_rec.date_set, idx_ptr, size);
1661 idx_ptr += size;
1662
1663 /* Store the fqdn itself */
1664 size = (strlen((char *) idx_ptr) + 1) * sizeof(char);
1665 memcpy(dnslist_rec.fqdn, idx_ptr, size);
1666 idx_ptr += size;
1667
1668 printf("\t");
1669
1670 if (realdate == true) {
1671 date_last_full = localtime(&dnslist_rec.date_set);
1672 printf("%4d-%02d-%02d:%02d:%02d:%02d,", date_last_full->tm_year + 1900, date_last_full->tm_mon + 1, date_last_full->tm_mday, date_last_full->tm_hour,
1673 date_last_full->tm_min, date_last_full->tm_sec);
1674 } else {
1675 printf("%d,", (int) dnslist_rec.date_set);
1676 }
1677
1678 check_n_fix_fqdn(dnslist_rec.fqdn);
1679 if (strlen(dnslist_rec.fqdn) > 0) {
1680 printf("%s", dnslist_rec.fqdn);
1681 } else {
1682 printf("%s", STR_NONAME);
1683 // printf ("%s", "nullname");
1684 }
1685
1686 memset(&str_address, 0, sizeof(str_address));
1687
1688 nbr_items--;
1689 }
1690 printf("\n");
1691
1692 }
1693
1694
1695 /************************************************************************
1696 * dump_dns_historydb *
1697 * *
1698 * Given a Database pointer, dump the DNS History values to stdout *
1699 ************************************************************************/
1700 void
dump_dns_historydb(DB ** db_ptr)1701 dump_dns_historydb(DB ** db_ptr)
1702 {
1703 /**********************************************************************/
1704 DBC *dbcurs_ptr = NULL; /* DB Pointer for cursor'ing */
1705 DBT dbt_key, dbt_data; /* Key and Data Pointers for results from BDB */
1706 int db_rtn; /* return value from DBD Get */
1707
1708 /**********************************************************************/
1709
1710 VPRINT(VERBOSE5, " History Dump!%s", "\n");
1711 /* Acquire a cursor for the database. */
1712 db_rtn = (*db_ptr)->cursor(*db_ptr, NULL, &dbcurs_ptr, 0);
1713 if (db_rtn != 0) {
1714 (*db_ptr)->err(*db_ptr, db_rtn, "DB->cursor");
1715 ERRVPRINT(VERBOSE0, msg_F_db_cursor, "");
1716 close_exit(V_EXIT_DB_CURSOR);
1717 }
1718 VPRINT(VERBOSE5, "dbcurs_ptr == %p", dbcurs_ptr);
1719
1720 memset(&dbt_key, 0, sizeof(dbt_key));
1721 memset(&dbt_data, 0, sizeof(dbt_data));
1722
1723 while ((db_rtn = dbcurs_ptr->c_get(dbcurs_ptr, &dbt_key, &dbt_data, DB_NEXT)) == 0) {
1724 display_record(&dbt_key, &dbt_data, false);
1725 }
1726 db_rtn = dbcurs_ptr->c_close(dbcurs_ptr);
1727 }
1728
1729
1730 /************************************************************************
1731 * import_dns_historydb *
1732 * *
1733 * Given a Database pointer, import the DNS History values to this BDB *
1734 ************************************************************************/
1735 void
import_dns_historydb(void)1736 import_dns_historydb(void)
1737 {
1738 FILE *import_file;
1739 char *fgets_rtn = NULL;
1740 char buffer[BUFSIZE];
1741
1742 char str_address[INET_ADDRSTRLEN]; /* Print buffer for converting stored IPAddresses to Normal w.x.y.z */
1743
1744 dns_record_t dnsrec; /* Current DNS Record to work with */
1745
1746 struct dns_record_lists_t *last_list_ptr; /* pointer to index thru the dns history */
1747 struct dns_record_lists_t *new_dns_rec = NULL; /* New list pointer to malloc */
1748 struct dns_record_lists_t *list, *list_next; /* list cleanup pointers */
1749
1750 int rtn_sscanf = 0; /* sscanf return value, for later checking */
1751 bool flag_is_bad_line;
1752
1753 /**********************************************************************/
1754 VPRINT(VERBOSE1, "DNS History Import!%s", "\n");
1755
1756 import_file = fopen(g_import_filename, "r");
1757 if (import_file == NULL) {
1758 ERRVPRINT(VERBOSE0, msg_F_file_open, g_import_filename);
1759 exit(1);
1760 }
1761
1762 while (1) {
1763 fgets_rtn = fgets(buffer, BUFSIZE, import_file);
1764 if (fgets_rtn == NULL) {
1765 /* Exit Main Loop at end of input */
1766 break;
1767 }
1768 total_lines++;
1769 VPRINT(VERBOSE5, "Line: %lu Importing: %s", total_lines, buffer);
1770 flag_is_bad_line = false;
1771
1772 /* check for log line too long */
1773 if (strpbrk(buffer, "\n") == NULL) {
1774 ERRVPRINT(VERBOSE0, msg_W_import_line_too_long, total_lines);
1775 while (1) {
1776 /* Loop over import till we reach the end of this line, then break and get the next line */
1777 fgets_rtn = fgets(buffer, BUFSIZE, import_file);
1778 if (strpbrk("\n", buffer) != NULL) {
1779 break;
1780 }
1781 }
1782 bad_lines++;
1783 continue;
1784 }
1785
1786 char *buf_ptr1 = NULL;
1787 char *buf_ptr2 = NULL;
1788
1789 memset(&dnsrec, 0, sizeof(dnsrec));
1790
1791 rtn_sscanf = sscanf(buffer, "%15s ", (char *) &str_address);
1792 if (rtn_sscanf < 1) {
1793 ERRVPRINT(VERBOSE0, msg_W_import_line_failure, total_lines);
1794 bad_lines++;
1795 continue;
1796 }
1797
1798 inet_pton(AF_INET, str_address, &dnsrec.ipaddress);
1799 dnsrec.list = NULL;
1800 last_list_ptr = NULL;
1801
1802 buf_ptr1 = strpbrk(buffer, "\t"); /* First tab */
1803 if (buf_ptr1 == NULL) {
1804 ERRVPRINT(VERBOSE0, msg_W_import_line_failure, total_lines);
1805 bad_lines++;
1806 continue;
1807 }
1808 buf_ptr2 = buf_ptr1;
1809
1810 /* Loop over each tab separator */
1811 while (buf_ptr2 != NULL) {
1812 new_dns_rec = XMALLOC(struct dns_record_lists_t, 1);
1813
1814 if (dnsrec.list == NULL) {
1815 dnsrec.list = new_dns_rec;
1816 }
1817
1818 rtn_sscanf = sscanf(buf_ptr2, " %lu,%1024s ", (unsigned long *) &new_dns_rec->date_set, (char *) &new_dns_rec->fqdn);
1819 VPRINT(VERBOSE2, " Scan: %lu --> %s\n", (unsigned long) new_dns_rec->date_set, new_dns_rec->fqdn);
1820 if (rtn_sscanf < 2) {
1821 ERRVPRINT(VERBOSE0, msg_W_import_line_failure, total_lines);
1822 bad_lines++;
1823 flag_is_bad_line = true;
1824 break;
1825 }
1826
1827 if (dnsrec.date_last == 0) {
1828 dnsrec.date_last = new_dns_rec->date_set;
1829 }
1830 if (last_list_ptr != NULL) {
1831 last_list_ptr->next = new_dns_rec;
1832 }
1833 last_list_ptr = new_dns_rec;
1834 new_dns_rec->next = NULL;
1835
1836 /* See if we've grabbed a 'NONAME' value, replace with a null if so
1837 * Don't want 'NONAME' as the FQDN... */
1838 if (strncmp(new_dns_rec->fqdn, STR_NONAME, strlen(new_dns_rec->fqdn)) == 0) {
1839 new_dns_rec->fqdn[0] = '\0';
1840 } else {
1841 check_n_fix_fqdn(new_dns_rec->fqdn);
1842 }
1843
1844 /* Find next tabstop */
1845 buf_ptr1 = buf_ptr2;
1846 buf_ptr2 = strpbrk(buf_ptr1 + 1, "\t");
1847 }
1848
1849 /* Don't store if we've got a funky internal/additional sscanf */
1850 if (flag_is_bad_line != true) {
1851 store_dns_records(&dnsrec);
1852 number_successful_fqdns++;
1853 }
1854
1855 /* Final tidy up - free the list of FQDN's */
1856 if (dnsrec.list != NULL) {
1857 list = dnsrec.list;
1858 } else {
1859 /* Should *never* be here - implies no valid line data - bad! */
1860 list = NULL;
1861 }
1862 while (list != NULL) {
1863 ERRVPRINT(VERBOSE3, " Freeing List Entry: %s\n", list->fqdn);
1864 list_next = list->next;
1865 XFREE(list);
1866 list = list_next;
1867
1868 }
1869
1870 } /* while (1) */
1871
1872 VPRINT(VERBOSE0, "Successfully Imported %lu Records\n", number_successful_fqdns);
1873 if (bad_lines > 0) {
1874 VPRINT(VERBOSE0, "%s%lu of %lu\n", msg_I_number_bad_lines, bad_lines, total_lines);
1875 }
1876
1877 }
1878
1879
1880 /************************************************************************
1881 * showhistory *
1882 * *
1883 * Given a Database pointer, dump the DNS History values to stdout *
1884 ************************************************************************/
1885 void
showhistory(DB ** db_ptr,int argc,char * argv[])1886 showhistory(DB ** db_ptr, int argc, char *argv[])
1887 {
1888 /**********************************************************************/
1889 char *str_ipaddr; /* The current IP address from cmd line */
1890 int ret_pton; /* Results: inet_pton */
1891
1892 DBT dbt_key, dbt_data; /* Key and Data Pointers for results from BDB */
1893 int db_rtn; /* return value from DBD Get */
1894
1895 dns_record_t dnsrec; /* Current DNS Record to work with */
1896 int arg_ctr = optind;
1897
1898 /**********************************************************************/
1899
1900 VPRINT(VERBOSE5, " History Lookup!%s", "\n");
1901 VPRINT(VERBOSE5, " Doing %d\n", argc);
1902
1903 while (arg_ctr < argc) {
1904 str_ipaddr = argv[arg_ctr];
1905 arg_ctr++;
1906 VPRINT(VERBOSE5, " IP: %s\n", str_ipaddr);
1907 /* Convert from char string to in_addr_t */
1908 ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec.ipaddress);
1909 if (ret_pton == 0) {
1910 ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
1911 continue;
1912 }
1913 memset(&dbt_key, 0, sizeof(dbt_key));
1914 memset(&dbt_data, 0, sizeof(dbt_data));
1915
1916 dbt_key.data = (char *) &dnsrec.ipaddress.s_addr;
1917 dbt_key.size = sizeof(dnsrec.ipaddress.s_addr);
1918
1919 db_rtn = (*db_ptr)->get(*db_ptr, NULL, &dbt_key, &dbt_data, 0);
1920 if (db_rtn == 0) {
1921 display_record(&dbt_key, &dbt_data, true);
1922 } else {
1923 printf("%-s\t%s\n", str_ipaddr, "ADDRESS NOT FOUND");
1924 }
1925 } /* while */
1926 }
1927
1928
1929 /************************************************************************
1930 * check_n_fix_fqdn *
1931 * *
1932 * Given a FQDN string, replace all non printable ASCII chars with *
1933 * underscores. *
1934 ************************************************************************/
1935 void
check_n_fix_fqdn(char * fqdn)1936 check_n_fix_fqdn(char *fqdn)
1937 {
1938 unsigned int length, i;
1939
1940 VPRINT(VERBOSE4, " Check'N Fix: %s --> ", fqdn);
1941 length = strlen(fqdn);
1942 for (i = 0; i < length; i++) {
1943 if ((fqdn[i] < 33) || (fqdn[i] > 126)) {
1944 fqdn[i] = '_';
1945 }
1946 }
1947 VPRINT(VERBOSE4, "%s\n", fqdn);
1948 }
1949
1950
1951 /************************************************************************
1952 * identify_log_format *
1953 * *
1954 * Attempt to identify the type of log format we've been given. *
1955 * Returns the LOG_type as defined in dnshistory.h *
1956 * returns -1 if unknown. *
1957 * *
1958 * Requires a line of the log to attempt to process *
1959 ************************************************************************/
1960 int
identify_log_format(char * buffer)1961 identify_log_format(char *buffer)
1962 {
1963 int ovector[OVECCOUNT]; /* RE substring offsets array */
1964 int rc; /* RE Check return value */
1965 int buffer_length;
1966
1967
1968 buffer_length = (int) strlen(buffer);
1969
1970 rc = pcre_exec(cmp_log_regexp_clf, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
1971 if (rc >= 0) {
1972 /* Matches against CLF */
1973 VPRINT(VERBOSE1, "Using CLF Log Format%s", "\n");
1974 return (LOG_CLF);
1975 }
1976
1977 rc = pcre_exec(cmp_log_regexp_xferlog, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
1978 if (rc >= 0) {
1979 /* Matches against FTP/XFERLOG */
1980 VPRINT(VERBOSE1, "Using FTP/XFERLOG Log Format%s", "\n");
1981 return (LOG_FTP);
1982 }
1983
1984 rc = pcre_exec(cmp_log_regexp_squid, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
1985 if (rc >= 0) {
1986 /* Matches against SQUID */
1987 VPRINT(VERBOSE1, "Using SQUID Log Format%s", "\n");
1988 return (LOG_SQUID);
1989 }
1990
1991 rc = pcre_exec(cmp_log_regexp_iptables, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
1992 if (rc >= 0) {
1993 /* Matches against IPTABLES */
1994 VPRINT(VERBOSE1, "Using IPTABLES Log Format%s", "\n");
1995 return (LOG_IPTABLES);
1996 }
1997
1998 rc = pcre_exec(cmp_log_regexp_syslog, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
1999 if (rc >= 0) {
2000 /* Matches against SYSLOG */
2001 VPRINT(VERBOSE1, "Using SYSLOG/IPTABLES Log Format%s", "\n");
2002 return (LOG_IPTABLES);
2003 }
2004
2005 VPRINT(VERBOSE1, "Unrecognised Log Format%s", "\n");
2006 return (-1); /* Failed to match any, unknown format */
2007 }
2008
2009
2010 /************************************************************************
2011 * re_compile_all_regexs *
2012 * *
2013 * Does what the name says, in a single function we compile all *
2014 * possibly used Regular expressions. *
2015 * Either forcibly exits on any failure, or happily finishes. *
2016 * No values needed or returned. *
2017 * *
2018 * Assigns the RE's to the various globals: *
2019 * cmp_log_regexp_* *
2020 ************************************************************************/
2021 void
re_compile_all_regexes(void)2022 re_compile_all_regexes(void)
2023 {
2024 char log_regexp_clf[MAX_RE_LENGTH] = PATTERN_CLF;
2025 char log_regexp_xferlog[MAX_RE_LENGTH] = PATTERN_XFERLOG;
2026 char log_regexp_squid[MAX_RE_LENGTH] = PATTERN_SQUID;
2027 char log_regexp_combined_enhanced[MAX_RE_LENGTH] = PATTERN_COMBINED_ENHANCED;
2028 char log_regexp_iptables[MAX_RE_LENGTH] = PATTERN_IPTABLES;
2029 char log_regexp_syslog[MAX_RE_LENGTH] = PATTERN_SYSLOG;
2030
2031 const char *error; /* RE error pointer, offset */
2032 int erroffset; /* RE error value */
2033
2034 /* CLF */
2035 cmp_log_regexp_clf = pcre_compile(log_regexp_clf, 0, &error, &erroffset, NULL);
2036 VPRINT(VERBOSE2, "PCRE: Compile CLF%s", "\n")
2037 if (cmp_log_regexp_clf == NULL) {
2038 re_compile_failed(erroffset, error, log_regexp_clf);
2039 }
2040
2041 /* Enhanced Combined */
2042 cmp_log_regexp_combined_enhanced = pcre_compile(log_regexp_combined_enhanced, 0, &error, &erroffset, NULL);
2043 VPRINT(VERBOSE2, "PCRE: Compile COMBINED_ENHANCED%s", "\n")
2044 if (cmp_log_regexp_combined_enhanced == NULL) {
2045 re_compile_failed(erroffset, error, log_regexp_combined_enhanced);
2046 }
2047
2048 /* FTP XFERLOG */
2049 cmp_log_regexp_xferlog = pcre_compile(log_regexp_xferlog, 0, &error, &erroffset, NULL);
2050 VPRINT(VERBOSE2, "PCRE: Compile PATTERN_XFERLOG%s", "\n")
2051 if (cmp_log_regexp_xferlog == NULL) {
2052 re_compile_failed(erroffset, error, log_regexp_xferlog);
2053 }
2054
2055 /* SQUID LOG */
2056 cmp_log_regexp_squid = pcre_compile(log_regexp_squid, 0, &error, &erroffset, NULL);
2057 VPRINT(VERBOSE2, "PCRE: Compile PATTERN_SQUID%s", "\n")
2058 if (cmp_log_regexp_squid == NULL) {
2059 re_compile_failed(erroffset, error, log_regexp_squid);
2060 }
2061
2062 /* SYSLOG/IPTABLES LOG */
2063 cmp_log_regexp_iptables = pcre_compile(log_regexp_iptables, 0, &error, &erroffset, NULL);
2064 VPRINT(VERBOSE2, "PCRE: Compile PATTERN_IPTABLES%s", "\n")
2065 if (cmp_log_regexp_iptables == NULL) {
2066 re_compile_failed(erroffset, error, log_regexp_iptables);
2067 }
2068
2069 /* SYSLOG */
2070 cmp_log_regexp_syslog = pcre_compile(log_regexp_syslog, 0, &error, &erroffset, NULL);
2071 VPRINT(VERBOSE2, "PCRE: Compile PATTERN_SYSLOG%s", "\n")
2072 if (cmp_log_regexp_syslog == NULL) {
2073 re_compile_failed(erroffset, error, log_regexp_syslog);
2074 }
2075 }
2076
2077
2078 /************************************************************************
2079 ************************************************************************
2080 * END OF FILE *
2081 ************************************************************************/
2082