1 /*************************************************************************** 2 * Copyright (C) 2004, 2005, 2006 by Stephen McInerney * 3 * spm@stedee.id.au * 4 * * 5 * $Id: dnshistory.h 66 2006-06-24 23:50:26Z steve $ 6 * * 7 * This program is free software; you can redistribute it and/or modify * 8 * it under the terms of the GNU General Public License as published by * 9 * the Free Software Foundation; either version 2 of the License, or * 10 * (at your option) any later version. * 11 * * 12 * This program is distributed in the hope that it will be useful, * 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of * 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 15 * GNU General Public License for more details. * 16 * * 17 * You should have received a copy of the GNU General Public License * 18 * along with this program; if not, write to the * 19 * Free Software Foundation, Inc., * 20 * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * 21 * * 22 ***************************************************************************/ 23 24 /*************************************************************************** 25 *************************************************************************** 26 * ModificationHistory: 27 ********************** 28 * 16-Mar-2005 steve Initial Creation 29 * 30 *************************************************************************** 31 ***************************************************************************/ 32 33 #ifndef DNSHISTORY_DNSHISTORY_H 34 #define DNSHISTORY_DNSHISTORY_H 1 35 36 #include "common.h" 37 #include "error.h" 38 #include "messages.h" 39 #include "db_dnshistory.h" 40 #include "xmalloc.h" 41 #include "regexp.h" 42 43 /************************************* 44 * Simplified Error Handling MACRO's 45 *************************************/ 46 /* This Macro will display errors if a given err value is non zero. 47 * Assumes that the message has a %d holder for the error value itself */ 48 #define ERR_NONZERO(err, verbose, message, ...) if ((err) != 0) { ERRVPRINT ((verbose), (message), __VA_ARGS__); } 49 /* Similar to above, but this is a fatal condition. Display the error and exit, closing the open DB */ 50 #define ERR_NONZERO_EXIT(err, exitcode, message, ...) if ((err) != 0) { ERRVPRINT ((VERBOSE0), (message), __VA_ARGS__); close_exit ((exitcode)); } 51 /* Similar to above, but instead check for a NULL value */ 52 #define ERR_NULL_EXIT(err, exitcode, message, ...) if ((err) == NULL) { ERRVPRINT ((VERBOSE0), (message), __VA_ARGS__); close_exit ((exitcode)); } 53 /*************************************/ 54 55 56 #define OVECCOUNT 300 57 #define BUFSIZE (128 * 2 * 512) /* size in characters of the buffer to read lines into */ 58 #define BUFCUTOFF 125 /* When printing out long buffers, cut off here */ 59 #define DECOMP_BUFSIZE (BUFSIZE) /* size in characters of the ZLIB buffer to read lines into */ 60 #define MAX_RE_LENGTH 512 /* The max size in chars of an RE */ 61 #define MAX_FILENAME_LENGTH 512 /* The max size in chars of any specified filenames */ 62 63 #define PATTERN_CLF "^([^ ]+) ([^ ]+) ([^ ]+) \\[([^ ]+)" 64 /* We only care up to the date/time space */ 65 #define PATTERN_COMBINED_ENHANCED "^([^ ]+) ([^ ]+) ([^ ]+) \\[([^ ]+) ([^ ]+)\\] \"([^ \"]+) ?([^ ]+)? ?([^\"]*)?\" ([^ ]+) ([^ ]+) \"(.*?)\" \"([^\"]*)\"" 66 /* Combined Log Format, handle escaped double-quotes (APACHE) */ 67 #define PATTERN_XFERLOG "^(... ... .. ..:..:.. ....) ([[:digit:]]+) ([[:digit:].]+)" 68 /* FTP/XFERLOG Log Format */ 69 #define PATTERN_SQUID "^([[:digit:]]+)\\.([[:digit:]]+)[ ]+([[:digit:]]+) ([[:digit:].]+)" 70 /* SQUID Log Format. */ 71 #define PATTERN_IPTABLES "^(... .. ..:..:..) ([^ ]+) kernel:.* SRC=([[:digit:].]+) DST=([[:digit:].]+)" 72 #define PATTERN_SYSLOG "^(... .. ..:..:..) ([^:]+):" 73 74 #define DATE_TIME_FORMAT "%d/%b/%Y:%H:%M:%S" /* Default DATE_TIME format for input to strptime */ 75 #define DATE_TIME_XFERLOG_FORMAT "%a %b %d %H:%M:%S %Y" 76 #define DATE_TIME_IPTABLES_FORMAT "%a %d %H:%M:%S" 77 78 #define SIZE_ADDRESS (NI_MAXHOST) /* Defined in netdb.h */ 79 #define SIZE_DATE_TIME 50 80 #define SIZE_URL 1024 81 #define SIZE_REFERER 1024 82 #define SIZE_BROWSER 1024 83 #define SIZE_COOKIE 1024 84 #define SIZE_CACHE (20 * 1024 * 1024) 85 #define THREAD_STACK_SIZE (1<<15) /* 32Kb. 14 (16K) is core'ing */ 86 #define THREAD_MAXIMUM_THREADS 100 /* 100 threads seems a good balance. */ 87 88 #define SAME_RUN 3600 /* How many seconds before we should check again that a record may have changed */ 89 #define DNS_RESULTS_RECHECK (7 * 24 * 60 * 60) /* Only re-check DNS results every 7 days */ 90 #define DNS_LOOKUPS 2 /* How many retries on name lookup failures */ 91 #define DNS_RETRY_DELAY 1 /* Delay time in seconds between retries in the same run */ 92 93 #define LF_NCSA_ADDRESS 1 94 #define LF_NCSA_USER 2 95 #define LF_NCSA_AUTHUSER 3 96 #define LF_NCSA_DATE_TIME 4 97 #define LF_NCSA_TIMEZONE 5 98 #define LF_NCSA_ACC_TYPE 6 99 #define LF_NCSA_URL 7 100 #define LF_NCSA_HTTP_VER 8 101 #define LF_NCSA_STATUS 9 102 #define LF_NCSA_BYTES 10 103 #define LF_NCSA_REFERER 11 104 #define LF_NCSA_BROWSER 12 105 #define LF_CUSTM_COOKIE 13 106 107 #define LF_SQUID_ADDRESS 4 108 #define LF_SQUID_DATE_TIME 1 109 110 #define LF_XFERLOG_ADDRESS 3 111 #define LF_XFERLOG_DATE_TIME 1 112 113 #define LF_IPTABLES_ADDRESS_SRC 3 114 #define LF_IPTABLES_ADDRESS_DST 4 115 #define LF_IPTABLES_DATE_TIME 1 116 117 #define DB_CACHE_SIZE 20 118 119 #define STR_NONAME "NONAME" 120 121 /* Log types */ 122 #define LOG_UNRECOGNISED -1 /* Unrecognised log format */ 123 #define LOG_AUTO 0 /* Try and discover the log format */ 124 #define LOG_CLF 1 /* CLF log type */ 125 #define LOG_FTP 2 /* wu-ftpd xferlog type */ 126 #define LOG_SQUID 3 /* squid proxy log */ 127 #define LOG_COMBINED 4 /* Apache Combined log type */ 128 #define LOG_IPTABLES 5 /* syslog/linux-iptables */ 129 130 /************************************************************************ 131 * STRUCTURES * 132 ************************************************************************/ 133 struct dns_record_lists_t { /* linked list */ 134 time_t date_set; /* When was 1st seen */ 135 char fqdn[NI_MAXHOST]; /* the complete name */ 136 struct dns_record_lists_t *next; 137 }; 138 139 typedef struct { 140 struct in_addr ipaddress; /* The IP Address */ 141 time_t date_last; /* When was last seen */ 142 struct dns_record_lists_t *list; /* List of IP Addresses etc */ 143 } dns_record_t; 144 145 struct buffer_position_struct { 146 char decomp_buf[DECOMP_BUFSIZE]; 147 char *current_pos_ptr; 148 char *end_decompbuf_ptr; 149 }; 150 typedef struct buffer_position_struct buffer_position; 151 152 typedef struct { 153 char regular_expression[MAX_RE_LENGTH]; /* Original Regular Expression */ 154 pcre *re_pcre; /* Compiled PCRE */ 155 int erroffset; /* Error Offset */ 156 const char *error; /* RE error pointer, offset */ 157 int ovector[OVECCOUNT]; /* Offset vectors */ 158 int ret; /* Return value from various calls */ 159 int cp_substr_ret; /* Return value from copy substring call */ 160 } pcre_struct; 161 162 /************************************************************************ 163 * GLOBALS * 164 ************************************************************************/ 165 /*-- Date/Time --*/ 166 extern time_t current_day; /* The current day, check to see if we need to update a record in this run 167 Also see SAME_RUN */ 168 169 /*-- DB Setup --*/ 170 extern u_int32_t db_cache_size; /* Size of the DB Cache */ 171 extern char g_db_dirfilename[MAX_FILENAME_LENGTH]; /* The name of the Database Directory Path/File to use for storage */ 172 173 extern int g_dns_timeout; /* The period that DNS queries won't be retried within. Seconds */ 174 extern char *g_filename; /* Command Line specified file to open. */ 175 extern char *g_recombine_filename; /* Command Line specified file to open. 176 This file is the source of existing FQDN's 177 It should be a near exact copy of the STDIN or g_filename log file */ 178 extern char *g_import_filename; /* Command Line specified file to open for Importing a dumped DB */ 179 180 extern int g_verbosity; /* How many extra stuff to show. If 0, none 181 Bigger numbers, show more. */ 182 183 extern bool g_dolookups; /* Default. Do lookups of addresses 184 If False then will retrieve results from DB 185 and output substituted log lines */ 186 extern bool g_dotranslate; /* Do Translations */ 187 extern bool g_dorecombine; /* Do Recombining */ 188 extern bool g_dumpdnsdb; /* Dump the DNS History Database to STDOUT */ 189 extern bool g_showhistory; /* Given one or more IP's show their history */ 190 extern bool g_doimport; /* Import a Dumped DB from a File */ 191 192 extern int g_dns_lookups; /* Number of attempts for each lookup if not successful */ 193 extern unsigned int g_max_threads; /* Maximum number of threads to spawn off */ 194 extern int g_dns_retry_delay; /* Delay time between dns lookup retries in a single run. Seconds */ 195 196 extern int g_log_type; /* What type of log file is this? LOG_???? */ 197 198 /************************************************************************ 199 * FUNCTIONS * 200 ************************************************************************/ 201 void process_options(int, char *[]); /* Process command line options */ 202 203 /************************************************************************ 204 * MUTEXS * 205 ************************************************************************/ 206 extern pthread_mutex_t mutex_thread_count; /* Lock access to thread_count */ 207 extern pthread_mutex_t mutex_db_access; /* Lock access to DB activites */ 208 209 extern pthread_cond_t cond_thread_count; /* Wait for another thread to signal termination */ 210 211 extern unsigned int thread_count; /* How many outsyanding threads - to know when to exit */ 212 extern unsigned int thread_count_max; /* How many maximum number of concurrent threads */ 213 extern unsigned int set_thread_maximum; /* Total Maximum number of concurrent threads */ 214 215 extern pthread_mutex_t mutex_malloc_dns_rec; 216 extern int malloc_dns_rec; 217 extern pthread_mutex_t mutex_malloc_dns_list; 218 extern int malloc_dns_list; 219 220 221 #endif /* DNSHISTORY_DNSHISTORY_H */ 222 /************************************************************************ 223 ************************************************************************ 224 * END OF FILE * 225 ************************************************************************/ 226