1#!/bin/sh 2 3########################################################################### 4# 5# Shell program to parse firewall logs and analyze them with Analog. 6# 7# Copyright 2001-2002, Balazs Barany balazs@tud.at 8# 9# Version 0.6.9 10# 11# This program is free software; you can redistribute it and/or 12# modify it under the terms of the GNU General Public License as 13# published by the Free Software Foundation; either version 2 of the 14# License, or (at your option) any later version. 15# 16# This program is distributed in the hope that it will be useful, but 17# WITHOUT ANY WARRANTY; without even the implied warranty of 18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 19# General Public License for more details. 20# 21# Description: 22# 23# 24# NOTE: You must be the superuser to run this script or at least have 25# access to the firewall logs. (See README.sudo for a solution.) 26# 27# Usage: 28# 29# fwanalog.sh [ -h | --help ] [-c conffile][-r] [-t] [-y] [-a IP-addr] [-p packet] 30# 31# Options: 32# 33# -h, --help Display this help message and exit. 34# -r Rotate log cache 35# -c conffile Use this config file instead of ./fwanalog.opts 36# -t Only update statistics for today (for hourly use in crontab) 37# The sep_hosts and sep_packets commands in fwanalog.opts 38# are ignored. 39# -y Like -t, only for yesterday 40# -a IP-addr Create a separate report for this host 41# -p packet Create a separate report for this packet 42# Format: target/protocol/portnumber 43# e.g. 192.168.0.1/tcp/21 or firewall/udp/137 44# 45# 46# $Id: fwanalog.sh,v 1.81 2005/02/24 16:59:44 bb Exp $ 47# 48# Revisions: 49# 50# 2001-04-07 File created 51# 2001-04-08 First release 0.1, announced on Freshmeat 52# 2001-04-15 Release 0.2: Linux 2.2 ipchains support 53# 2001-05-05 Release 0.2.1: Analog 5.0 support, bugfixes 54# 2001-06-07 Release 0.2.2: FreeBSD support 55# 2001-08-05 Release 0.3: Bugfixes; ICMP support on Linux; onehost=dynip 56# 2001-08-18 Version 0.4pre: Speed improvement in the diff phase 57# 2001-08-23 Release 0.4: -t option, bugfixes in the pre version 58# 2001-11-23 Release 0.4.1: regexp bugfixes in iptables() and ipchains() 59# 2001-12-22 Version 0.5pre: OpenBSD 3.0 pf and Solaris support 60# 2002-02-19 Version 0.5: iptables log-prefix support, portability fixes 61# 2002-02-23 Version 0.5.1: better error handling; analog 5.21 compatible 62# 2002-03-03 Version 0.5.2: added ZyNOS parser 63# 2002-03-07 Version 0.6pre: optional separate reports for each packet 64# and host 65# 2002-04-28 Version 0.6: integrated change requests from lots of people 66# 2002-04-28 Version 0.6.1: some bugfixes with packet/host report generation 67# 2003-01-03 Version 0.6.2pre1: Support for Cisco PIX firewall logs 68# 2003-01-08 Version 0.6.2: Released as pix() seems to work and because 69# the new analog version requires a new langfile 70# 2003-01-14 Version 0.6.3pre1: Support for Watchguard Firebox logs 71# 2003-01-18 Version 0.6.3pre2: New -y option for yesterday's logs, smaller 72# fixes 73# 2003-01-21 Version 0.6.3pre3: Bugfix in watchguard(): allow two-digit dates 74# 2003-01-22 Version 0.6.3pre4: pix(): allow [] around the logging host name 75# 2003-02-13 Version 0.6.3pre5: pix(): the PIX date seems to be optional with 76# some configurations 77# 2003-02-13 Version 0.6.3pre6: Added the -c option for using a different 78# config file 79# 2003-03-11 Version 0.6.3pre7: Added support for Firewall-One 80# (written by Jean-Louis Saint-Dizier) 81# 2003-03-17 Version 0.6.3: Finally releasing it as stable 82# 2003-03-20 Version 0.6.4pre1: Added support for Cisco routers with 83# access-lists, further fixes in cisco() 84# 2003-06-19 Version 0.6.4pre2: Fixes in many functions, mostly in cisco() 85# 2003-11-25 Version 0.6.4pre4: Smaller fixes, mainly for PIX 86# 2004-03-18 Version 0.6.4: PIX fixes, released as 0.6.4 on the request of 87# the Debian maintainer 88# 2005-02-24 Version 0.6.9: PIX fix, added contributed ipfw, sonicwall 89# parsers 90# 91########################################################################### 92 93########################################################################### 94# Constants 95########################################################################### 96 97# Script options 98 99PROGNAME=$(basename $0) 100VERSION="0.6.9" 101 102########################################################################### 103# Variables 104########################################################################### 105 106# 107# Only update today's page - initialize with false 108# 109today_only=false 110 111# 112# Only update yesterday's page - initialize with false 113# 114yesterday_only=false 115 116########################################################################### 117# Commands - Assist in platform portability - with defaults 118########################################################################### 119 120sed=${sed:-sed} 121perl=${perl:-perl} 122grep=${grep:-grep} 123egrep=${egrep:-egrep} 124zegrep=${zegrep:-zegrep} 125analog=${analog:-analog} 126date=${date:-date} 127 128########################################################################### 129# Functions 130########################################################################### 131 132main () 133{ 134 # Function to do everything the script normally does. 135 136 # Get today's date for the daily reports. 137 TODAY=`$date +%y%m%d` 138 139 if [ X"$configfile" != X ]; then 140 # A config file is given 141 if [ -r "$configfile" ]; then 142 # use this if it is readable 143 . "$configfile" 144 145 # change into /usr/local/etc/fwanalog 146 cd /usr/local/etc/fwanalog 147 else 148 # specified config file isn't readable or doesn't exist 149 echo "fwanalog: couldn't read specified config file '$configfile'" >> /dev/stderr 150 exit 1 151 fi 152 else 153 # change into /usr/local/etc/fwanalog 154 cd /usr/local/etc/fwanalog 155 156 # Load the user-settable options from the config file 157 . ./`basename $0 | $sed 's/$/.opts/'` 158 fi 159 160 if [ -z "$inputfiles" ]; then 161 echo "fwanalog: No input files in the '$inputfiles_dir' directory " >> /dev/stderr 162 echo "named $inputfiles_mask and under $inputfiles_mtime days old." >> /dev/stderr 163 exit 1 164 fi 165 166 # create the output directory if necessary, ignore errors 167 mkdir -p $outdir 168 169 # Check if the lock file is there. If yes, warn and exit. 170 if [ -e $outdir/fwanalog.lock ]; then 171 echo "fwanalog: found lockfile '$outdir/fwanalog.lock'. " >> /dev/stderr 172 echo "This could mean that another instance is running." >> /dev/stderr 173 echo "If this is not the case, please remove the lock file." >> /dev/stderr 174 exit 1 175 fi 176 177 # Create lock file 178 touch $outdir/fwanalog.lock 179 180 # Parse the logs into a format Analog understands 181 $logformat 182 183 # make sure the "all logs" file exists 184 touch $outdir/fwanalog.all.log 185 186 # Find the new lines since the last invocation 187 if [ -s $outdir/fwanalog.all.log ]; then 188 # there is already an old log - find its last line and use it 189 # to determine the new contents of the grepped/converted file 190 $grep . $outdir/fwanalog.all.log \ 191 | tail -n 1 \ 192 | $sed 's/[^a-zA-Z0-9 _-]/./g' \ 193 | $sed 's#^\(.*\)$#0,/^\1$/d#' \ 194 > $outdir/match_last_line.sed 195 # match_last_line.sed now contains the last line in regexp form, so 196 # it can be searched in the new file. Most non-alphanumeric chars 197 # have been replaced with . so they don't act as metacharacters. 198 199 $grep . $outdir/fwanalog.all.log \ 200 | tail -n 1 \ 201 | $sed 's/[^a-zA-Z0-9 _-]/./g' \ 202 > $outdir/match_last_line.pattern 203 # create the regexp for grep 204 205 # The two "$grep ."-s are for RedHat 7.1 systems with a broken zegrep 206 # which appends a blank line at the end of its output 207 208 # Check if there is a common part in the old an the new log 209 if $grep --silent "`cat $outdir/match_last_line.pattern`" $outdir/fwanalog.current.log ; then 210 # there is a common part 211 212 # Delete the common lines in the current log so only the new ones 213 # stay and write it to the end of the global log 214 $sed -f $outdir/match_last_line.sed $outdir/fwanalog.current.log \ 215 >> $outdir/fwanalog.all.log 216 217 # Save the new lines in current.log.1 and move that over current.log 218 $sed -f $outdir/match_last_line.sed $outdir/fwanalog.current.log \ 219 > $outdir/fwanalog.current.log.1 220 mv $outdir/fwanalog.current.log.1 $outdir/fwanalog.current.log 221 else 222 # no common part 223 cat $outdir/fwanalog.current.log >> $outdir/fwanalog.all.log 224 fi 225 else 226 # There is no old log. We can use the entire current log. 227 cp $outdir/fwanalog.current.log $outdir/fwanalog.all.log 228 fi 229 230 # Create an empty domain cache for analog so it doesn't complain 231 touch $outdir/analog-domains.tab 232 233 # Ask Analog's version number 234 analogver=`$analog --help 2>&1 \ 235 | $grep "This is analog version" \ 236 | $sed 's!^.*\([0-9]\)\.[0-9]*/.*$!\1!'` 237 238 # If the version couldn't be determined, chances are that analog is not 239 # really executable (misconfiguration!) 240 if [ "X$analogver" = "X" ]; then 241 echo "fwanalog: Analog's version could not be determined." 242 echo " Please check if it is installed, executable and" 243 echo " correctly given in fwanalog.opts." 244 echo "exiting." 245 246 exit 1 247 fi 248 249 # Command line option for the debugging phase: list corrupt logfile entries 250 # This is important if bugs appear but doesn't disturb if they don't 251 analogopts="$analogopts +V+C" 252 253 # Version-dependent Analog config file 254 touch $outdir/fwanalog.analog.conf.ver 255 256 # Don't warn in case of empty reports, this is good for daily reports 257 # in case you're not attacked on this day 258 noemptyreportwarning=" +q-R" 259 260 rm -f "$outdir/analog.err" 261 262 # Generate a runtime config file 263 genconffile="$outdir/fwanalog.analog.conf.gen" 264 265 echo "DNSFILE $outdir/analog-domains.tab" > "$genconffile" 266 echo "DNSLOCKFILE $outdir/analog-domains.lck" >> "$genconffile" 267 echo "LOGFILE $outdir/fwanalog.all.log*" >> "$genconffile" 268 echo "CONFIGFILE ./fwanalog.analog.conf.local" >> "$genconffile" 269 270 # Call analog for today with ascii output, suitable for an e-mailed daily report 271 $yesterday_only || $analog \ 272 -G +g./fwanalog.analog.conf $analogopts \ 273 -C"OUTFILE $outdir/today.txt" -C"OUTPUT ASCII" -d -W -m -4 -o -z -f -v \ 274 -C"GOTOS OFF" -C"RUNTIME OFF" -C"LASTSEVEN OFF" \ 275 +F$TODAY $noemptyreportwarning \ 276 +g"$genconffile" \ 277 2>> $outdir/analog.err 278 279 # Call analog for yesterday with ascii output, suitable for an e-mailed daily report 280 $yesterday_only && $analog \ 281 -G +g./fwanalog.analog.conf $analogopts \ 282 -C"OUTFILE $outdir/yesterday.txt" -C"OUTPUT ASCII" -d -W -m -4 -o -z -f -v \ 283 -C"GOTOS OFF" -C"RUNTIME OFF" -C"LASTSEVEN OFF" \ 284 +F-00-00-01:0000 +T-00-00-01:2359 $noemptyreportwarning \ 285 +g"$genconffile" \ 286 2>> $outdir/analog.err 287 288 #Determine if there is an active date limitation 289 datelimit=false 290 $today_only && datelimit=true 291 $yesterday_only && datelimit=true 292 293 # Set special options for Analog version 5 and higher 294 if [ $analogver -ge 5 ] 295 then 296 # Charts go to the output directory, name prefix is "alldates-" 297 echo "LOCALCHARTDIR $outdir/alldates-" >> "$genconffile" 298 echo "CHARTDIR alldates-" >> "$genconffile" 299 fi 300 301 # Call analog with all data 302 $datelimit || $analog \ 303 -G +g./fwanalog.analog.conf $analogopts \ 304 -C"OUTFILE $outdir/alldates.html" \ 305 +g"$genconffile" \ 306 2>> $outdir/analog.err 307 308 if [ X"$reportmagic" = "Xtrue" ]; then 309 # Call analog with all data for ReportMagic 310 $datelimit || $analog \ 311 -G +g./fwanalog.analog.conf $analogopts \ 312 -C"OUTFILE $outdir/alldates.dat" -C"OUTPUT COMPUTER" \ 313 +g"$genconffile" \ 314 2>> $outdir/analog.err 315 fi 316 317 # Set special options for Analog version 5 and higher 318 if [ $analogver -ge 5 ] 319 then 320 # Charts go to the output directory, name prefix is "today-" 321 $perl -pwi -e "s!^LOCALCHARTDIR.+!LOCALCHARTDIR $outdir/today-!" \ 322 "$genconffile" 323 $perl -pwi -e "s!^CHARTDIR.+!CHARTDIR today-!" \ 324 "$genconffile" 325 fi 326 327 # Call analog for today, with the additional quarter-hour-report 328 $yesterday_only || $analog \ 329 -G +g./fwanalog.analog.conf $analogopts \ 330 -C"OUTFILE $outdir/today.html" -d -W -m \+4 \ 331 +F$TODAY $noemptyreportwarning \ 332 +g"$genconffile" \ 333 2>> $outdir/analog.err 334 335 # Call analog for yesterday if -y was specified, with html output 336 $yesterday_only && $analog \ 337 -G +g./fwanalog.analog.conf $analogopts \ 338 -C"OUTFILE $outdir/yesterday.html" -d -W -m \+4 \ 339 -C"LASTSEVEN OFF" \ 340 +F-00-00-01:0000 +T-00-00-01:2359 $noemptyreportwarning \ 341 +g"$genconffile" \ 342 2>> $outdir/analog.err 343 344 # Set special options for Analog version 5 and higher 345 if [ $analogver -ge 5 ] 346 then 347 # Charts go to the output directory, name prefix is "lastweek-" 348 $perl -pwi -e "s!^LOCALCHARTDIR.+!LOCALCHARTDIR $outdir/lastweek-!" "$genconffile" 349 $perl -pwi -e "s!^CHARTDIR.+!CHARTDIR lastweek-!" "$genconffile" 350 fi 351 352 # Call analog for the last 7 days, with the additional hourly report 353 $datelimit || $analog \ 354 -G +g./fwanalog.analog.conf $analogopts \ 355 -C"OUTFILE $outdir/lastweek.html" +H \ 356 +F-00-00-06 \ 357 +g"$genconffile" \ 358 2>> $outdir/analog.err 359 360 # Remove the unnecessary "HTML Conformant" lines from the output 361 # ignore error messages 362 $perl -pwi -e 's!^.+(validator\.w3\.org/"|nonehtml2\.(gif|png)|HTML 2\.0 Conformant).+$!!' \ 363 $outdir/alldates.html $outdir/today.html $outdir/lastweek.html \ 364 2> /dev/null 365 366 # If only today's or yesterday's report is generated, don't create separate 367 # host and packet reports unless $host_to_report or $packet_to_report is set 368 # (checked later) 369 if $datelimit; then 370 sep_hosts=false 371 sep_packets=false 372 fi 373 374 # Check if -a was used: create a separate report. 375 if [ X"$host_to_report" != X ]; then 376 sep_hosts=true 377 # must create the separate host report 378 fi 379 380 # If configured, create separate logs for each host from the current log or 381 # for the host given with -a 382 if [ "X$sep_hosts" = "Xtrue" ]; then 383 384 # The following characters are allowed in domain names - this is 385 # important because people could (perhaps) manipulate their reverse DNS 386 # to point to "../../../etc/passwd>.../something" and we would then, 387 # as root, overwrite that file with a report. 388 hostchars='a-zA-Z0-9._-' 389 390 if [ X"$host_to_report" = X ]; then 391 # Create a list of unique IPs in the current log 392 $sed 's/^\([0-9.]*\) .*$/\1/' $outdir/fwanalog.current.log \ 393 | sort -u \ 394 > $outdir/fwanalog.current.hosts.log 395 else 396 # Just use the provided IP address for the report 397 if $egrep --silent " ([0-9].){4} $host_to_report" $outdir/analog-domains.tab; then 398 # The address given on the command line is a domain name (not an 399 # IP), so extract the IP address. 400 host_to_report=`$egrep "([0-9].){4} $host_to_report" \ 401 $outdir/analog-domains.tab \ 402 | $perl -pwe "s/^\\d+ ([0-9.]+) $host_to_report/\$1/i"` 403 fi 404 405 echo "$host_to_report" > $outdir/fwanalog.current.hosts.log 406 fi 407 408 mkdir -p $outdir/hosts 409 410 # Create a separate report for each host 411 for host in `cat $outdir/fwanalog.current.hosts.log`; do 412 413 # Determine the dns name of this host, if existent 414 hostname=`$egrep "$host [$hostchars]+\$" $outdir/analog-domains.tab \ 415 | $perl -pwe 's/^[0-9]* [0-9.]* (.*)$/\L$1/' ` 416 #hostname can contain only the allowed characters ($hostchars). 417 #It is lowercased because analog also lowercases the names. 418 419 # Use the IP address if the name couldn't be resolved 420 if [ X"$hostname" = "X" ]; then 421 hostname=$host 422 fi 423 424 # Set special options for Analog version 5 and higher 425 if [ $analogver -ge 5 ] 426 then 427 # Charts go to the output directory, name prefix is "hosts/NAME-" 428 $perl -pwi -e "s!^LOCALCHARTDIR.+!LOCALCHARTDIR $outdir/hosts/$hostname-!" "$genconffile" 429 $perl -pwi -e "s!^CHARTDIR.+!CHARTDIR $hostname-!" "$genconffile" 430 fi 431 432 # Call analog with all data 433 $analog \ 434 -G +g./fwanalog.analog.conf $analogopts \ 435 -C"OUTFILE $outdir/hosts/$hostname.html" \ 436 -C"ORGANISATION OFF" -C"DOMAIN OFF" \ 437 -C"HOSTINCLUDE $hostname" \ 438 +g"$genconffile" \ 439 2>> $outdir/analog.err 440 441 # Remove the unnecessary "HTML Conformant" lines from the output 442 $perl -pwi -e 's!^.+(validator\.w3\.org/"|nonehtml2\.(gif|png)|HTML 2\.0 Conformant).+$!!' \ 443 $outdir/hosts/$hostname.html 444 done 445 446 fi 447 448 # Check if -p was used: create a separate packet report. 449 if [ X"$packet_to_report" != X ]; then 450 sep_packets=true 451 # must create the separate packet report 452 fi 453 454 # If configured, create separate logs for each packet from the current log 455 # or for the packet given with -p 456 if [ "X$sep_packets" = "Xtrue" ]; then 457 458 if [ X"$packet_to_report" = X ]; then 459 # Create a list of unique packets in the current log, and remove 460 # slashes from the end (for protocols without a port number) 461 $sed 's!^.*"GET /\([0-9a-zA-Z./]*\)/ HTTP.*$!\1!' $outdir/fwanalog.current.log \ 462 | sort -u \ 463 | $sed 's!/$!!' \ 464 > $outdir/fwanalog.current.packets.log 465 else 466 # Just use the provided packet for the report 467 echo "$packet_to_report" > $outdir/fwanalog.current.packets.log 468 fi 469 470 mkdir -p $outdir/packets 471 472 # Create a separate report for each packet 473 for packet in `cat $outdir/fwanalog.current.packets.log`; do 474 475 # Convert the packet into a matching pattern for analog's FILEINCLUDE 476 analogmatch=`echo $packet \ 477 | $perl -pwe 's!(firewall|[0-9.]+)/([a-z]+)/([0-9]+)!/$1/$2/*($3)/*!gi'` 478 479 # Convert the packet, which contains slashes, into a 480 # filesystem-friendly form 481 fsform=`echo $packet | $sed 's!/!-!g'` 482 483 # Set special options for Analog version 5 and higher 484 if [ $analogver -ge 5 ] 485 then 486 # Charts go to the output directory, name prefix is "packets/PACKET-" 487 $perl -pwi -e "s!^LOCALCHARTDIR.+!LOCALCHARTDIR $outdir/packets/$fsform-!" "$genconffile" 488 $perl -pwi -e "s!^CHARTDIR.+!CHARTDIR $fsform-!" "$genconffile" 489 fi 490 491 # Call analog with all data 492 $analog \ 493 -G +g./fwanalog.analog.conf $analogopts \ 494 -C"OUTFILE $outdir/packets/$fsform.html" \ 495 -C"FILEINCLUDE /$packet/*" \ 496 -C"FILEINCLUDE $analogmatch" \ 497 +g"$genconffile" \ 498 2>> $outdir/analog.err 499 500 # Remove the unnecessary "HTML Conformant" lines from the output 501 $perl -pwi -e 's!^.+(validator\.w3\.org/"|nonehtml2\.(gif|png)|HTML 2\.0 Conformant).+$!!' \ 502 $outdir/packets/$fsform.html 503 done 504 fi 505 506 # Change hosts in each generated report to point to the page about this host 507 508 # Search for host reports in the output directory and edit the output 509 # files to link to them 510 for hostlog in $outdir/hosts/*.html; do 511 512 # Get the hostname from the filename 513 hostname=`echo $hostlog | $sed 's/^.*hosts.\(.*\).html/\1/' ` 514 515 if [ X"$hostname" != "X*" ]; then 516 # there are files 517 518 # Replace all hosts with a URL pointing to the separate report 519 $perl -pwi -e \ 520 "s!(\\d+): ($hostname)\$!\$1: <a href=\"hosts/\$2.html\">\$2</a>!i" \ 521 $outdir/alldates.html $outdir/lastweek.html $outdir/today.html 522 523 # Do the same in each file in the packet directory 524 for packetlog in `$egrep -l "$hostname" $outdir/packets/*.html 2> /dev/null`; do 525 526 if [ -e "$packetlog" ]; then 527 $perl -pwi -e \ 528 "s!(\\d+): ($hostname)\$!\$1: <a href=\"../hosts/\$2.html\">\$2</a>!i" \ 529 "$packetlog" 530 fi 531 done 532 fi 533 done 534 535 # The same for packets 536 537 # Search for packet reports in the output directory and edit the output 538 # files to link to them 539 for packetlog in $outdir/packets/*.html; do 540 541 # Get the packet from the filename 542 packet=`echo $packetlog \ 543 | $sed 's!^.*/packets/\(.*\).html$!\1!' \ 544 | $sed 's!-!/!g' ` 545 546 # Get the relative filename 547 packetfile=`echo $packetlog \ 548 | $sed 's!^.*/packets/\(.*.html\)$!\1!' ` 549 550 # Convert the first form into a matching pattern 551 packetform1=`echo $packet \ 552 | $perl -pwe 's!^(firewall|[0-9.]+)/([a-z0-9]+)/([0-9]*)$!$1:$3/$2!i'` 553 # Convert the second form into a matching pattern 554 packetform2=`echo $packet \ 555 | $perl -pwe 's!^(firewall|[0-9.]+)/([a-z0-9]+)/([0-9]*)$!$1:[a-z0-9_*-]+ \\\\($3\\\\)/$2!i'` 556 557 if [ X"$packet" != "X*" ]; then 558 # there are packet logs 559 560 # Replace all packets with a URL pointing to the separate report 561 # - both possible forms 562 $perl -pwi -e \ 563 "s!(\\d+: +)($packetform1)\$!\$1<a href=\"packets/$packetfile\">\$2</a>!i" \ 564 $outdir/alldates.html $outdir/lastweek.html $outdir/today.html 565 $perl -pwi -e \ 566 "s!(\\d+: +)($packetform2)\$!\$1<a href=\"packets/$packetfile\">\$2</a>!i" \ 567 $outdir/alldates.html $outdir/lastweek.html $outdir/today.html 568 569 # Do the same in each host log 570 for hostlog in `$egrep -l "$packetform1|$packetform2" $outdir/hosts/*.html 2> /dev/null`; do 571 572 if [ -e "$hostlog" ]; then 573 574 $perl -pwi -e \ 575 "s!(\\d+: +)($packetform1)\$!\$1<a href=\"../packets/$packetfile\">\$2</a>!i" \ 576 "$hostlog" 577 $perl -pwi -e \ 578 "s!(\\d+: +)($packetform2)\$!\$1<a href=\"../packets/$packetfile\">\$2</a>!i" \ 579 "$hostlog" 580 fi 581 done 582 fi 583 done 584 585 keeperrfile=false 586 587 # check if there were corrupt lines 588 corruptlines=`$grep "^C: " $outdir/analog.err | wc -l` 589 if [ $corruptlines -ge 1 ]; then 590 echo "Analog found $corruptlines corrupt lines. Please consider sending " 591 echo "$outdir/analog.err to balazs@tud.at " 592 echo "so the author is able to fix the problem." 593 keeperrfile=true 594 fi 595 596 # check if Analog complains of an old language file 597 corruptlines=`$grep -i "error.*language file.*exiting" $outdir/analog.err | wc -l` 598 if [ $corruptlines -ge 1 ]; then 599 echo "Analog isn't happy about the language file. Probably you updated" 600 echo "to a new version. " 601 echo "Use the mklangfile.*.sh scripts in the fwanalog distribution" 602 echo "to create a new language file for fwanalog or get the current" 603 echo "version of fwanalog (or just the language files) from" 604 echo "http://tud.at/programm/fwanalog/" 605 keeperrfile=true 606 fi 607 608 # Check if there is an error which wasn't catched 609 corruptlines=`$grep "." $outdir/analog.err | wc -l` 610 if [ $corruptlines -ge 1 ]; then 611 if [ "X$keeperrfile" != 'Xtrue' ]; then 612 # There was no specific error message 613 echo "fwanalog: Analog printed the following error messages ($outdir/analog.err):" >> /dev/stderr 614 cat "$outdir/analog.err" >> /dev/stderr 615 keeperrfile=true 616 fi 617 fi 618 619 if [ "X$keeperrfile" != 'Xtrue' ]; then 620 # no problem, remove the error log 621 rm $outdir/analog.err 622 fi 623 624 # Clean up old logfiles 625 rm -f $outdir/fwanalog.curr* $outdir/fwanalog.new*.log $outdir/convdate.sed \ 626 $outdir/fwanalog.analog.conf.ver $outdir/fwanalog.analog.conf.gen $outdir/match_last_line.* 627 628 # Delete the lock file 629 rm -f $outdir/fwanalog.lock 630} 631 632iptables () 633{ 634 # Parse iptables logfiles into an analog-compatible "URL log" 635 636 $zegrep -h "IN.+OUT.+SRC.+DST.+LEN.+TTL.+PROTO.+" $inputfiles \ 637 | $sed 's/TYPE=\([0-9]\+\)/SPT= DPT=\1/' \ 638 > $outdir/fwanalog.current 639 640 mkdateconvscript 641 # Create script to convert lines without year to fully specified date 642 643 $sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 644 # Use the script on the current logfile 645 646 # Example of converted log line: 647 # 2001 Mar 31 00:58:17 www kernel: packet_explanation IN=eth1 OUT= MAC=00...:00 SRC=131....38 DST=212....31 LEN=44 \ 648 # TOS=0x00 PREC=0x00 TTL=57 ID=58478 PROTO=TCP SPT=61636 DPT=21 WINDOW=16384 RES=0x00 SYN URGP=0 649 650 # Example of desired output: 651 # 131....38 - packet_explanation [31/Mar/2001:00:58:17 +0200] "GET /212....31/TCP/21 HTTP/1.0" 200 \ 652 # 44 "61636" "00....:00" 10 eth1 653 # 654 # Which means: 655 # ip - iptables_log-prefix [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 656 # Sourceport is in the referrer field, macadr in the user-agent, interface 657 # in the VirtualHost. (The interface comes from IN= and/or OUT=) 658 # There is not always a MAC address, e.g. if the interface is ppp0 659 660 # Decide if the source or the destination host is included in the 661 # Blocked Packet Report (option "onehost" in fwanalog.opts) 662 if [ $onehost = true ]; then 663 reqhost="\$9" # The analog "request" contains the source ip 664 elif [ $onehost = dynip ]; then 665 reqhost="firewall" # The analog "request" contains this string 666 else 667 reqhost="\$10" # The analog "request" contains the destination ip 668 fi 669 # 1 2 3 4 5 6 7 8 9 10 11 12 13 14 670 $perl -pwe "s!^(\d+) +(\w+) +(\d+) ([0-9:]+) [^:]+:? ?([a-zA-Z0-9/.,:_-]*).*IN=(.*) OUT=(\S*) ?M?A?C?=?(.*) SRC=([0-9.]+) DST=([0-9.]+) LEN=(\d+)[^[]+PROTO=([a-zA-Z0-9]+)(?: SPT=)?(\d*)(?: DPT=)?(\d*).*\$!\$9 - \$5 [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/\$12/\$14/ HTTP/1.0\" 200 \$11 \"http://\$13/\" \"\$8\" 0 \$6\$7!" \ 671 $outdir/fwanalog.current.withyear > $outdir/fwanalog.current.log 672 673 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 674} 675 676ipf () 677{ 678 openbsd 679 # For backward compatibility. 680 # Initially, I thought that each BSD with ipf uses the same format. Wrong. 681} 682 683solarisipf () 684{ 685 # Adapted from the openbsd function below 686 687 # Parse Solaris ipf syslog files into an analog-compatible "URL log" 688 # Tested with Solaris 8 INTEL and ipf 3.4.20 689 690 ${zegrep} -h 'ipmon.+@[0-9:]+ b.+ -> .+ PR.+len' $inputfiles \ 691 > $outdir/fwanalog.current 692 693 mkdateconvscript 694 # Create script to convert lines without year to fully specified date 695 696 ${sed} -f $outdir/convdate.sed $outdir/fwanalog.current \ 697 > $outdir/fwanalog.current.withyear 698 # Use the script on the current logfile 699 700 # Example of converted log line: 701 # 2001 Apr 5 16:55:55 fw ipmon[1875]: 16:55:54.150871 xl0 @0:2 b 702 # 217.....93,3819 -> 195.....201,1080 PR tcp len 20 48 -S IN 703 # Example of desired output: 704 # 217....93 - - [5/Apr/2001:16:55:54 +0200] "GET /195.....201/tcp/1080 HTTP/1.0" 200 \ 705 # 20 "3819" "" 0 xl0 706 # 707 # Which means: 708 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 709 # Sourceport is in the referrer field, macadr in the user-agent, interface 710 # in the VirtualHost. There is no macadr in the BSD log. 711 712 # Decide if the source or the destination host is included in the 713 # Blocked Packet Report (option "onehost" in fwanalog.opts) 714 if [ $onehost = true ]; then 715 reqhost="\$7" # The analog "request" contains the source ip 716 elif [ $onehost = dynip ]; then 717 reqhost="firewall" # The analog "request" contains this string 718 else 719 reqhost="\$10" # The analog "request" contains the destination ip 720 fi 721 722 ${perl} -pwe \ 723 's! 724 ^ # Begin of line :-) 725 (\d+)\s+(\w+)\s+(\w+) \s+ # syslog year($1) month($2) day($3) 726 [0-9:]+ \s+ # syslog time 727 [a-zA-Z0-9_.]+ \s+ # syslog hostname 728 ipmon\[\d+\]: \s+ # ipmon process identifier 729 \[ID\s+\d+\s+\w+\.\w+\] \s+ # logging info 730 ([0-9:]+)\.\d+ \s+ # time($4).hirestime 731 .*\s*(\w+) \s+ # optional multipler and interface name($5) 732 \@[0-9:]+ \s+ # ruleset 733 . \s+ # action 734 ([a-zA-Z0-9-_.]+\[)?([0-9.]+)\]? # optional source name($6), source ip($7) 735 ,?([a-zA-Z0-9\-_]*) \s+ # source port($8) - may be name or number 736 -\> \s+ # the arrow :-) 737 ([a-zA-Z0-9-_.]+\[)?([0-9.]+)\]? # optional destination name($9), destination ip($10) 738 ,?([a-zA-Z0-9\-_]*) \s+ # destination port($11) - may be name or number 739 PR\s+(\w+) \s+ # protocol($12) 740 len\s+(\d+) # length($13) 741 .+ # ignore the rest 742 $ # End of line :-) 743 !$7 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/$12/$11/ HTTP/1.0" 200 $13 "http://$10/" "" 0 $5!x' \ 744 $outdir/fwanalog.current.withyear > $outdir/fwanalog.current.log 745 746 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 747} 748 749openbsd () 750{ 751 # Parse OpenBSD ipf logfiles into an analog-compatible "URL log" 752 # Tested with OpenBSD 2.8 ipf. 753 754 $zegrep -h "ipmon.+@[0-9:]+ b.+ -> .+ PR.+len" $inputfiles \ 755 > $outdir/fwanalog.current 756 757 mkdateconvscript 758 # Create script to convert lines without year to fully specified date 759 760 $sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 761 # Use the script on the current logfile 762 763 # Example of converted log line: 764 # 2001 Apr 5 16:55:55 fw ipmon[1875]: 16:55:54.150871 xl0 @0:2 b 765 # 217.....93,3819 -> 195.....201,1080 PR tcp len 20 48 -S IN 766 # Example of desired output: 767 # 217....93 - - [5/Apr/2001:16:55:54 +0200] "GET /195.....201/tcp/1080 HTTP/1.0" 200 \ 768 # 20 "3819" "" 0 xl0 769 # 770 # Which means: 771 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 772 # Sourceport is in the referrer field, macadr in the user-agent, interface 773 # in the VirtualHost. There is no macadr in the BSD log. 774 775 # Decide if the source or the destination host is included in the 776 # Blocked Packet Report (option "onehost" in fwanalog.opts) 777 if [ $onehost = true ]; then 778 reqhost="\$6" # The analog "request" contains the source ip 779 elif [ $onehost = dynip ]; then 780 reqhost="firewall" # The analog "request" contains this string 781 else 782 reqhost="\$8" # The analog "request" contains the destination ip 783 fi 784 785 # 1 2 3 4 5 6 7 8 9 10 11 786 $perl -pwe "s!^(\d+) +(\w+) +(\w+) .+: ([0-9:]+)\.\d+.+ +(\w+) @.+ . ([0-9.]+),?(\d*) -\\> ([0-9.]+),?(\d*) PR (\w+) len (\d+).+\$!\$6 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/\$10/\$9/ HTTP/1.0\" 200 \$11 \"http://\$7/\" \"\" 0 \$5!" \ 787 $outdir/fwanalog.current.withyear > $outdir/fwanalog.current.log 788 789 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 790} 791 792pf_30 () 793{ 794 # Parse OpenBSD 3.0 pf logfiles into an analog-compatible "URL log" 795 # This *must* happen on an OpenBSD 3.0 system as it requires the OpenBSD 796 # version of tcpdump. 797 798 (for log in $inputfiles ; do 799 $gzcat -f $log \ 800 | $tcpdump -n -e -ttt -q -r - 801 done) \ 802 | $egrep -h "rule .+: block .+ on .+ [0-9.]{7}" \ 803 > $outdir/fwanalog.current 804 805 mkdateconvscript 806 # Create script to convert lines without year to fully specified date 807 808 $sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 809 # Use the script on the current logfile 810 811 # Example of converted log line: 812 # TCP: 813 # 2001 Dec 21 17:48:50.760648 rule 12/0(match): block in on ae0: 814 # 192.168.49.2.2081 > 192.168.49.3.22: S 2901914301:2901914301(0) 815 # win 5840 <mss 1460,sackOK,timestamp 6674376 0,nop,wscale 0> (DF) 816 # UDP: 817 # 2001 Dec 20 20:16:24.674266 rule 2/0(match): block in on ae0: 818 # 192.168.49.3.137 > 192.168.49.255.137: udp 50 (ttl 64, id 61825) 819 # ICMP: 820 # 2001 Dec 20 20:21:00.324025 rule 3/0(match): block in on ae0: 821 # 192.168.49.1 > 192.168.49.3: icmp: echo reply (id:23464 seq:2) (ttl 255, id 21394) 822 # 823 # Example of desired output: 824 # 192.168.49.2 - - [5/Apr/2001:16:55:54 +0200] "GET /192.168.49.3/tcp/22 HTTP/1.0" 825 # 200 20 "2081" "" 0 ae0 826 # 827 # Which means: 828 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 829 # Sourceport is in the referrer field, macadr in the user-agent, interface 830 # in the VirtualHost. There is no macadr in the BSD log. 831 832 # Decide if the source or the destination host is included in the 833 # Blocked Packet Report (option "onehost" in fwanalog.opts) 834 # altreqhost is needed for unknown protocols (e.g. esp, ah) 835 if [ $onehost = true ]; then 836 reqhost="\$6" # The analog "request" contains the source ip 837 altreqhost="\$7" 838 elif [ $onehost = dynip ]; then 839 reqhost="firewall" # The analog "request" contains this string 840 altreqhost="firewall" 841 else 842 reqhost="\$8" # The analog "request" contains the destination ip 843 altreqhost="\$9" 844 fi 845 846 #first TCP, then UDP, then ICMP, then others (hopefully this works) 847 # 1 2 3 4 5 6 7 8 9 848 $perl -pwe "s!^(\d+) +(\w+) +(\d+) +([0-9:]+)\.\d+ rule.+block \w+ on (\w+): ([0-9.]+)\.(\d+) \\> ([0-9.]+)\.(\d+): tcp (\d+)(.*)\$!\$6 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/tcp/\$9/ HTTP/1.0\" 200 \$10 \"http://\$7/\" \"\" 0 \$5!" \ 849 $outdir/fwanalog.current.withyear \ 850 |$perl -pwe "s!^(\d+) +(\w+) +(\d+) +([0-9:]+)\.\d+ rule.+block \w+ on (\w+): ([0-9.]+)\.(\d+) \\> ([0-9.]+)\.(\d+): +udp (\d+).*\$!\$6 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/udp/\$9/ HTTP/1.0\" 200 \$10 \"http://\$7/\" \"\" 0 \$5!" \ 851 |$perl -pwe "s!icmp: echo re(quest|ply)!icmp: echo_re\$1!" \ 852 |$perl -pwe "s!icmp: host(.+)unreachable!icmp: host_unreachable!" \ 853 |$perl -pwe "s!^(\d+) +(\w+) +(\d+) +([0-9:]+)\.\d+ rule.+block \w+ on (\w+): ([0-9.]+)(X?) \\> ([0-9.]+): icmp: ([a-z][a-z_]+).*\$!\$6 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/icmp/\$9/ HTTP/1.0\" 200 0 \"http://\$7/\" \"\" 0 \$5!" \ 854 |$perl -pwe "s!^(\d+) +(\w+) +(\d+) +([0-9:]+)\.\d+ rule.+block \w+ on (\w+): ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.?(\d*) \\> ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.?(\d*): (\S*) ?(\d*).*\$!\$6 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/\$10/\$9/ HTTP/1.0\" 200 0\$11 \"http://\$7/\" \"\" 0 \$5!" \ 855 |$perl -pwe "s!^(\d+) +(\w+) +(\d+) +([0-9:]+)\.\d+ rule.+block \w+ on (\w+): (\w+) ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.?(\d*) \\> ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.?(\d*).*len (\d+).*\$!\$7 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$altreqhost/\$6\$10/ HTTP/1.0\" 200 \$11 \"http://\$8\" \"\" 0 \$5!" \ 856 > $outdir/fwanalog.current.log 857 858 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 859} 860 861freebsd () 862{ 863 # Parse FreeBSD ipf logfiles into an analog-compatible "URL log" 864 # Tested with FreeBSD ipf 865 866 $zegrep -h " -> .+ PR.+len" $inputfiles \ 867 > $outdir/fwanalog.current 868 869 mkmonthconvscript 870 # Create script to convert lines with a numeric month to the alphanumeric month (Jan...Dec) 871 872 $sed -f $outdir/convdate.sed $outdir/fwanalog.current \ 873 > $outdir/fwanalog.current.withmonth 874 # Use the script on the current logfile 875 876 # Example of converted log line: 877 # 04/06/2001 16:55:55.418398 tun0 @0:2 b 878 # 217.....93,3819 -> 195.....201,1080 PR tcp len 20 48 -S IN 879 # Example of desired output: 880 # 217....93 - - [5/Apr/2001:16:55:54 +0200] "GET /195.....201/tcp/1080 HTTP/1.0" 200 \ 881 # 20 "3819" "" 0 xl0 882 # 883 # Which means: 884 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 885 # Sourceport is in the referrer field, macadr in the user-agent, interface 886 # in the VirtualHost. There is no macadr in the BSD log. 887 888 # Decide if the source or the destination host is included in the 889 # Blocked Packet Report (option "onehost" in fwanalog.opts) 890 if [ $onehost = true ]; then 891 reqhost="\$4" # The analog "request" contains the source ip 892 elif [ $onehost = dynip ]; then 893 reqhost="firewall" # The analog "request" contains this string 894 else 895 reqhost="\$6" # The analog "request" contains the destination ip 896 fi 897 898 # 1 2 3 4 5 6 7 8 9 899 $perl -pwe "s!^(\d+/\w+/\d+) ([0-9:]+)\.\d+ *[0-9]*x? +(\w+) @.+ . ([0-9a-f.:]+),*(\d*) -\\> ([0-9a-f.:]+),*(\d*) PR (\w+) len (\d+).+\$!\$4 - - [\$1:\$2 $timezone] \"GET /$reqhost/\$8/\$7/ HTTP/1.0\" 200 \$9 \"http://\$5/\" \"\" 0 \$3!" \ 900 $outdir/fwanalog.current.withmonth > $outdir/fwanalog.current.log 901 902 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 903} 904 905ipchains () 906{ 907 # Parse ipchains logfiles into an analog-compatible "URL log" 908 # ipchains doesn't write the protocol name into the log, only the protocol number. 909 # So we convert them here manually. 910 911 $zegrep -h "Packet log: .+ (DENY|REJECT) .+PROTO=.+L=.+S.+I=.+F=.+T=" $inputfiles \ 912 | $sed 's/PROTO=1 /PROTO=icmp /' \ 913 | $sed 's/PROTO=2 /PROTO=igmp /' \ 914 | $sed 's/PROTO=6 /PROTO=tcp /' \ 915 | $sed 's/PROTO=17 /PROTO=udp /' \ 916 > $outdir/fwanalog.current 917 918 mkdateconvscript 919 # Create script to convert lines without year to fully specified date 920 921 $sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 922 # Use the script on the current logfile 923 924 # Example of converted log line: 925 # 2001 Apr 18 06:26:18 extdevel kernel: Packet log: input DENY eth0 PROTO=17 \ 926 # 193.83.115.48:137 193.83.115.255:137 L=78 S=0x00 I=60301 F=0x0000 T=128 (#9) 927 # Example of desired output: 928 # 131....38 - - [31/Mar/2001:00:58:17 +0200] "GET /212....31/TCP/21 HTTP/1.0" 200 \ 929 # 44 "61636" "00....:00" 10 eth1 930 # 931 # Which means: 932 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 933 # Sourceport is in the referrer field, macadr in the user-agent, interface 934 # in the VirtualHost. 935 # There is no MAC address in ipchains logs. 936 937 # Decide if the source or the destination host is included in the 938 # Blocked Packet Report (option "onehost" in fwanalog.opts) 939 if [ $onehost = true ]; then 940 reqhost="\$8" # The analog "request" contains the source ip 941 elif [ $onehost = dynip ]; then 942 reqhost="firewall" # The analog "request" contains this string 943 else 944 reqhost="\$10" # The analog "request" contains the destination ip 945 fi 946 947 # 1 2 3 4 5 6 7 8 9 10 11 12 948 cat $outdir/fwanalog.current.withyear \ 949 | $perl -pwe "s!^(\d+) +(\w+) +(\d+) ([0-9:]+) .+(DENY|REJECT) ([a-z0-9]+) PROTO=([\w-]+) ([0-9.]+):?(\d*) ([0-9.]+):?(\d*) L=(\d+).+\$!\$8 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/\$7/\$11/ HTTP/1.0\" 200 \$12 \"http://\$9/\" \"\" 0 \$6!" \ 950 | $perl -pwe "s!^(\d+) +(\w+) +(\d+) ([0-9:]+) .+(DENY|REJECT) ([a-z0-9]+) PROTO=(ICMP/[0-9]+):?[0-9]* ([0-9.]+)(x?) ([0-9.]+)(x?) L=(\d+).+\$!\$8 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/\$7/ HTTP/1.0\" 200 \$12 \"http://\$9/\" \"\" 0 \$6!" \ 951 > $outdir/fwanalog.current.log 952 953 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 954} 955 956 957ipfw () 958{ 959 # fwanalog extension for freebsds ipfw 960 # 15/Sept/2002 Peter Hunkirchen <phunkirchen@t-online.de> 961 962 # Parse ipfw logfiles into an analog-compatible "URL log" 963 964 $zegrep -h "Deny" $inputfiles \ 965 > $outdir/fwanalog.current 966 967 mkdateconvscript 968 # Create script to convert lines without year to fully specified date 969 970 $sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 971 # Use the script on the current logfile 972 973 # Example of converted log line: 974 # 2002 Sep 15 07:47:04 yepp /kernel: ipfw: 65435 Deny UDP 80.133.123.52:1042 165.132.149.211:4665 out via tun0 975 # Example of desired output: 976 # 131....38 - - [31/Mar/2001:00:58:17 +0200] "GET /212....31/TCP/21 HTTP/1.0" 200 \ 977 # 44 "61636" "00....:00" 10 eth1 978 # 979 # Which means: 980 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 981 # Sourceport is in the referrer field, macadr in the user-agent, interface 982 # in the VirtualHost. 983 # There is no MAC address in ipchains logs. 984 985 # Decide if the source or the destination host is included in the 986 # Blocked Packet Report (option "onehost" in fwanalog.opts) 987 if [ $onehost = true ]; then 988 reqhost="\$8" # The analog "request" contains the source ip 989 elif [ $onehost = dynip ]; then 990 reqhost="firewall" # The analog "request" contains this string 991 else 992 reqhost="\$10" # The analog "request" contains the destination ip 993 fi 994 995 # 1 2 3 4 5 6 7 8 9 10 11 12 13 996 $perl -pwe "s!^(\d+) +(\w+) +(\d+) ([0-9:]+) .+(Deny|Reject) ([\w-]+) ([0-9.]+):(\d*) ([0-9.]+):(\d*) ([\w-]+) ([\w-]+) ([\w-]+)\$!\$7 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/\$6/\$10/ HTTP/1.0\" 200 1 \"http://\$8/\" \"\" 0 \$13 !" $outdir/fwanalog.current.withyear > $outdir/fwanalog.current.log 997 998 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 999} 1000 1001zynos () 1002{ 1003 # Parse ZynOS (ZyXEL, NETGEAR) logfiles into an analog-compatible "URL log" 1004 1005 # This pattern excludes "last message repeated X times" lines 1006 # so the count will be artificially low. How to handle?!? 1007 $zegrep -h "IP.+Src.+Dst.+(ICMP|TCP|UDP).+spo.+dpo.+" $inputfiles \ 1008 > $outdir/fwanalog.current 1009 1010 mkdateconvscript 1011 # Create script to convert lines without year to fully specified date 1012 1013 $sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 1014 # Use the script on the current logfile 1015 1016 # Example of converted log line: 1017 # 2002 Feb 27 14:43:51 router host: IP[Src=164....2 Dst=65....189 TCP \ 1018 # spo=02945 dpo=00080]}S03>R02mD 1019 1020 # Example of desired output: 1021 # 164....2 - - [27/Feb/2002:14:43:51 +0500] "GET /65....189/TCP/80 HTTP/1.0" 200 \ 1022 # 1 "http://2945/" "" 0 router 1023 # 1024 # Which means: 1025 # SrcIP - - [date] "GET ReqHost/Protocol/DstPort HTTP/1.0" 200 1026 # FakePacketLen "http://SrcPort/" "" 0 routerName 1027 # SrcPort is in the referrer field, routerName in the VirtualHost. 1028 # There is no MAC address or packet length in NETGEAR/ZyXEL logs. 1029 1030 # Decide if the source or the destination host is included in the 1031 # Blocked Packet Report (option "onehost" in fwanalog.opts) 1032 if [ $onehost = true ]; then 1033 reqhost="\$6" # The analog "request" contains the source ip 1034 elif [ $onehost = dynip ]; then 1035 reqhost="firewall" # The analog "request" contains this string 1036 else 1037 reqhost="\$7" # The analog "request" contains the destination ip 1038 fi 1039 # 1 2 3 4 5 6 7 8 9 10 11 1040 $perl -pwe "s!^(\d+) +(\w+) +(\d+) ([0-9:]+) (\S+) [^:]+: +IP\[Src=([0-9\.]+) +Dst=([0-9\.]+) +(\S+) +spo=0*([0-9]+) +dpo=0*([0-9]+)\]}(\S+)\$!\$6 - - [\$3/\$2/\$1:\$4 $timezone] \"GET /$reqhost/\$8/\$10/ HTTP/1.0\" 200 1 \"http://\$9/\" \"\" 0 \$5!" \ 1041 $outdir/fwanalog.current.withyear > $outdir/fwanalog.current.log 1042 1043 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 1044} 1045 1046cisco () 1047{ 1048 # Parse Cisco PIX and router logfiles into an analog-compatible "URL log" 1049 # Tested with logs from Cisco PIX and routers with access-lists. 1050 # Adapted from the solarisipf() function 1051 1052 # Note: Cisco doesn't log packet lengts so each packet is faked to have 0 byte. 1053 # See Analog's SIZE and *COLS commands to turn off packet size reports. 1054 1055 pixpatterns="Inbound .+ connection denied from [0-9./]+ to [0-9./]+" 1056 pixpatterns="$pixpatterns|Deny inbound (udp|icmp|tcp) from [0-9./]+ to [0-9./]+" 1057 pixpatterns="$pixpatterns|Deny (inbound )?(\(No xlate\) )?(udp|icmp|tcp) src [^:]+:[0-9./]+ dst [^:]+:[0-9./]" 1058 pixpatterns="$pixpatterns|Deny TCP (\(no connection\) )?from [0-9./]+ to [0-9./]+.+on interface" 1059 pixpatterns="$pixpatterns|translation creation failed for (udp|icmp|tcp) src [^:]+:[0-9./]+ dst [^:]+:[0-9./]+" 1060 pixpatterns="$pixpatterns|No translation group found for (udp|icmp|tcp) src [^:]+:[0-9./]+ dst [^:]+:[0-9./]+" 1061 pixpatterns="$pixpatterns|: list .+ denied [a-z0-9]+ [0-9.()]+ -> [0-9.()]+, .+ packets?" 1062 ${zegrep} -hi "$pixpatterns" $inputfiles \ 1063 > $outdir/fwanalog.current 1064 1065 mkdateconvscript 1066 # Create script to convert lines without year to fully specified date 1067 1068 ${sed} -f $outdir/convdate.sed $outdir/fwanalog.current \ 1069 > $outdir/fwanalog.current.withyear 1070 # Use the script on the current logfile 1071 1072 # Examples of converted log lines: 1073 # 2002 Dec 24 09:14:18 example.com Dec 24 2002 15:14:18: 1074 # %PIX-2-306001: Inbound TCP connection denied from 1075 # 10.206.26.58/4011 to 10.96.160.115/80 flags SYN on interface outside 1076 # 2002 Dec 24 09:05:40 example.com Dec 24 2002 15:05:40: 1077 # %PIX-2-306006: Deny inbound UDP 1078 # from 10.114.112.73/1028 to 10.96.160.196/137 on interface outside 1079 # 2002 Dec 24 07:18:05 example.com Dec 24 2002 13:18:05: 1080 # %PIX-3-306011: Deny inbound (No xlate) icmp 1081 # src outside:10.249.118.254 dst outside:10.96.160.84 (type 8, code 0) 1082 1083 # Example of desired output: 1084 # 217....93 - - [5/Apr/2001:16:55:54 +0200] "GET /195.....201/tcp/1080 HTTP/1.0" 200 \ 1085 # 20 "3819" "" 0 xl0 1086 # 1087 # Which means: 1088 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 1089 # Sourceport is in the referrer field, macadr in the user-agent, interface 1090 # in the VirtualHost. There is no macadr in the BSD log. 1091 1092 # Decide if the source or the destination host is included in the 1093 # Blocked Packet Report (option "onehost" in fwanalog.opts) 1094 if [ $onehost = true ]; then 1095 reqhost="\$5" # The analog "request" contains the source ip 1096 reqhost_1="\$7" # in a more complex regexp is the position $7 instead of $5 1097 reqhost_2="\$6" # ... or $6 1098 elif [ $onehost = dynip ]; then 1099 reqhost="firewall" # The analog "request" contains this string 1100 else 1101 reqhost="\$7" # The analog "request" contains the destination ip 1102 reqhost_1="\$10" # in a more complex regexp is the position $10 instead of $7 1103 reqhost_2="\$8" # ... or $8 1104 fi 1105 1106 cat $outdir/fwanalog.current.withyear \ 1107 | ${perl} -pwe \ 1108 's! # Inbound TCP connection denied 1109 ^ # Begin of line :-) 1110 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1111 (?:\d{4}\s)? # optional year when timestamp is switched on 1112 ([0-9:]+) \s+ # syslog time $4 1113 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1114 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1115 .+ Inbound.TCP.connection.denied \s # PIX ID (?); verbose description 1116 from \s ([0-9.]+)/([0-9]+) \s # Source IP ($5), port ($6) 1117 to \s ([0-9.]+)/([0-9]+) \s # Destination IP ($7), port ($8) 1118 flags \s ([A-Z ]*) [ \t]+ # TCP flags ($9) 1119 (?:on.interface.)?([a-zA-Z0-9&_-]*)[ \t]* # interface ($10), possible whitespace 1120 $ # End of line :-) 1121 !$5 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/tcp/$8/ HTTP/1.0" 200 0 "http://$6/" "" 0 $10!x' \ 1122 | ${perl} -pwe \ 1123 's! # Deny TCP (no connection) 1124 ^ # Begin of line :-) 1125 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1126 (?:\d{4}\s)? # optional year when timestamp is switched on 1127 ([0-9:]+) \s+ # syslog time $4 1128 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1129 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1130 .+ Deny.TCP..no.connection. \s # PIX ID (?); verbose description 1131 from \s ([0-9.]+)/([0-9]+) \s # Source IP ($5), port ($6) 1132 to \s ([0-9.]+)/([0-9]+) \s # Destination IP ($7), port ($8) 1133 flags \s ([A-Z ]*) \s+ # TCP flags ($9) 1134 on.interface.([a-zA-Z0-9&_-]+) # interface ($10) 1135 .* $ # possible junk, end of line 1136 !$5 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/tcp/$8/ HTTP/1.0" 200 0 "http://$6/" "" 0 $10!x' \ 1137 | ${perl} -pwe \ 1138 's! # Deny inbound UDP, first version (with "on interface") 1139 ^ # Begin of line :-) 1140 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1141 (?:\d{4}\s)? # optional year when timestamp is switched on 1142 ([0-9:]+) \s+ # syslog time $4 1143 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1144 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1145 .+ Deny.inbound.UDP \s # PIX ID (?); verbose description 1146 from \s ([0-9.]+)/([0-9]+) \s # Source IP ($5), port ($6) 1147 to \s ([0-9.]+)/([0-9]+) \s # Destination IP ($7), port ($8) 1148 on.interface \s ([a-zA-Z0-9&_-]+)? # interface ($9), 1149 .* # possible whitespace or junk 1150 $ # End of line :-) 1151 !$5 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/udp/$8/ HTTP/1.0" 200 0 "http://$6/" "" 0 $9!x' \ 1152 | ${perl} -pwe \ 1153 's! # Deny inbound UDP, second version (without "on interface") 1154 ^ # Begin of line :-) 1155 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1156 (?:\d{4}\s)? # optional year when timestamp is switched on 1157 ([0-9:]+) \s+ # syslog time $4 1158 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1159 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1160 .+ Deny.inbound.UDP \s # PIX ID (?); verbose description 1161 from \s ([0-9.]+)/([0-9]+) \s # Source IP ($5), port ($6) 1162 to \s ([0-9.]+)/([0-9]+) \s # Destination IP ($7), port ($8) 1163 .* # possible whitespace or junk 1164 $ # End of line :-) 1165 !$5 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/udp/$8/ HTTP/1.0" 200 0 "http://$6/" "" 0 unknown!x' \ 1166 | ${perl} -pwe \ 1167 's! # Deny inbound (No xlate) (tcp|udp) 1168 ^ # Begin of line :-) 1169 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1170 (?:\d{4}\s)? # optional year when timestamp is switched on 1171 ([0-9:]+) \s+ # syslog time $4 1172 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1173 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1174 .+.Deny.inbound.(?:.No.xlate..)?(udp|tcp) \s # PIX ID (?); desc; protocol ($5) 1175 src \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/([0-9]+) \s # Interface $6, Source IP $7, port $8 1176 dst \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/([0-9]+) # Interface $9, Dest IP $10, port $11 1177 .* # possible whitespace or junk 1178 $ # End of line :-) 1179 !$7 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_1'/$5/$11/ HTTP/1.0" 200 0 "http://$8/" "" 0 $6-$9!xi' \ 1180 | ${perl} -pwe \ 1181 's! # Deny inbound (No xlate) icmp 1182 ^ # Begin of line :-) 1183 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1184 (?:\d{4}\s)? # optional year when timestamp is switched on 1185 ([0-9:]+) \s+ # syslog time $4 1186 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1187 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1188 .+.Deny.inbound(?:..No.xlate.)?.icmp \s # PIX ID (?); desc; protocol 1189 src \s ([a-zA-Z0-9&_.-]+):([0-9.]+) \s # Interface $5, Source IP $6 1190 dst \s ([a-zA-Z0-9&_.-]+):([0-9.]+) \s # Interface $7, Dest IP $8 1191 .type \s (\w+),.code .+ # ICMP type $9 1192 $ # End of line :-) 1193 !$6 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_2'/icmp/$9/ HTTP/1.0" 200 0 "http:///" "" 0 $5-$7!x' \ 1194 | ${perl} -pwe \ 1195 's! # Deny PROTOCOL src inside:... dst ... by access-group "ACL" 1196 ^ # Begin of line :-) 1197 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1198 (?:\d{4}\s)? # optional year when timestamp is switched on 1199 ([0-9:]+) \s+ # syslog time $4 1200 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1201 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1202 .+.Deny.(udp|tcp|icmp)\s # PIX ID (?); desc; protocol $5 1203 src \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/?(\d*)\s # Interface $6, Source IP $7, src port $8 1204 dst \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/?(\d*)\s # Interface $9, Dest IP $10, dest port $11 1205 (?:\(type.)?(\d*)(?:,.code.\d+\)\s)? # optional ICMP type $12 1206 (?:by.access-group.")?(\w*)"?.* # ACL group $13 1207 $ # End of line :-) 1208 !$7 - $13 [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_1'/$5/$11$12/ HTTP/1.0" 200 0 "http://$8/" "" 0 $6-$9!x' \ 1209 | ${perl} -pwe \ 1210 's! # translation creation failed for (tcp|udp) 1211 ^ # Begin of line :-) 1212 (\d+)\s(\w+)\s+(\d+) \s+ # syslog year $1 month $2 day $3 1213 (?:\d{4}\s)? # optional year when timestamp is switched on 1214 ([0-9:]+) \s+ # syslog time $4 1215 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1216 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1217 .+translation.creation.failed.for.(udp|tcp) \s # PIX ID (?); desc; protocol ($5) 1218 src \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/([0-9]+) \s # Interface $6, Source IP $7, port $8 1219 dst \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/([0-9]+) # Interface $9, Dest IP $10, port $11 1220 .* # possible whitespace or junk 1221 $ # End of line :-) 1222 !$7 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_1'/$5/$11/ HTTP/1.0" 200 0 "http://$8/" "" 0 $6-$9!x' \ 1223 | ${perl} -pwe \ 1224 's! # translation creation failed for icmp 1225 ^ # Begin of line :-) 1226 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1227 (?:\d{4}\s)? # optional year when timestamp is switched on 1228 ([0-9:]+) \s+ # syslog time $4 1229 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1230 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1231 .+translation.creation.failed.for.icmp \s # PIX ID (?); desc; protocol 1232 src \s ([a-zA-Z0-9&_.-]+):([0-9.]+) \s # Interface $5, Source IP $6 1233 dst \s ([a-zA-Z0-9&_.-]+):([0-9.]+) \s # Interface $7, Dest IP $8 1234 .type \s (\w+),.code .+ # ICMP type $9 1235 $ # End of line :-) 1236 !$6 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_2'/icmp/$9/ HTTP/1.0" 200 0 "http:///" "" 0 $5-$7!x' \ 1237 | ${perl} -pwe \ 1238 's! # No translation group found for udp/tcp 1239 ^ # Begin of line :-) 1240 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1241 (?:\d{4}\s)? # optional year when timestamp is switched on 1242 ([0-9:]+) \s+ # syslog time $4 1243 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1244 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1245 .+No.translation.group.found.for.(udp|tcp) \s # PIX ID (?); desc; protocol ($5) 1246 src \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/([0-9]+) \s # Interface $6, Source IP $7, port $8 1247 dst \s ([a-zA-Z0-9&_.-]+):([0-9.]+)/([0-9]+) # Interface $9, Dest IP $10, port $11 1248 .* # possible whitespace or junk 1249 $ # End of line :-) 1250 !$7 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_1'/$5/$11/ HTTP/1.0" 200 0 "http://$8/" "" 0 $6-$9!x' \ 1251 | ${perl} -pwe \ 1252 's! # No translation group found for icmp 1253 ^ # Begin of line :-) 1254 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1255 (?:\d{4}\s)? # optional year when timestamp is switched on 1256 ([0-9:]+) \s+ # syslog time $4 1257 (?:\[?[a-zA-Z0-9_.-]*\]?\s)?# optional syslog hostname 1258 (?:\w\w\w \s+ \d+ \s+ \d+ \s+ [0-9:]+ \s)? # optional PIX date/time in UTC 1259 .+No.translation.group.found.for.icmp \s # PIX ID (?); desc; protocol 1260 src \s ([a-zA-Z0-9&_.-]+):([0-9.]+) \s # Interface $5, Source IP $6 1261 dst \s ([a-zA-Z0-9&_.-]+):([0-9.]+) \s # Interface $7, Dest IP $8 1262 .type \s (\w+),.code .+ # ICMP type $9 1263 $ # End of line :-) 1264 !$6 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_2'/icmp/$9/ HTTP/1.0" 200 0 "http:///" "" 0 $5-$7!x' \ 1265 | ${perl} -pwe \ 1266 's! # list 101 denied tcp 64.70.54.95(20) -> 64.5.47.191(40436), 7 packets 1267 ^ # Begin of line :-) 1268 (\d+)\s(\w+)\s+(\w+) \s+ # syslog year $1 month $2 day $3 1269 (?:\d{4}\s)? # optional year when timestamp is switched on 1270 ([0-9:]+) \s+ # syslog time $4 1271 .+ # uninteresting data 1272 : \s list\s([^ ]+)\sdenied \s # rule number $5 1273 (\w+) \s # protocol $6 1274 ([0-9.]+)\(?(\d*)\)?\s(->)\s# Source IP $7, optional port $8; $9 just for compatibility with $reqhost_1 1275 ([0-9.]+)\(?(\d*)\)? # Dest IP $10, optional port $11 1276 , \s (\d+) \s packets? # Packet count $12 1277 $ # End of line :-) 1278 !$7 - $5 [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost_1'/$6/$11/ HTTP/1.0" 200 $12 "http://$8/" "" 0 !x' \ 1279 > $outdir/fwanalog.current.log 1280 1281 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 1282} 1283 1284pix () 1285{ 1286 # Alias for cisco() 1287 cisco 1288} 1289 1290watchguard () 1291{ 1292 # Parse Watchguard Firebox logfiles into an analog-compatible "URL log" 1293 # Tested with System 6.1 1294 # Adapted from the pix() function 1295 1296 wgpatterns=": deny (in|out) [a-z]+[0-9] [0-9]+ [a-z]+ [0-9]+ [0-9]+ ([0-9.]+ )+" 1297 ${zegrep} -hi "$wgpatterns" $inputfiles \ 1298 > $outdir/fwanalog.current 1299 1300 mkdateconvscript 1301 # Create script to convert lines without year to fully specified date 1302 1303 ${sed} -f $outdir/convdate.sed $outdir/fwanalog.current \ 1304 > $outdir/fwanalog.current.withyear 1305 # Use the script on the current logfile 1306 1307 # Examples of converted log lines: 1308 # 2003 Jan 4 15:41:01 216.234.247.49 firewalld[110]: 1309 # deny in eth0 84 icmp 20 254 216.234.234.120 216.234.249.147 1310 # 8 0 (blocked site) 1311 # 2003 Jan 4 15:41:56 216.234.247.49 firewalld[110]: 1312 # deny in eth0 78 udp 20 128 10.11.12.120 10.11.12.255 1313 # 137 137 (blocked site) 1314 1315 1316 # Example of desired output: 1317 # 217....93 - blocked_site [5/Apr/2001:16:55:54 +0200] "GET /195.....201/tcp/1080 HTTP/1.0" 200 \ 1318 # 20 "3819" "" 0 xl0 1319 # 1320 # Which means: 1321 # ip - reason [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "Macadr" 0 interface 1322 # Sourceport is in the referrer field, interface in the VirtualHost. 1323 1324 # Decide if the source or the destination host is included in the 1325 # Blocked Packet Report (option "onehost" in fwanalog.opts) 1326 if [ $onehost = true ]; then 1327 reqhost="\$8" # The analog "request" contains the source ip 1328 elif [ $onehost = dynip ]; then 1329 reqhost="firewall" # The analog "request" contains this string 1330 else 1331 reqhost="\$9" # The analog "request" contains the destination ip 1332 fi 1333 1334 1335 # First, add an empty "reason" to lines that don't have it 1336 1337 cat $outdir/fwanalog.current.withyear \ 1338 | ${perl} -pwe 's/([a-z0-9] )$/$1() /i' \ 1339 | ${perl} -pwe \ 1340 's! # ICMP 1341 ^ # Begin of line 1342 (\d+)\s+(\w+)\s+(\d+) \s+ # syslog year $1 month $2 day $3 1343 ([0-9:]+) \s+ # syslog time $4 1344 [0-9a-zA-Z_.-]+\s[a-z]+\[\d+\]:\s # ip/hostname, process name, PID 1345 deny\s[a-z]+ \s ([a-z]+\d+) \s # deny in/out, interface $5 1346 (\d+)\s(icmp) \s \d+\s\d+\s # Packet length $6, protocol $7 1347 ([0-9.]+) \s ([0-9.]+) \s # Source, dest IP $8, $9 1348 ([0-9]+) \s ([0-9]+) \s # ICMP type $10 1349 [a-z ()]* # sometimes TCP options 1350 \(([a-zA-Z0-9_. -]*)\) \s # Reason $12 1351 $ # End of line 1352 !$8 - $12 [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/$7/$10/ HTTP/1.0" 200 $6 "" "" 0 $5!x' \ 1353 | ${perl} -pwe \ 1354 's! # IGMP 1355 ^ # Begin of line 1356 (\d+)\s+(\w+)\s+(\d+) \s+ # syslog year $1 month $2 day $3 1357 ([0-9:]+) \s+ # syslog time $4 1358 [0-9a-zA-Z_.-]+\s[a-z]+\[\d+\]:\s # ip/hostname, process name, PID 1359 deny\s[a-z]+ \s ([a-z]+\d+) \s # deny in/out, interface $5 1360 (\d+)\s(igmp) \s \d+\s\d+\s # Packet length $6, protocol $7 1361 ([0-9.]+) \s ([0-9.]+) \s # Source, dest IP $8, $9 1362 ([^ ]*) \s* (.*) # Some optional info $10 1363 \(([a-zA-Z0-9_. -]*)\) \s # Reason $12 1364 $ # End of line 1365 !$8 - $12 [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/$7/$10/ HTTP/1.0" 200 $6 "" "" 0 $5!x' \ 1366 | ${perl} -pwe \ 1367 's! # TCP and UDP 1368 ^ # Begin of line 1369 (\d+)\s+(\w+)\s+(\d+) \s+ # syslog year $1 month $2 day $3 1370 ([0-9:]+) \s+ # syslog time $4 1371 [0-9a-zA-Z_.-]+\s[a-z]+\[\d+\]:\s # ip/hostname, process name, PID 1372 deny\s[a-z]+ \s ([a-z]+\d+) \s # deny in/out, interface $5 1373 (\d+)\s([a-z]+) \s \d+\s\d+\s # Packet length $6, protocol $7 1374 ([0-9.]+) \s ([0-9.]+) \s # Source, dest IP $8, $9 1375 ([0-9]+) \s ([0-9]+) \s # Source $10, destination port $11 1376 [a-z ()]* # sometimes TCP options 1377 \(([a-zA-Z0-9_. -]*)\) \s # Reason $12 1378 $ # End of line 1379 !$8 - $12 [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/$7/$11/ HTTP/1.0" 200 $6 "http://$10/" "" 0 $5!x' \ 1380 | ${perl} -pwe \ 's!^([0-9.]+) - ([^ ]+) ([a-z0-9_. -]+) \[(\d+/.+)$!$1 - $2_$3 [$4!i' \ 1381 | ${perl} -pwe \ 's!^([0-9.]+) - ([^ ]+) ([a-z0-9_. -]+) \[(\d+/.+)$!$1 - $2_$3 [$4!i' \ 1382 | ${perl} -pwe \ 's!^([0-9.]+) - ([^ ]+) ([a-z0-9_. -]+) \[(\d+/.+)$!$1 - $2_$3 [$4!i' \ 1383 > $outdir/fwanalog.current.log 1384 1385 # The last 3 perl lines convert spaces to underscores in the reason field. Therefore, 1386 # only three spaces (or fewer) are allowed. Duplicate the lines if you need more. 1387 1388 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 1389} 1390 1391fw1 () 1392{ 1393 # Parse FireWall-1 export logfiles into an analog-compatible "URL log" 1394 1395 $zegrep -h 'FireWall-1" "' $inputfiles \ 1396 | $egrep -v '"Accept"' \ 1397 > $outdir/fwanalog.current.withyear 1398 1399 ##mkdateconvscript 1400 # Create script to convert lines without year to fully specified date 1401 1402 ##$sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 1403 # Use the script on the current logfile 1404 1405 # Example of converted log line: 1406 # "6" "28May2002" "9:47:09" "VPN-1 & FireWall-1" "E100B2" "ntz" "Log" "Drop" \ 1407 # "Http-8080" "213.56.82.222" "213.56.82.223" "tcp" "14" "28122" "" "" 1408 # 1409 # Example of desired output: 1410 # 213.56.82.222 - RuleNr [28/May/2002:9:47:09 +0000] "GET /213.56.82.223/tcp/Http-8080 HTTP/1.0" 200 \ 1411 # 0 "http://28122/" "14" 0 E100B2 1412 # 1413 # Which means: 1414 1415 # ip - - [date] "GET Desthost/Protocol/Port" 200 PcktLen "http://Sourceport/" "rule" 0 interface 1416 # Sourceport is in the referrer field, rule in the user-agent, interface 1417 # in the VirtualHost. 1418 # There is no MAC address or packet length in FireWall-1 logs. 1419 1420 # Decide if the source or the destination host is included in the 1421 # Blocked Packet Report (option "onehost" in fwanalog.opts) 1422 if [ $onehost = true ]; then 1423 reqhost="\$12" # The analog "request" contains the source ip 1424 elif [ $onehost = dynip ]; then 1425 reqhost="firewall" # The analog "request" contains this string 1426 else 1427 reqhost="\$13" # The analog "request" contains the destination ip 1428 fi 1429 1430 $perl -pwe \ 1431 's! 1432 ^ # Begin of line :-) 1433 \"([0-9]+)\" \s+ # "Number": ($1) 1434 \"([0-9\s]+)([a-zA-Z]+)([0-9]+)\" \s+ # "Date": day($2) 1435 # month($3) year($4) 1436 \"([0-9:]+)\" \s+ # "Time": time($5) 1437 \"([a-zA-Z0-9 &-]+)\" \s+ # "Product": ($6) 1438 \"([a-zA-Z0-9-]+)\" \s+ # "Interface": ($7) 1439 \"([a-zA-Z0-9-]+)\" \s+ # "Origin": ($8) 1440 \"([a-zA-Z0-9]+)\" \s+ # "Type": ($9) 1441 \"([a-zA-Z0-9]+)\" \s+ # "Action": ($10) 1442 \"([a-zA-Z0-9_.\-]*)\" \s+ # "Service": destination port($11) - may be name or number or null 1443 \"([a-zA-Z0-9_.\-]*)\" \s+ # "Source": ($12) - may be name or number or null 1444 \"([a-zA-Z0-9_.\-]+)\" \s+ # "Destination": ($13) - may be name or number 1445 \"([a-zA-Z0-9]*)\" \s+ # "Protocol": ($14) - may be name or number or null 1446 \"([0-9]*)\" \s+ # "Rule": ($15) 1447 \"([a-zA-Z0-9_.\-]*)\" \s+ # "Source port": ($16) - may be name or number or null 1448 \"([a-zA-Z0-9]*)\" \s+ # "User": ($17) may be name or number or null 1449 .+ # ignore the rest 1450 $ # End of line :-) 1451 !$12 - $15 [$2/$3/$4:$5 '$timezone'] \"GET /'$reqhost'/$14/$11/ HTTP/1.0" 200 0 "http://$16/" "$15" 0 $7!x' \ 1452 $outdir/fwanalog.current.withyear > $outdir/fwanalog.current.log 1453 1454 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 1455} 1456 1457sonicwall () 1458{ 1459 # Parse SonicWall TZ-170 syslog logfiles into an analog-compatible 1460 # "URL log" 1461 1462 $zegrep -h " connection dropped" $inputfiles \ 1463 > $outdir/fwanalog.current 1464 1465 mkdateconvscript 1466 # Create script to convert lines without year to fully specified date 1467 1468 $sed -f $outdir/convdate.sed $outdir/fwanalog.current > $outdir/fwanalog.current.withyear 1469 # Use the script on the current logfile 1470 1471 # Example of converted log line: 1472 # 2004 Dec 9 14:50:38 192.168.1.1 id=firewall sn=0123456789AB \ 1473 # time="2004-12-09 15:07:38" fw=400.300.200.100 pri=5 c=64 m=36 \ 1474 # msg="TCP connection dropped" n=38533 src=66.167.62.131:2930:WAN \ 1475 # dst=400.300.200.100:135:WAN 1476 1477 # Example of desired output: 1478 # 66.167.62.131 - - [9/Dec/2004:14:50:38 -0500] \ 1479 # "GET /400.300.200.100/TCP/135 HTTP/1.0" 200 36 \ 1480 # "http://2930/" "-" 0 WAN 1481 # 1482 # Which means: 1483 # ip - - [date] \ 1484 # "GET Desthost/Protocol/Port HTTP/1.0" 200 PcktLen \ 1485 # "http://Sourceport/" "Mac" 0 interface 1486 1487 # Sourceport is in the referrer field, macadr in the user-agent, 1488 # interface in the VirtualHost. 1489 1490 # Decide if the source or the destination host is included in the 1491 # Blocked Packet Report (option "onehost" in fwanalog.opts) 1492 if [ $onehost = true ]; then 1493 reqhost="\$7" # The analog "request" contains the source ip 1494 elif [ $onehost = dynip ]; then 1495 reqhost="firewall" # The analog "request" contains this string 1496 else 1497 reqhost="\$10" # The analog "request" contains the destination ip 1498 fi 1499 1500 $perl -pwe 's!^ 1501 (\d+) \s+ # 1 year 1502 (\w+) \s+ # 2 month 1503 (\d+) \s+ # 3 day 1504 ([0-9:]+) \s+ # 4 time 1505 [0-9.]+ \s+ # skip IP-address 1506 \w+=\w+ \s+ # skip id=firewall 1507 \w+=\w+ \s+ # skip sn=serial-number 1508 \w+=\"[0-9-]+ \s+ # skip time="yyyy-mm-dd 1509 [0-9:]+\" \s+ # hh:mm:ss" 1510 \w+=[0-9.]+ \s+ # skip fw=IP-address 1511 \w+=\d+ \s+ # skip pri=num 1512 \w+=\d+ \s+ # skip c=num 1513 \w+= # skip m= 1514 (\d+) \s+ # 5 packet-length 1515 \w+=\" # skip msg=" 1516 (\w+) \s+ # 6 protocol 1517 \w+\s+\w+\" \s+ # skip connection dropped" 1518 \w+=\d+ \s+ # skip n=num 1519 \w+= # skip src= 1520 ([0-9.]+)\: # 7 source-IP: 1521 (\d+)\: # 8 source-port: 1522 (\w+) \s+ # 9 source-interface 1523 \w+= # skip dst= 1524 ([0-9.]+)\: # 10 dest-IP: 1525 (\d+)\: # 11 dest-port: 1526 \w+ # skip dest-interface 1527 .+ # skip the-rest 1528 !$7 - - [$3/$2/$1:$4 '$timezone'] \"GET /'$reqhost'/$6/$11/ HTTP/1.0\" 200 $5 \"http://$8/\" \"-\" 0 $9!x' \ 1529 $outdir/fwanalog.current.withyear > $outdir/fwanalog.current.log 1530 1531 # $outdir/fwanalog.current.log now contains the data in the Analog URL format. 1532} 1533 1534mkdateconvscript () 1535{ 1536 # Creates a sed script in the output dir which converts the firewall logs 1537 # (that don't have the year specified) to the real year (if your logs 1538 # aren't too old) 1539 1540 currmo=`$date +%m` 1541 curryear=`$date +%Y` 1542 lastyear=`echo $curryear | awk '{ print($1 - 1) }'` 1543 1544 ( 1545 if [ $currmo -ge 1 ]; then echo "s/^Jan/$curryear Jan/"; else echo "s/^Jan/$lastyear Jan/"; fi 1546 if [ $currmo -ge 2 ]; then echo "s/^Feb/$curryear Feb/"; else echo "s/^Feb/$lastyear Feb/"; fi 1547 if [ $currmo -ge 3 ]; then echo "s/^Mar/$curryear Mar/"; else echo "s/^Mar/$lastyear Mar/"; fi 1548 if [ $currmo -ge 4 ]; then echo "s/^Apr/$curryear Apr/"; else echo "s/^Apr/$lastyear Apr/"; fi 1549 if [ $currmo -ge 5 ]; then echo "s/^May/$curryear May/"; else echo "s/^May/$lastyear May/"; fi 1550 if [ $currmo -ge 6 ]; then echo "s/^Jun/$curryear Jun/"; else echo "s/^Jun/$lastyear Jun/"; fi 1551 if [ $currmo -ge 7 ]; then echo "s/^Jul/$curryear Jul/"; else echo "s/^Jul/$lastyear Jul/"; fi 1552 if [ $currmo -ge 8 ]; then echo "s/^Aug/$curryear Aug/"; else echo "s/^Aug/$lastyear Aug/"; fi 1553 if [ $currmo -ge 9 ]; then echo "s/^Sep/$curryear Sep/"; else echo "s/^Sep/$lastyear Sep/"; fi 1554 if [ $currmo -ge 10 ]; then echo "s/^Oct/$curryear Oct/"; else echo "s/^Oct/$lastyear Oct/"; fi 1555 if [ $currmo -ge 11 ]; then echo "s/^Nov/$curryear Nov/"; else echo "s/^Nov/$lastyear Nov/"; fi 1556 if [ $currmo -ge 12 ]; then echo "s/^Dec/$curryear Dec/"; else echo "s/^Dec/$lastyear Dec/"; fi 1557 ) > $outdir/convdate.sed 1558} 1559 1560mkmonthconvscript () 1561{ 1562 # Creates a sed script in the output dir which converts the firewall logs 1563 # (that have the month specified numerically) to the month's abbreviation (Jan...Dec) 1564 1565 ( 1566 echo "s!/01/!/Jan/!" 1567 echo "s!/02/!/Feb/!" 1568 echo "s!/03/!/Mar/!" 1569 echo "s!/04/!/Apr/!" 1570 echo "s!/05/!/May/!" 1571 echo "s!/06/!/Jun/!" 1572 echo "s!/07/!/Jul/!" 1573 echo "s!/08/!/Aug/!" 1574 echo "s!/09/!/Sep/!" 1575 echo "s!/10/!/Oct/!" 1576 echo "s!/11/!/Nov/!" 1577 echo "s!/12/!/Dec/!" 1578 ) > $outdir/convdate.sed 1579} 1580 1581rotate_cache () 1582{ 1583 # Greps all entries not from the current month from $outdir/fwanalog.all.log 1584 # to another file. This is good because if fwanalog.all.log is smaller, it 1585 # can be diffed faster. However, this is entirely optional. 1586 1587 echo "Note: rotating is not necessary anymore!" 1588 1589 # change into /usr/local/etc/fwanalog 1590 cd /usr/local/etc/fwanalog 1591 1592 # Load the user-settable options from the config file 1593 . `basename $0 | $sed 's/$/.opts/'` 1594 1595 # Month and year as they appear in the web server log 1596 grepdate=`$date +/%b/%Y:` 1597 # Name to indicate that this file is older 1598 newlogname=fwanalog.all.log.`$date +%Y-%m` 1599 1600 $grep -vh $grepdate $outdir/fwanalog.all.log > $outdir/$newlogname 1601 echo "$grep -vh $grepdate $outdir/fwanalog.all.log > $outdir/$newlogname" 1602 $grep -h $grepdate $outdir/fwanalog.all.log > $outdir/fwanalog.all.log.current 1603 $echo "$grep -h $grepdate $outdir/fwanalog.all.log > $outdir/fwanalog.all.log.current" 1604 1605 rm $outdir/fwanalog.all.log 1606 echo "rm $outdir/fwanalog.all.log" 1607 mv $outdir/fwanalog.all.log.current $outdir/fwanalog.all.log 1608 echo "mv $outdir/fwanalog.all.log.current $outdir/fwanalog.all.log" 1609} 1610 1611clean_up () 1612{ 1613 1614 ##### 1615 # Function to remove temporary files and other housekeeping 1616 # No arguments 1617 ##### 1618 1619 # Delete the lock file 1620 if [ -e $outdir/fwanalog.lock ]; then 1621 rm -f $outdir/fwanalog.lock 1622 fi 1623} 1624 1625 1626graceful_exit () 1627{ 1628 ##### 1629 # Function called for a graceful exit 1630 # No arguments 1631 ##### 1632 1633 clean_up 1634 exit 1635} 1636 1637 1638error_exit () 1639{ 1640 ##### 1641 # Function for exit due to fatal program error 1642 # Accepts 1 argument 1643 # string containing descriptive error message 1644 ##### 1645 1646 1647 echo "${PROGNAME}: ${1:-"Unknown Error"}" >&2 1648 clean_up 1649 exit 1 1650} 1651 1652 1653term_exit () 1654{ 1655 ##### 1656 # Function to perform exit if termination signal is trapped 1657 # No arguments 1658 ##### 1659 1660 echo "${PROGNAME}: Terminated" 1661 clean_up 1662 exit 1663} 1664 1665 1666int_exit () 1667{ 1668 ##### 1669 # Function to perform exit if interrupt signal is trapped 1670 # No arguments 1671 ##### 1672 1673 echo "${PROGNAME}: Aborted by user" 1674 clean_up 1675 exit 1676} 1677 1678 1679usage () 1680{ 1681 ##### 1682 # Function to display usage message (does not exit) 1683 # No arguments 1684 ##### 1685 1686 echo "Usage: ${PROGNAME} [-h | --help] [-c conffile] [-r] [-t] [-y] [-a IP-addr] [-p packet]" 1687} 1688 1689 1690helptext () 1691{ 1692 ##### 1693 # Function to display help message for program 1694 # No arguments 1695 ##### 1696 1697 local tab=$(echo -en "\t\t") 1698 1699 cat <<- -EOF- 1700 1701 ${PROGNAME} ver. ${VERSION} 1702 This is a program to parse firewall logs and analyze them with Analog. 1703 1704 $(usage) 1705 1706 Options: 1707 1708 -h, --help Display this help message and exit. 1709 -c conffile Use this config file instead of fwanalog.opts 1710 -r Rotate log cache (not necessary anymore) 1711 -t Only update statistics for today (e.g. for hourly use) 1712 The sep_hosts and sep_packets commands in fwanalog.opts 1713 are ignored. 1714 -y The same as -t, only for yesterday 1715 -a IP-addr Create a separate report for this host 1716 -p packet Create a separate report for this packet 1717 Format: target/protocol/portnumber 1718 e.g. 192.168.0.1/tcp/21 or firewall/udp/137 1719 1720 NOTE: You must be the superuser to run this script, or have at least 1721 read rights to the firewall log. (See README.sudo for how to 1722 do this as a normal user.) 1723-EOF- 1724} 1725 1726 1727########################################################################### 1728# Program starts here 1729########################################################################### 1730 1731# Trap TERM, HUP, and INT signals and properly exit 1732 1733trap term_exit TERM HUP 1734trap int_exit INT 1735 1736# Process command line arguments 1737 1738if [ "$1" = "--help" ]; then 1739 helptext 1740 graceful_exit 1741fi 1742 1743# Process arguments - edit to taste 1744 1745while getopts ":hrtymc:a:p:" opt; do 1746 case $opt in 1747 r ) rotate_cache 1748 graceful_exit ;; 1749 1750 h ) helptext 1751 graceful_exit ;; 1752 1753 t ) today_only=true ;; 1754 1755 y ) yesterday_only=true ;; 1756 1757 m ) reportmagic=true ;; 1758 1759 a ) host_to_report="$OPTARG" ;; 1760 1761 p ) packet_to_report="$OPTARG" ;; 1762 1763 c ) configfile="$OPTARG" ;; 1764 1765 * ) usage 1766 exit 1 1767 esac 1768done 1769 1770# No arguments - normal case 1771if [ $OPTIND -gt $# ]; then 1772 (main) 1773fi 1774 1775graceful_exit 1776