1#!/usr/local/bin/perl
2#
3# Copyright (c) 2006 by Raffael Marty
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program; if not, write to the Free Software
17# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18#
19# Title: 	TCPdump 2 CSV
20#
21# File: 	tcpdump2csv.pl
22#
23# Version: 	1.5
24#
25# Description:	Takes a tcpdump pcap file and parses it into a csv output.
26#
27# Usage:	tcpdump -vttttnnelr /tmp/log.tcpdump | tcpdump2csv.pl ["field list"]
28#
29# Running in conjunction with afterglow:
30# 		tcpdump -vttttnnelr /tmp/log.tcpdump | tcpdump2csv.pl "sip dip sport" |
31# 		perl ../graph/afterglow.pl | neato -Tgif -o test.gif
32#
33# Possible fields:
34# 		timestamp  dip  sip  ttl  tos  id  offset  flags  len
35# 		sourcemac  destmac  ipflags  sport  dport
36#
37# Known Issues:
38# 		Does not parse ARP packets
39#		Does not parse SAP packets
40#		Does not parse multi-line DNS packets
41#
42#		ATTENTION: SHOULD work with tcpdump 3.8.x
43#		ATTENTION: ONLY works with tcpdump 3.9.x
44#
45# URL:		http://afterglow.sourceforge.net
46#
47# Changes:
48#
49# 06/13/05	Initial Version by ram
50# 06/25/05	ram's birthday: taking care of source and target swapping
51# 		Server: receives a SYN or sends a SYN ACK
52# 		If no SYN or SYN ACK seen for a connection, assume the machine with
53# 		port < 1024 is the server
54# 12/17/05	Fixing error handling. Should not exit when an unknown packet arrives
55# 		Also introducing the $DEBUG variable
56# 05/24/06	Version 1.5:
57# 		Changing the parsing to support tcpdump 3.9.4. Rewrote parsing
58# 		part to be a bit more sane ;)
59#
60###############################################################################/
61
62use strict vars;
63
64
65
66my $output=$ARGV[0] || "full";
67
68my $DEBUG=0;
69
70our ($timestamp,$etherproto,$dip,$sip,$ttl,$tos,$id,$offset,$flags,$len,$sourcemac,$destmac,$ipflags,$sport,$dport,$proto,$rest, $dnshostresponse, $dnslookup, $dnsipresponse, $dnstype, $dnslookup);
71
72our %clientServerConn;
73
74while (<STDIN>) {
75	chomp;
76
77	# tcpdump 3.9.4 : 2006-05-14 11:05:58.683193 00:05:4e:44:b7:25 > 00:01:4e:00:b6:59, ethertype IPv4 (0x0800), length 54: (tos 0x0, ttl  64, id 6321, offset 0, flags [DF], proto: TCP (6), length: 40) 10.69.69.13.1555 > 10.69.69.20.52912: R, cksum 0x6c1c (correct), 0:0(0) ack 368294482 win 0
78	# 2006-05-23 08:15:39.809918 00:05:4e:44:b7:25 > 00:50:59:85:1b:60, ethertype IPv4 (0x0800), length 85: (tos 0x0, ttl  64, id 49494, offset 0, flags [DF], proto: UDP (17), length: 71) 10.149.3.174.35837 > 206.13.29.12.53:  19302+ A? 02.presence.userplane.com. (43)
79	# 2006-05-15 xxx 00:05:4e:44:b7:25 > 00:50:59:85:1b:60, ethertype IPv4 (0x0800), length 80: (tos 0x0, ttl  64, id 38402, offset 0, flags [DF], proto: UDP (17), length: 66) 10.149.3.174.35843 > 206.13.29.12.53:  25351+ AAAA? webmail.arcsight.com. (38)
80
81	# 2006-05-15 xxx 00:50:59:85:1b:60 > 00:05:4e:44:b7:25, ethertype IPv4 (0x0800), length 101: (tos 0x60, ttl 123, id 16394, offset 0, flags [none], proto: UDP (17), length: 87) 206.13.29.12.53 > 10.149.3.174.35843:  29873 1/0/0 02.presence.userplane.com. A 8.3.208.204 (59)
82
83		if (/(\d+-\d+-\d+ \d+:\d+:\d+\.\d+) (\S+) > (\S+), ethertype (\S+) \(\S+\), length:? (\d+):? (?:\S+ )?\((?:tos +(\S+), )?(?:ttl +(\d+), )?(?:id +(\d+), )?(?:offset +(\d+), )?(?:flags \[(\S+)\], )?(?:proto: (\S+).*?, )?(?:length: (\d+))?.*?\) (\S+?)(?:\.(\d+))? > (\S+?)(?:\.(\d+))?: +(?:(\S+),? (.*?)|\d+[\+\*\-]* \d+\/\d+\/\d+ (\S+) (\S+) (\S+) .*?|\d+[\+\*\-]* (\S+) (\S+) .*?)?/ )  {
84
85		$timestamp = $1 || "";
86		$sourcemac = $2 || "";
87		$destmac = $3 || "";
88		$etherproto = $4 || "";
89		$len = $5 || "";
90		$tos = $6 || "";
91		$ttl = $7 || "";
92		$id = $8 || "";
93		$offset = $9 || "";
94		$ipflags = $10 || "";
95		$proto = $11 || "";
96		$len = $12 || "";
97		$sip = $13 || "";
98		$sport = $14 || "";
99		$dip = $15 || "";
100		$dport = $16 || "";
101		$flags = $17 || "";
102		$rest = $18 || "";
103		$dnshostresponse = $20 || "";
104		$dnslookup = $21 || "";
105		$dnsipresponse = $22 || "";
106		$dnstype = $23 || "";
107		$dnslookup = $24 || $dnslookup;
108
109		# skip 802.3 packets:
110		next if ($etherproto eq "802.3,");
111		# skip ARP
112		next if ($etherproto eq "ARP");
113
114		$timestamp =~ s/(.*?)\.\d+$/\1/;
115		$sourcemac =~ s/,$//;
116		$destmac =~ s/,$//;
117		$len =~ s/:$//;
118
119	} else {
120
121		$DEBUG && print STDERR "ERROR: $_\n";
122		next;
123
124	}
125
126	my @fields = split (" ",$_);
127
128	# adding ACK flag as an "A"
129	if ($_ =~ / ack /) { $flags .= "A"; }
130
131	my $connId = $sip.$dip.$sport.$dport;
132	my $reverseConnId = $dip.$sip.$dport.$sport;
133
134	# trying to find the client and the server and the act opon that
135	if ($flags =~ /S.*A/) {		# server to client
136
137		$clientServerConn{$reverseConnId}="1";
138		# swap source and dest:
139		($sourcemac,$destmac) = ($destmac,$sourcemac);
140		($sip,$dip) = ($dip,$sip);
141		($sport,$dport) = ($dport,$sport);
142
143	} elsif ($flags =~ /S/) {	# client to server
144
145		$clientServerConn{$connId}="1";
146
147	} elsif ($clientServerConn{$reverseConnId}) {
148
149		# swap source and dest:
150		($sourcemac,$destmac) = ($destmac,$sourcemac);
151		($sip,$dip) = ($dip,$sip);
152		($sport,$dport) = ($dport,$sport);
153
154	} elsif ((!$clientServerConn{$reverseConnId}) && (!$clientServerConn{$connId}) && ($proto eq "tcp")) {
155
156		# we never saw a SYN or a SYN ACK and we are in TCP, let us try the port numbers
157		# This is better than not doing it :)
158
159		if (($sport < 1024) && ($dport > 1024)) {
160			$clientServerConn{$reverseConnId}="1";
161			# swap source and dest:
162			($sourcemac,$destmac) = ($destmac,$sourcemac);
163			($sip,$dip) = ($dip,$sip);
164			($sport,$dport) = ($dport,$sport);
165		}
166
167	}
168
169	if ($output eq "full") {
170		print "$timestamp $sourcemac $destmac $sip $dip $sport $dport $flags $len $proto $ttl $id $offset $tos $ipflags\n";
171	} else {
172		my @tokens = split / /,$output;
173		print ${shift(@tokens)};
174		for my $token (@tokens) {
175			if (!defined($$token)) {
176				$DEBUG && print STDERR "$token is not a known field\n";
177				#exit;
178			} else {
179				print ','.$$token;
180			}
181		}
182		print "\n";
183	}
184
185}
186
187# To verify:
188# tcpdump 3.8.x : 2002-08-24 05:34:18.634488 00:00:0c:04:b2:33 > 00:03:e3:d9:26:c0, ethertype IPv4 (0x0800), length 223: IP (tos 0x0, ttl 122, id 544, offset 0, flags [DF], length: 209, bad cksum ff6b (->14ac)!) 138.97.18.88.61924 > 64.4.12.158.1863: P [bad tcp cksum 6c66 (->aca6)!] 9384:9553(169) ack 9641 win 17390
189# tcpdump 3.8.x : 2002-08-24 10:52:42.184488 00:03:e3:d9:26:c0 > 00:00:0c:04:b2:33, ethertype IPv4 (0x0800), length 60: IP (tos 0x0, ttl 232, id 0, offset 8896, flags [+, DF], length: 40, bad cksum ff99 (->b4d9)!) 192.9.100.88 > 138.97.10.219: tcp
190
191# 2006-05-23 08:15:39.809918 00:50:59:85:1b:60 > 00:12:f0:b1:c7:c6, ethertype ARP (0x0806), length 64: arp reply 10.149.0.1 is-at 00:50:59:85:1b:60
192# 2006-05-23 08:54:58.278518 00:50:59:85:1b:60 > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 64: arp who-has 10.149.3.187 tell 10.149.3.250
193	# 2006-05-23 08:15:35.985245 00:50:59:85:1b:60 > 00:40:96:a3:5f:58, ethertype IPv4 (0x0800), length 110: (tos 0x60, ttl  49, id 14118, offset 0, flags [none], proto: UDP (17), length: 96) 216.191.40.60.500 > 10.149.3.139.3210: isakmp 1.0 msgid : phase 2/others ? inf[E]: [encrypted hash]
194
195
196
197# To be done: (need to get the 3.9.x output for these!
198# 2005-01-12 14:38:20.660616 00:0d:56:e3:44:33 > 33:33:00:00:00:02, ethertype IPv6 (0x86dd), length 70: fe80::20d:56ff:fee3:4433 > ff02::2: [icmp6 sum ok] icmp6: router solicitation (src lladdr: 00:0d:56:e3:44:33) (len 16, hlim 255)
199# 2005-05-03 18:42:31.274438 00:0d:56:74:c4:d9 > ff:ff:ff:ff:ff:ff, 802.3, length 94: LLC, dsap Global (0xff), ssap Global (0xff), cmd 0x00, (NOV-802.3) 00000000.00:0d:56:74:c4:d9.0455 > 00000000.ff:ff:ff:ff:ff:ff.0455: ipx-netbios 50
200# 2005-09-08 16:38:14.906293 00:14:69:1f:b3:00 > 01:00:0c:cc:cc:cc, 802.3, length 338: LLC, dsap SNAP (0xaa), ssap SNAP (0xaa), cmd 0x03, CDPv2, ttl: 180s, checksum: 692 (unverified), length 316
201# 2005-09-08 16:38:11.013187 00:03:93:ea:dc:2f > 33:33:ff:ea:dc:2f, ethertype IPv6 (0x86dd), length 86: fe80::203:93ff:feea:dc2f > ff02::1:ffea:dc2f: HBH (padn)(rtalert: 0x0000) [icmp6 sum ok] icmp6: multicast listener report max resp delay: 0 addr: ff02::1:ffea:dc2f [hlim 1] (len 32)
202# 2005-09-08 16:38:08.146159 00:03:93:ea:dc:2f > 33:33:00:00:00:02, ethertype IPv6 (0x86dd), length 86: fe80::203:93ff:feea:dc2f > ff02::2: HBH (padn)(rtalert: 0x0000) [icmp6 sum ok] icmp6: multicast listener done max resp delay: 0 addr: ff02::fb [hlim 1] (len 32)
203# 2005-09-08 16:38:05.896611 00:03:93:ea:dc:2f > 33:33:00:00:00:fb, ethertype IPv6 (0x86dd), length 459: fe80::203:93ff:feea:dc2f.5353 > ff02::fb.5353: [udp sum ok]  0 [8q] [8n] ANY? Altair._ftp._tcp.local. ANY? Altair [00:03:93:d5:81:02]._workstation._tcp.local. ANY? Altair._ssh._tcp.local. ANY? Altair._sftp-ssh._tcp.local. ANY? Ari Serim._http._tcp.local. ANY? AriM-bM-^@M-^Ys Beats._daap._tcp.local. ANY? iTunes_Ctrl_AE2BB3BEAAAB7A8B._dacp._tcp.local. ANY? Altair.local. (397) (len 405, hlim 255)
204# 2005-09-08 16:38:05.696393 00:03:93:ea:dc:2f > 33:33:00:00:00:fb, ethertype IPv6 (0x86dd), length 349: fe80::203:93ff:feea:dc2f.5353 > ff02::fb.5353: [udp sum ok]  0*- [0q] 8/0/0 _services._dns-sd._udp.local. PTR _ftp._tcp.local., _services._dns-sd._udp.local. PTR _workstation._tcp.local., _services._dns-sd._udp.local. PTR _ssh._tcp.local., _services._dns-sd._udp.local. PTR _sftp-ssh._tcp.local., _services._dns-sd._udp.local. PTR _http._tcp.local., _services._dns-sd._udp.local. PTR _daap._tcp.local., _services._dns-sd._udp.local. PTR _dacp._tcp.local., F.2.C.D.A.E.E.F.F.F.3.9.3.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.E.F.ip6.arpa. (Cache flush) PTR Altair.local. (287) (len 295, hlim 255)
205
206
207
208