1#!/usr/bin/perl
2
3use strict;
4use warnings;
5
6use Getopt::Long ('GetOptions');
7use Data::Dumper ();
8use File::Basename ('dirname');
9
10our $InDir = '/var/lib/collectd';
11our $OutDir = '/tmp/collectd-4';
12our $Hostname = 'localhost';
13
14# Types:
15# +------------+----------------------+----+----+----+
16# ! Subdir     ! Type                 ! ti ! pi ! ex !
17# +------------+----------------------+----+----+----+
18# ! apache     ! apache_bytes         !    !    !    !
19# ! apache     ! apache_requests      !    !    !    !
20# ! apache     ! apache_scoreboard    ! x  !    !    !
21# ! battery    ! charge               !    ! x  !    !
22# ! apcups     ! charge_percent       !    !    !    !
23# !            ! cpu                  ! x  !    ! x  !
24# !            ! cpufreq              ! x  !    !    !
25# ! battery    ! current              !    ! x  !    !
26# ! ntpd       ! delay                ! x  !    !    !
27# !            ! df                   ! x  !    !    !
28# !            ! disk                 ! x  !    !    !
29# ! dns        ! dns_traffic          !    !    !    !
30# ! apple_se.. ! fanspeed             ! x  !    !    !
31# ! mbmon      ! fanspeed             ! x  !    !    !
32# ! apcups     ! frequency            ! x  !    !    !
33# ! ntpd       ! frequency_offset     ! x  !    !    !
34# !            ! hddtemp              ! x  !    !    !
35# ! interface  ! if_errors            !    ! x  !    !
36# ! interface  ! if_packets           !    ! x  !    !
37# !            ! lm_sensors           !    !    !    !
38# !            ! load                 !    !    !    !
39# ! apcups     ! load_percent         !    !    !    !
40# !            ! memory               !    !    !    !
41# !            ! multimeter           !    !    !    !
42# ! mysql      ! mysql_commands       ! x  !    !    !
43# ! mysql      ! mysql_handler        ! x  !    !    !
44# ! mysql      ! mysql_qcache         !    !    !    !
45# ! mysql      ! mysql_threads        !    !    !    !
46# !            ! nfs2_procedures      ! x  !    ! x  !
47# !            ! nfs3_procedures      ! x  !    ! x  !
48# ! dns        ! opcode               ! x  !    !    !
49# !            ! partition            ! x  !    !    !
50# !            ! ping                 ! x  !    !    !
51# !            ! processes            !    !    !    !
52# ! processes  ! ps_count             ! x  !    !    !
53# ! processes  ! ps_cputime           ! x  !    !    !
54# ! processes  ! ps_pagefaults        ! x  !    !    !
55# ! processes  ! ps_rss               ! x  !    !    !
56# ! dns        ! qtype                ! x  !    !    !
57# ! dns        ! rcode                ! x  !    !    !
58# ! (*)        ! sensors              ! x  !    !    !
59# !            ! serial               ! x  !    !    !
60# !            ! swap                 !    !    !    !
61# !            ! tape                 ! x  !    !    !
62# ! apple_se.. ! temperature          ! x  !    !    !
63# ! mbmon      ! temperature          ! x  !    !    !
64# ! ntpd       ! time_dispersion      ! x  !    !    !
65# ! ntpd       ! time_offset          ! x  !    !    !
66# ! apcups     ! timeleft             !    !    !    !
67# !            ! traffic              ! x  !    !    ! ->rx,tx
68# ! vserver    ! traffic              ! x  ! x  !    ! ->rx.tx
69# !            ! users                !    !    !    !
70# ! apucups    ! voltage              ! x  !    !    !
71# ! battery    ! voltage              !    ! x  !    !
72# ! mbmon      ! voltage              ! x  !    !    !
73# ! vserver    ! vs_memory            !    ! x  !    !
74# ! vserver    ! vs_processes         !    ! x  !    !
75# ! vserver    ! vs_threads           !    ! x  !    !
76# !            ! wireless             ! x  !    !    !
77# +------------+----------------------+----+----+----+
78
79our %Subdirs =
80(
81	apache => 0,
82	apcups => 0,
83	apple_sensors => 0,
84	battery => 1,
85	dns => 0,
86	interface => 1,
87	mbmon => 0,
88	mysql => 0,
89	ntpd => 0,
90	processes => 0,
91	sensors => 1,
92	vserver => 1
93);
94
95our %TypeTranslate =
96(
97	cpu => sub { $_ = shift; $_->{'plugin_instance'} = $_->{'type_instance'}; $_->{'type_instance'} = undef; $_; },
98	hddtemp => sub { $_ = shift; $_->{'plugin'} = 'hddtemp'; $_->{'type'} = 'temperature'; $_->{'type_instance'} = $_->{'type_instance'}; $_; },
99	if_errors => sub { $_ = shift; $_->{'type_instance'} = $_->{'plugin_instance'}; $_->{'plugin_instance'} = undef; $_; },
100	if_packets => sub { $_ = shift; $_->{'type_instance'} = $_->{'plugin_instance'}; $_->{'plugin_instance'} = undef; $_; },
101	nfs2_procedures => sub { $_ = shift; @$_{qw(plugin plugin_instance type type_instance)} = ('nfs', 'v2' . $_->{'type_instance'}, 'nfs_procedure', undef); $_; },
102	nfs3_procedures => sub { $_ = shift; @$_{qw(plugin plugin_instance type type_instance)} = ('nfs', 'v3' . $_->{'type_instance'}, 'nfs_procedure', undef); $_; },
103	partition => sub { $_ = shift; $_->{'plugin'} = 'disk'; $_; },
104	processes => sub { $_ = shift; $_->{'type'} = 'ps_state'; $_; },
105	traffic => sub { $_ = shift; $_->{'plugin'} =~ s/^traffic$/interface/; @$_{qw(plugin_instance type)} = (undef, 'if_octets'); $_; }
106);
107
108our %TypeSplit =
109(
110	cpu => { from => [qw(user nice syst idle wait)], to => 'value', type_instance => [qw(user nice system idle wait)] },
111	memory => { from => [qw(used free buffers cached)], to => 'value', type_instance => [qw(used free buffered cached)] },
112	nfs3_procedures => { from => [qw(null getattr lookup access readlink
113		read write create mkdir symlink mknod remove rmdir rename link
114		readdir readdirplus fsstat fsinfo pathconf commit)], to => 'value' },
115	nfs2_procedures => { from => [qw(create fsstat getattr link lookup
116		mkdir null read readdir readlink remove rename rmdir root
117		setattr symlink wrcache write)], to => 'value' },
118	processes => { from => [qw(running sleeping zombies stopped paging blocked)], to => 'value' },
119	swap => { from => [qw(cached free used resv)], to => 'value', type_instance => [qw(cached free used reserved)] }
120);
121
122our %TypeRename =
123(
124	traffic => { from => [qw(incoming outgoing)], to => [qw(rx tx)] },
125	vs_processes => { from => [qw(total)], to => [qw(value)] },
126);
127
128GetOptions ("indir|i=s" => \$InDir,
129	"outdir|o=s" => \$OutDir,
130	"hostname=s" => \$Hostname) or exit_usage ();
131
132die "No such directory: $InDir" if (!-d $InDir);
133
134our @Files = ();
135our %OutDirs = ();
136
137@Files = find_files ();
138
139for (@Files)
140{
141	my $orig_filename = $_;
142	my $orig = parse_file ($orig_filename);
143	my $dest = translate_file ($orig);
144	my $dest_filename = get_filename ($dest);
145
146	my $dest_directory = dirname ($dest_filename);
147	if (!exists ($OutDirs{$dest_directory}))
148	{
149		print "[ -d '$OutDir/$dest_directory' ] || mkdir -p '$OutDir/$dest_directory'\n";
150		$OutDirs{$dest_directory} = 1;
151	}
152
153	if (($orig->{'type'} eq 'disk') || ($orig->{'type'} eq 'partition'))
154	{
155		special_disk ($orig_filename, $orig, $dest_filename, $dest);
156	}
157	elsif (exists ($TypeSplit{$orig->{'type'}}))
158	{
159		my $src_dses = $TypeSplit{$orig->{'type'}}->{'from'};
160		my $dst_ds = $TypeSplit{$orig->{'type'}}->{'to'};
161		my $type_instances = exists ($TypeSplit{$orig->{'type'}}->{'type_instance'})
162			? $TypeSplit{$orig->{'type'}}->{'type_instance'}
163			: $TypeSplit{$orig->{'type'}}->{'from'};
164
165		for (my $i = 0; $i < @$src_dses; $i++)
166		{
167			my $src_ds = $src_dses->[$i];
168			$dest->{'type_instance'} = $type_instances->[$i];
169			$dest_filename = get_filename ($dest);
170			print "./rrd_filter.px -i '$InDir/$orig_filename' -m '${src_ds}:${dst_ds}' -o '$OutDir/$dest_filename'\n";
171		}
172	}
173	else
174	{
175		print "cp '$InDir/$orig_filename' '$OutDir/$dest_filename'\n";
176	}
177
178	if (exists ($TypeRename{$orig->{'type'}}))
179	{
180		my $src_dses = $TypeRename{$orig->{'type'}}->{'from'};
181		my $dst_dses = $TypeRename{$orig->{'type'}}->{'to'};
182
183		print "rrdtool tune '$OutDir/$dest_filename'";
184		for (my $i = 0; $i < @$src_dses; $i++)
185		{
186			print " --data-source-rename "
187				. $src_dses->[$i] . ':' . $dst_dses->[$i];
188		}
189		print "\n";
190	}
191}
192
193exit (0);
194
195sub translate_file
196{
197	my $orig = shift;
198	my $dest = {};
199	%$dest = %$orig;
200
201	if (defined ($TypeTranslate{$orig->{'type'}}))
202	{
203		$TypeTranslate{$orig->{'type'}}->($dest);
204	}
205
206	return ($dest);
207} # translate_file
208
209sub get_filename
210{
211	my $args = shift;
212	my $filename = $args->{'host'}
213	. '/' . $args->{'plugin'} . (defined ($args->{'plugin_instance'}) ? '-'.$args->{'plugin_instance'} : '')
214	. '/' . $args->{'type'} . (defined ($args->{'type_instance'}) ? '-'.$args->{'type_instance'} : '') . '.rrd';
215
216	return ($filename);
217}
218
219sub parse_file
220{
221	my $fullname = shift;
222	my @parts = split ('/', $fullname);
223
224	my $filename;
225
226	my $host;
227	my $plugin;
228	my $plugin_instance;
229	my $type;
230	my $type_instance;
231
232	$filename = pop (@parts);
233
234	if ($filename =~ m/^([^-]+)(?:-(.*))?\.rrd$/)
235	{
236		$type = $1;
237		$type_instance = $2;
238	}
239	else
240	{
241		return;
242	}
243
244	if (@parts)
245	{
246		my $dirname = pop (@parts);
247		my $regex_str = join ('|', keys (%Subdirs));
248		if ($dirname =~ m/^($regex_str)(?:-(.*))?$/)
249		{
250			$plugin = $1;
251			$plugin_instance = $2;
252		}
253		else
254		{
255			push (@parts, $dirname);
256		}
257	}
258	if (!$plugin)
259	{
260		$plugin = $type;
261	}
262
263	if (@parts)
264	{
265		$host = pop (@parts);
266	}
267	else
268	{
269		$host = $Hostname;
270	}
271
272	return
273	({
274		host => $host,
275		plugin => $plugin,
276		plugin_instance => $plugin_instance,
277		type => $type,
278		type_instance => $type_instance
279	});
280} # parse_file
281
282sub find_files
283{
284	my $reldir = @_ ? shift : '';
285	my $absdir = $InDir . ($reldir ? "/$reldir" : '');
286
287	my $dh;
288
289	my @files = ();
290	my @dirs = ();
291
292	opendir ($dh, $absdir) or die ("opendir ($absdir): $!");
293	while (my $file = readdir ($dh))
294	{
295		next if ($file =~ m/^\./);
296		next if (-l "$absdir/$file");
297		if (-d "$absdir/$file")
298		{
299			push (@dirs, ($reldir ? "$reldir/" : '') . $file);
300		}
301		elsif ($file =~ m/\.rrd$/)
302		{
303			push (@files, ($reldir ? "$reldir/" : '') . $file);
304		}
305	}
306	closedir ($dh);
307
308	for (my $i = 0; $i < @dirs; $i++)
309	{
310		push (@files, find_files ($dirs[$i]));
311	}
312
313	return (@files);
314} # find_files
315
316{my $cache;
317sub _special_disk_instance
318{
319	my $orig_instance = shift;
320
321	if (!defined ($cache))
322	{
323		my $fh;
324		open ($fh, "< /proc/diskstats") or die ("open (/proc/diststats): $!");
325
326		$cache = {};
327		while (my $line = <$fh>)
328		{
329			chomp ($line);
330			my @fields = split (' ', $line);
331			$cache->{$fields[0] . '-' . $fields[1]} = $fields[2];
332		}
333		close ($fh);
334	}
335
336	return (defined ($cache->{$orig_instance})
337		? $cache->{$orig_instance}
338		: $orig_instance);
339}}
340
341sub special_disk
342{
343	my $orig_filename = shift;
344	my $orig = shift;
345	my $dest_filename = shift;
346	my $dest = shift;
347	my $dest_directory;
348
349	$dest->{'type_instance'} = undef;
350	$dest->{'plugin_instance'} = _special_disk_instance ($orig->{'type_instance'});
351	if ($dest->{'plugin_instance'} eq $orig->{'type_instance'})
352	{
353		print qq(echo "You may need to rename these files" >&2\n);
354	}
355
356	$dest->{'type'} = 'disk_merged';
357	$dest_filename = get_filename ($dest);
358
359	$dest_directory = dirname ($dest_filename);
360	if (!exists ($OutDirs{$dest_directory}))
361	{
362		print "[ -d '$OutDir/$dest_directory' ] || mkdir -p '$OutDir/$dest_directory'\n";
363		$OutDirs{$dest_directory} = 1;
364	}
365
366	print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rmerged:read' -m 'wmerged:write' -o '$OutDir/$dest_filename'\n";
367
368	$dest->{'type'} = 'disk_octets';
369	$dest_filename = get_filename ($dest);
370	print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rbytes:read' -m 'wbytes:write' -o '$OutDir/$dest_filename'\n";
371
372	$dest->{'type'} = 'disk_ops';
373	$dest_filename = get_filename ($dest);
374	print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rcount:read' -m 'wcount:write' -o '$OutDir/$dest_filename'\n";
375
376	$dest->{'type'} = 'disk_time';
377	$dest_filename = get_filename ($dest);
378	print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rtime:read' -m 'wtime:write' -o '$OutDir/$dest_filename'\n";
379}
380
381sub exit_usage
382{
383	print <<EOF;
384Usage: $0 [-i indir] [-o outdir] [--hostname myhostname]
385EOF
386	exit (1);
387}
388