1#!/usr/bin/perl 2 3use strict; 4use warnings; 5 6use Getopt::Long ('GetOptions'); 7use Data::Dumper (); 8use File::Basename ('dirname'); 9 10our $InDir = '/var/lib/collectd'; 11our $OutDir = '/tmp/collectd-4'; 12our $Hostname = 'localhost'; 13 14# Types: 15# +------------+----------------------+----+----+----+ 16# ! Subdir ! Type ! ti ! pi ! ex ! 17# +------------+----------------------+----+----+----+ 18# ! apache ! apache_bytes ! ! ! ! 19# ! apache ! apache_requests ! ! ! ! 20# ! apache ! apache_scoreboard ! x ! ! ! 21# ! battery ! charge ! ! x ! ! 22# ! apcups ! charge_percent ! ! ! ! 23# ! ! cpu ! x ! ! x ! 24# ! ! cpufreq ! x ! ! ! 25# ! battery ! current ! ! x ! ! 26# ! ntpd ! delay ! x ! ! ! 27# ! ! df ! x ! ! ! 28# ! ! disk ! x ! ! ! 29# ! dns ! dns_traffic ! ! ! ! 30# ! apple_se.. ! fanspeed ! x ! ! ! 31# ! mbmon ! fanspeed ! x ! ! ! 32# ! apcups ! frequency ! x ! ! ! 33# ! ntpd ! frequency_offset ! x ! ! ! 34# ! ! hddtemp ! x ! ! ! 35# ! interface ! if_errors ! ! x ! ! 36# ! interface ! if_packets ! ! x ! ! 37# ! ! lm_sensors ! ! ! ! 38# ! ! load ! ! ! ! 39# ! apcups ! load_percent ! ! ! ! 40# ! ! memory ! ! ! ! 41# ! ! multimeter ! ! ! ! 42# ! mysql ! mysql_commands ! x ! ! ! 43# ! mysql ! mysql_handler ! x ! ! ! 44# ! mysql ! mysql_qcache ! ! ! ! 45# ! mysql ! mysql_threads ! ! ! ! 46# ! ! nfs2_procedures ! x ! ! x ! 47# ! ! nfs3_procedures ! x ! ! x ! 48# ! dns ! opcode ! x ! ! ! 49# ! ! partition ! x ! ! ! 50# ! ! ping ! x ! ! ! 51# ! ! processes ! ! ! ! 52# ! processes ! ps_count ! x ! ! ! 53# ! processes ! ps_cputime ! x ! ! ! 54# ! processes ! ps_pagefaults ! x ! ! ! 55# ! processes ! ps_rss ! x ! ! ! 56# ! dns ! qtype ! x ! ! ! 57# ! dns ! rcode ! x ! ! ! 58# ! (*) ! sensors ! x ! ! ! 59# ! ! serial ! x ! ! ! 60# ! ! swap ! ! ! ! 61# ! ! tape ! x ! ! ! 62# ! apple_se.. ! temperature ! x ! ! ! 63# ! mbmon ! temperature ! x ! ! ! 64# ! ntpd ! time_dispersion ! x ! ! ! 65# ! ntpd ! time_offset ! x ! ! ! 66# ! apcups ! timeleft ! ! ! ! 67# ! ! traffic ! x ! ! ! ->rx,tx 68# ! vserver ! traffic ! x ! x ! ! ->rx.tx 69# ! ! users ! ! ! ! 70# ! apucups ! voltage ! x ! ! ! 71# ! battery ! voltage ! ! x ! ! 72# ! mbmon ! voltage ! x ! ! ! 73# ! vserver ! vs_memory ! ! x ! ! 74# ! vserver ! vs_processes ! ! x ! ! 75# ! vserver ! vs_threads ! ! x ! ! 76# ! ! wireless ! x ! ! ! 77# +------------+----------------------+----+----+----+ 78 79our %Subdirs = 80( 81 apache => 0, 82 apcups => 0, 83 apple_sensors => 0, 84 battery => 1, 85 dns => 0, 86 interface => 1, 87 mbmon => 0, 88 mysql => 0, 89 ntpd => 0, 90 processes => 0, 91 sensors => 1, 92 vserver => 1 93); 94 95our %TypeTranslate = 96( 97 cpu => sub { $_ = shift; $_->{'plugin_instance'} = $_->{'type_instance'}; $_->{'type_instance'} = undef; $_; }, 98 hddtemp => sub { $_ = shift; $_->{'plugin'} = 'hddtemp'; $_->{'type'} = 'temperature'; $_->{'type_instance'} = $_->{'type_instance'}; $_; }, 99 if_errors => sub { $_ = shift; $_->{'type_instance'} = $_->{'plugin_instance'}; $_->{'plugin_instance'} = undef; $_; }, 100 if_packets => sub { $_ = shift; $_->{'type_instance'} = $_->{'plugin_instance'}; $_->{'plugin_instance'} = undef; $_; }, 101 nfs2_procedures => sub { $_ = shift; @$_{qw(plugin plugin_instance type type_instance)} = ('nfs', 'v2' . $_->{'type_instance'}, 'nfs_procedure', undef); $_; }, 102 nfs3_procedures => sub { $_ = shift; @$_{qw(plugin plugin_instance type type_instance)} = ('nfs', 'v3' . $_->{'type_instance'}, 'nfs_procedure', undef); $_; }, 103 partition => sub { $_ = shift; $_->{'plugin'} = 'disk'; $_; }, 104 processes => sub { $_ = shift; $_->{'type'} = 'ps_state'; $_; }, 105 traffic => sub { $_ = shift; $_->{'plugin'} =~ s/^traffic$/interface/; @$_{qw(plugin_instance type)} = (undef, 'if_octets'); $_; } 106); 107 108our %TypeSplit = 109( 110 cpu => { from => [qw(user nice syst idle wait)], to => 'value', type_instance => [qw(user nice system idle wait)] }, 111 memory => { from => [qw(used free buffers cached)], to => 'value', type_instance => [qw(used free buffered cached)] }, 112 nfs3_procedures => { from => [qw(null getattr lookup access readlink 113 read write create mkdir symlink mknod remove rmdir rename link 114 readdir readdirplus fsstat fsinfo pathconf commit)], to => 'value' }, 115 nfs2_procedures => { from => [qw(create fsstat getattr link lookup 116 mkdir null read readdir readlink remove rename rmdir root 117 setattr symlink wrcache write)], to => 'value' }, 118 processes => { from => [qw(running sleeping zombies stopped paging blocked)], to => 'value' }, 119 swap => { from => [qw(cached free used resv)], to => 'value', type_instance => [qw(cached free used reserved)] } 120); 121 122our %TypeRename = 123( 124 traffic => { from => [qw(incoming outgoing)], to => [qw(rx tx)] }, 125 vs_processes => { from => [qw(total)], to => [qw(value)] }, 126); 127 128GetOptions ("indir|i=s" => \$InDir, 129 "outdir|o=s" => \$OutDir, 130 "hostname=s" => \$Hostname) or exit_usage (); 131 132die "No such directory: $InDir" if (!-d $InDir); 133 134our @Files = (); 135our %OutDirs = (); 136 137@Files = find_files (); 138 139for (@Files) 140{ 141 my $orig_filename = $_; 142 my $orig = parse_file ($orig_filename); 143 my $dest = translate_file ($orig); 144 my $dest_filename = get_filename ($dest); 145 146 my $dest_directory = dirname ($dest_filename); 147 if (!exists ($OutDirs{$dest_directory})) 148 { 149 print "[ -d '$OutDir/$dest_directory' ] || mkdir -p '$OutDir/$dest_directory'\n"; 150 $OutDirs{$dest_directory} = 1; 151 } 152 153 if (($orig->{'type'} eq 'disk') || ($orig->{'type'} eq 'partition')) 154 { 155 special_disk ($orig_filename, $orig, $dest_filename, $dest); 156 } 157 elsif (exists ($TypeSplit{$orig->{'type'}})) 158 { 159 my $src_dses = $TypeSplit{$orig->{'type'}}->{'from'}; 160 my $dst_ds = $TypeSplit{$orig->{'type'}}->{'to'}; 161 my $type_instances = exists ($TypeSplit{$orig->{'type'}}->{'type_instance'}) 162 ? $TypeSplit{$orig->{'type'}}->{'type_instance'} 163 : $TypeSplit{$orig->{'type'}}->{'from'}; 164 165 for (my $i = 0; $i < @$src_dses; $i++) 166 { 167 my $src_ds = $src_dses->[$i]; 168 $dest->{'type_instance'} = $type_instances->[$i]; 169 $dest_filename = get_filename ($dest); 170 print "./rrd_filter.px -i '$InDir/$orig_filename' -m '${src_ds}:${dst_ds}' -o '$OutDir/$dest_filename'\n"; 171 } 172 } 173 else 174 { 175 print "cp '$InDir/$orig_filename' '$OutDir/$dest_filename'\n"; 176 } 177 178 if (exists ($TypeRename{$orig->{'type'}})) 179 { 180 my $src_dses = $TypeRename{$orig->{'type'}}->{'from'}; 181 my $dst_dses = $TypeRename{$orig->{'type'}}->{'to'}; 182 183 print "rrdtool tune '$OutDir/$dest_filename'"; 184 for (my $i = 0; $i < @$src_dses; $i++) 185 { 186 print " --data-source-rename " 187 . $src_dses->[$i] . ':' . $dst_dses->[$i]; 188 } 189 print "\n"; 190 } 191} 192 193exit (0); 194 195sub translate_file 196{ 197 my $orig = shift; 198 my $dest = {}; 199 %$dest = %$orig; 200 201 if (defined ($TypeTranslate{$orig->{'type'}})) 202 { 203 $TypeTranslate{$orig->{'type'}}->($dest); 204 } 205 206 return ($dest); 207} # translate_file 208 209sub get_filename 210{ 211 my $args = shift; 212 my $filename = $args->{'host'} 213 . '/' . $args->{'plugin'} . (defined ($args->{'plugin_instance'}) ? '-'.$args->{'plugin_instance'} : '') 214 . '/' . $args->{'type'} . (defined ($args->{'type_instance'}) ? '-'.$args->{'type_instance'} : '') . '.rrd'; 215 216 return ($filename); 217} 218 219sub parse_file 220{ 221 my $fullname = shift; 222 my @parts = split ('/', $fullname); 223 224 my $filename; 225 226 my $host; 227 my $plugin; 228 my $plugin_instance; 229 my $type; 230 my $type_instance; 231 232 $filename = pop (@parts); 233 234 if ($filename =~ m/^([^-]+)(?:-(.*))?\.rrd$/) 235 { 236 $type = $1; 237 $type_instance = $2; 238 } 239 else 240 { 241 return; 242 } 243 244 if (@parts) 245 { 246 my $dirname = pop (@parts); 247 my $regex_str = join ('|', keys (%Subdirs)); 248 if ($dirname =~ m/^($regex_str)(?:-(.*))?$/) 249 { 250 $plugin = $1; 251 $plugin_instance = $2; 252 } 253 else 254 { 255 push (@parts, $dirname); 256 } 257 } 258 if (!$plugin) 259 { 260 $plugin = $type; 261 } 262 263 if (@parts) 264 { 265 $host = pop (@parts); 266 } 267 else 268 { 269 $host = $Hostname; 270 } 271 272 return 273 ({ 274 host => $host, 275 plugin => $plugin, 276 plugin_instance => $plugin_instance, 277 type => $type, 278 type_instance => $type_instance 279 }); 280} # parse_file 281 282sub find_files 283{ 284 my $reldir = @_ ? shift : ''; 285 my $absdir = $InDir . ($reldir ? "/$reldir" : ''); 286 287 my $dh; 288 289 my @files = (); 290 my @dirs = (); 291 292 opendir ($dh, $absdir) or die ("opendir ($absdir): $!"); 293 while (my $file = readdir ($dh)) 294 { 295 next if ($file =~ m/^\./); 296 next if (-l "$absdir/$file"); 297 if (-d "$absdir/$file") 298 { 299 push (@dirs, ($reldir ? "$reldir/" : '') . $file); 300 } 301 elsif ($file =~ m/\.rrd$/) 302 { 303 push (@files, ($reldir ? "$reldir/" : '') . $file); 304 } 305 } 306 closedir ($dh); 307 308 for (my $i = 0; $i < @dirs; $i++) 309 { 310 push (@files, find_files ($dirs[$i])); 311 } 312 313 return (@files); 314} # find_files 315 316{my $cache; 317sub _special_disk_instance 318{ 319 my $orig_instance = shift; 320 321 if (!defined ($cache)) 322 { 323 my $fh; 324 open ($fh, "< /proc/diskstats") or die ("open (/proc/diststats): $!"); 325 326 $cache = {}; 327 while (my $line = <$fh>) 328 { 329 chomp ($line); 330 my @fields = split (' ', $line); 331 $cache->{$fields[0] . '-' . $fields[1]} = $fields[2]; 332 } 333 close ($fh); 334 } 335 336 return (defined ($cache->{$orig_instance}) 337 ? $cache->{$orig_instance} 338 : $orig_instance); 339}} 340 341sub special_disk 342{ 343 my $orig_filename = shift; 344 my $orig = shift; 345 my $dest_filename = shift; 346 my $dest = shift; 347 my $dest_directory; 348 349 $dest->{'type_instance'} = undef; 350 $dest->{'plugin_instance'} = _special_disk_instance ($orig->{'type_instance'}); 351 if ($dest->{'plugin_instance'} eq $orig->{'type_instance'}) 352 { 353 print qq(echo "You may need to rename these files" >&2\n); 354 } 355 356 $dest->{'type'} = 'disk_merged'; 357 $dest_filename = get_filename ($dest); 358 359 $dest_directory = dirname ($dest_filename); 360 if (!exists ($OutDirs{$dest_directory})) 361 { 362 print "[ -d '$OutDir/$dest_directory' ] || mkdir -p '$OutDir/$dest_directory'\n"; 363 $OutDirs{$dest_directory} = 1; 364 } 365 366 print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rmerged:read' -m 'wmerged:write' -o '$OutDir/$dest_filename'\n"; 367 368 $dest->{'type'} = 'disk_octets'; 369 $dest_filename = get_filename ($dest); 370 print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rbytes:read' -m 'wbytes:write' -o '$OutDir/$dest_filename'\n"; 371 372 $dest->{'type'} = 'disk_ops'; 373 $dest_filename = get_filename ($dest); 374 print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rcount:read' -m 'wcount:write' -o '$OutDir/$dest_filename'\n"; 375 376 $dest->{'type'} = 'disk_time'; 377 $dest_filename = get_filename ($dest); 378 print "./rrd_filter.px -i '$InDir/$orig_filename' -m 'rtime:read' -m 'wtime:write' -o '$OutDir/$dest_filename'\n"; 379} 380 381sub exit_usage 382{ 383 print <<EOF; 384Usage: $0 [-i indir] [-o outdir] [--hostname myhostname] 385EOF 386 exit (1); 387} 388