1#!/usr/local/bin/perl
2# vim:ts=4
3##############################################################################
4# rrd-archive.pl v0.5
5# S Shipway 2003,2004,2013.  Distributed under the GNU GPL
6#
7# This Perl script can be run on a nightly basis.
8#
9# This script will check the specified .cfg files, and will archive the
10# corresponding .rrd files  as defined (default is to archive for one month).
11# This is different from the graph 'Archive' function - this will archive
12# the raww .rrd data, not the graph itself and is therefore far more
13# flexible (although more costly in disk space).
14#
15# It will also delete expired archives - default is to keep for 1 month,
16# except for the 1st of each month which is kept for a year, and the first
17# of Jan which is kept indefinitely.
18#
19# First change the conffile location defined below, and the perl location
20# definedin the first line.
21#
22# Run this script at just before midnight via cron or your favourite scheduler
23# 55 23 * * * /usr/local/bin/rrd-archive.pl
24#
25# Usage:
26#    perl rrd-archive.pl
27#
28# Will also read the routers.conf file, and look in the [archive] section,
29# if it exists.  See the example for the options.
30#
31# Added options to your MRTG .cfg file:
32#
33# routers.cgi*Archive[targetname]:
34#   can take 'daily xxx' for some number xxx, 'monthly yyy' for some number yyy
35#   to specify expiry of daily archives (in days) and monthly (in months).
36#   Default is 30 days, 12 months.
37#
38# Steve Shipway, Oct 2003
39##############################################################################
40
41use strict;
42################# CONFIGURABLE LINES START ###############
43# default location of routers.conf file
44my( $conffile ) = "/u01/etc/routers2.conf";
45# default number of days after which to expire archived logs
46my( $expiredaily ) = 31;
47my( $expiremonthly ) = 12; # 1st of the month are Monthly
48################# CONFIGURABLE LINES END #################
49
50my($VERSION) = "0.5";
51my(@cfgfiles) = ();
52my($pattern, $thisfile);
53my($workdir, $rrd, $opt );
54my($expd, $expm);
55my(%targets,$t);
56my( %config );
57my( $NT ) = 0;
58my( $pathsep ) = "/";
59my( $confpath, $cfgfiles );
60my( $debug ) = 0;
61
62my( @now, $today );
63
64#################################
65# For RRD archives: make 2chr subdir name from filename
66sub makehash($) {
67    my($x);
68# This is more balanced
69    $x = unpack( '%8C*',$_[0] );
70# This is easier to follow
71#   $x = substr($_[0],0,2);
72    return $x;
73}
74
75###########################################################################
76# readconf: pass it a list of section names
77sub readconf(@)
78{
79	my ($inlist, $i, @secs, $sec);
80
81	@secs = @_;
82	%config = ();
83
84	# set defaults
85	%config = ( 'routers.cgi-confpath' => ".",);
86
87	( open CFH, "<".$conffile ) || do {
88		print "Error: unable to open file $conffile\n";
89		exit(0);
90	};
91
92	$inlist=0;
93	$sec = "";
94	while( <CFH> ) {
95		/^\s*#/ && next;
96		/\[(\S*)\]/ && do {
97			$sec = $1;
98			$inlist=0;
99			foreach $i ( @secs ) {
100				if ( $i eq $1 ) { $inlist=1; last; }
101			}
102			next;
103		};
104		# note final \s* to strip all trailing spaces (which doesnt work
105		# because the * operator is greedy!)
106		if ( $inlist ) { /(\S+)\s*=\s*(\S.*?)\s*$/ and $config{"$sec-$1"}=$2; }
107	}
108	close CFH;
109
110	# Activate NT compatibility options.
111	# $^O is the OS name, NT usually produces 'MSWin32'.  By checking for 'Win'
112	# we should be able to cover most possibilities.
113	if ( (defined $config{'web-NT'} and $config{'web-NT'}=~/[1y]/i)
114		or $^O =~ /Win/ or $^O =~ /DOS/i  ) {
115		$pathsep = "\\";
116		$NT = 1;
117	}
118
119	# some path corrections: remove trailing path separators on f/s paths
120	foreach ( qw/dbpath confpath graphpath graphurl/ ) {
121		$config{"routers.cgi-$_"} =~ s/[\/\\]\s*$//;
122	}
123
124}
125###########################################################################
126# Run the archive for the specified RRD
127sub do_archive($$$) {
128	my( $rrd, $expd, $expm ) = @_;
129	my( $archdir, $rrdfile, $rrdpath );
130	my( $newfile );
131	my( $y, $m, $d, $afile, $age );
132
133	print "--Target $rrd\n" if($debug);
134
135	if(!$expd) { # If expiredaily is 0, then we dont archive at all.
136		print "  No archiving required for this target.\n" if($debug);
137		return;
138	}
139
140	# Identify and create the archive location. Should really use Basename
141	if( $rrd =~ /^(.*)[\\\/]([^\\\/]+\.rrd)$/ ) {
142		$rrdpath = $1; $rrdfile = $2;
143	} else {
144		$rrdpath =  $pathsep; $rrdfile = $rrd;
145	}
146	$archdir = $rrdpath.$pathsep."archive";
147	if(!-d $archdir) {
148		if(!mkdir($archdir,0755)) {
149			print "Unable to create directory $archdir\n";
150			return;
151		}
152	}
153	if($config{'routers.cgi-archive-mode'} and
154        $config{'routers.cgi-archive-mode'}=~/hash/i ) {
155		if(!-d $archdir.$pathsep.makehash($rrdfile)) {
156			if(!mkdir($archdir.$pathsep.makehash($rrdfile),0755)) {
157				print "Unable to create directory $archdir/hash\n";
158				return;
159			} else {
160				print "Created directory for hash\n";
161			}
162		}
163		if(!-d $archdir.$pathsep.makehash($rrdfile).$pathsep.$rrdfile.".d") {
164			if(!mkdir($archdir.$pathsep.makehash($rrdfile).$pathsep.$rrdfile.".d",0755)) {
165				print "Unable to create directory $archdir/hash/rrd\n";
166				return;
167			} else {
168				print "Created directory for hash/rrd\n";
169			}
170		}
171		$newfile = $archdir.$pathsep.makehash($rrdfile).$pathsep.$rrdfile.".d".$pathsep.$today.".rrd";
172	} else {
173		# do we need to create a new date directory?
174		if(!-d $archdir.$pathsep.$today) {
175			if(!mkdir($archdir.$pathsep.$today,0755)) {
176				print "Unable to create directory $archdir.$pathsep.$today\n";
177				return;
178			} else {
179				print "Created directory for $today\n";
180			}
181		}
182		$newfile = $archdir.$pathsep.$today.$pathsep.$rrdfile;
183	}
184	# Now we have an archive location.
185
186	# Next, we want to archive the current .rrd file into this location.
187	if( -f $newfile ) {
188		print "Archive $newfile already exists!\n" if($debug);
189		print "!" if(!$debug);
190	} else {
191		my($buf);
192		print "." if(!$debug);
193		if(open (CURRENT, "<$rrd") and open (NEW, ">$newfile")) {
194		binmode CURRENT or die("Bad filehandle");
195		binmode NEW or die("Bad filehandle");
196		while( read CURRENT, $buf, 16384 ) { print NEW $buf; }
197		close NEW;
198		close CURRENT;
199		print "  (A) Archived $rrdfile for $today\n" if($debug);
200		} else {
201			print "$rrd\n$newfile\nProblem opening files: $!\n";
202		}
203	}
204
205	# Now we want to expire anything that is too old in this tree
206	# This is probably not the most efficient way of achieving this
207	foreach $afile (glob(
208        ($config{'routers.cgi-archive-mode'} and
209            $config{'routers.cgi-archive-mode'}=~/hash/i )?
210		($archdir.$pathsep.makehash($rrdfile).$pathsep.$rrdfile.".d".$pathsep."*-*-*.rrd"):($archdir.$pathsep."*".$pathsep.$rrdfile)
211		)) {
212		$afile =~ /[\\\/](\d\d\d\d)-(\d\d)-(\d\d)(.rrd)?[\\\/]/;
213		($y,$m,$d) = ($1,$2,$3);
214		if(!$y) {
215			# error parsing filename.  This should never happen, but does.
216			print "ERROR: Cannot find yyyy-mm-dd in $afile\n";
217			next;
218		}
219		$age = ($now[5]+1900)-$y; # years
220		$age = ($age * 12) + ($now[4]+1) - $m; # months
221
222		if( $d == 1 ) {
223			# month aging
224			if( $expm and ( $age >= $expm )) {
225				unlink($afile);
226				print "  (M) Deleted $afile\n" if($debug);
227			} # zap it
228		} else {
229			# day aging
230			$age = ($age * 30) + $now[3] - $d; # days (approx)
231			# we might zap things a day early in feb though
232			if( $age >= $expd ) {
233				unlink($afile);
234				print "  (D) Deleted $afile\n" if($debug);
235			} # zap it
236		}
237	}
238
239	# Remove any empty directories
240	# We have to do this per-target since targets may have differnt
241	# retention times for their archives, and may have different
242	# workdirs.  Really we should make a list of unique workdirs
243	# and process them after.
244
245	foreach $afile ( glob($archdir.$pathsep."*") ) {
246		rmdir $afile; # this will fail unless afile is empty
247	}
248}
249
250###########################################################################
251
252
253############### MAIN CODE STARTS HERE #######
254
255# get parameters
256print "Reading configuration\n" if($debug);
257readconf('routers.cgi','web','archive');
258
259$confpath = $config{'routers.cgi-confpath'};
260$confpath = $config{'archive-confpath'}
261	if(defined $config{'archive-confpath'});
262$cfgfiles = $config{'routers.cgi-cfgfiles'};
263$cfgfiles = $config{'archive-cfgfiles'}
264	if(defined $config{'archive-cfgfiles'});
265if(! -d $confpath ) {
266	print "Error: MRTG directory $confpath does not exist.\n";
267	exit 1;
268}
269$expiredaily = $config{'archive-expiredaily'}
270	if(defined $config{'archive-expiredaily'});
271$expiredaily = $config{'archive-keepdaily'}
272	if(defined $config{'archive-keepdaily'});
273$expiremonthly = $config{'archive-expiremonthly'}
274	if(defined $config{'archive-expiremonthly'});
275$expiremonthly = $config{'archive-keepmonthly'}
276	if(defined $config{'archive-keepmonthly'});
277
278# What day do we log as?
279if( $config{'archive-asyesterday'} =~ /[y1]/i ) {
280	@now = localtime(time-(24*3600));
281} else {
282	@now = localtime(time);
283}
284$today = sprintf("%04d-%02d-%02d",$now[5]+1900,$now[4]+1,$now[3]);
285
286# Now we have the defaults, and we know which files to process.
287# We can optimise our processing of the .cfg files.
288
289foreach $pattern ( split " ",$cfgfiles ) {
290#	print "$confpath$pathsep$pattern\n" if($debug);
291	push @cfgfiles, glob( $confpath.$pathsep.$pattern );
292}
293
294if( @ARGV ) { @cfgfiles = @ARGV; }
295
296foreach $thisfile ( @cfgfiles ) {
297	next if(!-f $thisfile);
298	open CFG,"<$thisfile" or next;
299	print "Processing $thisfile\n" ;
300	$workdir = $config{'routers.cgi-dbpath'}; # default
301	%targets = ( '_' => { expd => $expiredaily, expm => $expiremonthly });
302	while ( <CFG> ) {
303		if( /^\s*Include\s*:\s*(\S+)/i ) { push @cfgfiles,$1; next; }
304		if( /^\s*WorkDir\s*:\s*(\S+)/i ) {
305			$workdir = $1; next;
306		}
307		if( /^\s*Directory\[(\S+)\]\s*:\s*(\S+)/i ) {
308			$t = lc $1;
309			$targets{$t} = {} if(!defined $targets{$t});
310			$targets{$t}->{directory} = $2;
311			next;
312		}
313		if( /^\s*Target\[(\S+)\]/i ) {
314			$t = lc $1;
315			$targets{$t} = {} if(!defined $targets{$t});
316			$targets{$t}->{file} = "$t.rrd";
317			next;
318		}
319		if( /^\s*routers\.cgi\*Archive\[(\S+)\]\s*:\s*(\S.+)/i ) {
320			$t = lc $1;
321			$opt = $2;
322			($expd, $expm) = ($expiredaily, $expiremonthly);
323			if( $opt =~ /no/i ) {
324				($expd, $expm) = (0,0);
325			} elsif( $opt =~ /daily/ or $opt =~ /monthly/) {
326				if( $opt =~ /daily\s+(\d+)/i ) { $expd = $1; }
327				if( $opt =~ /monthly\s+(\d+)/i ) { $expm = $1; }
328			} elsif( $opt =~ /(\d+)/i ) { $expd = $1; }
329			$targets{$t}->{expd} = $expd;
330			$targets{$t}->{expm} = $expm;
331			next;
332		}
333	}
334	close CFG;
335	# now process the archiving
336	foreach $t ( keys %targets ) {
337		next if(!defined $targets{$t}->{file}); # skip dummy ones
338		foreach ( keys %{$targets{'_'}} ) {
339			$targets{$t}->{$_} = $targets{'_'}->{$_}
340				if(!defined $targets{$t}->{$_});
341		}
342		$rrd = $workdir;
343		$rrd .= $pathsep.$targets{$t}->{directory} if(defined $targets{$t}->{directory});
344		$rrd .= $pathsep.$targets{$t}->{file};
345		$targets{$t}->{rrd} = $rrd;
346		do_archive ( $rrd, $targets{$t}->{expd}, $targets{$t}->{expm});
347	}
348	print "\n" if(!$debug);
349
350}
351
352print "All finished.\n" ;
353exit(0);
354