1#! /usr/bin/perl 2 3use strict; 4use warnings; 5 6use Getopt::Long; 7use File::Tail::Scribe; 8use POSIX (); 9use FindBin (); 10use File::Basename (); 11use File::Spec::Functions; 12use Pod::Usage; 13use Sys::Hostname; 14use YAML::Any; 15use Proc::ProcessTable; 16 17my $script = File::Basename::basename($0); 18my $SELF = catfile $FindBin::Bin, $script; 19my @saved_argv = @ARGV; 20 21my $sigset = POSIX::SigSet->new(); 22my $hup = POSIX::SigAction->new('sigHUP_handler', 23 $sigset, 24 &POSIX::SA_NODEFER); 25POSIX::sigaction(&POSIX::SIGHUP, $hup); 26my $term = POSIX::SigAction->new('sigTERM_handler', 27 $sigset, 28 &POSIX::SA_NODEFER); 29POSIX::sigaction(&POSIX::SIGTERM, $term); 30POSIX::sigaction(&POSIX::SIGINT, $term); 31POSIX::sigaction(&POSIX::SIGQUIT, $term); 32 33my @cat_re; 34my %args = ( 35 config => '/etc/tail_to_scribe.conf', 36 dirs => [ '/var/log/httpd' ], 37 filter => '[._]log$', 38 'exclude-dir' => [], 39 'exclude-re' => [], 40 'follow-symlinks' => 0, 41 'sleep-interval' => 2, 42 host => 'localhost', 43 port => 1463, 44 level => 'info', 45 'retry-plan-a' => 'buffer', 46 'retry-plan-b' => 'discard', 47 'retry-buffer-size' => 100000, 48 'retry-count' => 100, 49 'retry-delay' => 10, 50 'state-file-name' => '.tailtoscribe', 51 'no-init' => 0, 52); 53 54GetOptions(\%args, 55 'category=s', 56 'config=s', 57 'dirs=s{1,}', 58 'excluded-dir=s{1,}', 59 'excluded-re=s{1,}', 60 'follow-symlinks', 61 'sleep-interval=i', 62 'filter=s', 63 'port=i', 64 'host=s', 65 'level=s', 66 'no-init', 67 'retry-plan-a=s', 68 'retry-plan-b=s', 69 'retry-buffer-size=i', 70 'retry-count=i', 71 'retry-delay=i', 72 'state-file-name=s', 73 'debug:s', 74 'daemon', 75 'pidfile=s', 76 "help|?", 77 ) or pod2usage(-exitval => 2, -verbose => 0); 78 79pod2usage(-exitval => 0, -verbose => 2) if $args{'help'}; 80 81 82my $dbg_file; 83my $debug; 84if (defined $args{debug}) { 85 $debug++; 86 if ($args{debug}) { 87 open($dbg_file, '>', $args{debug}) or die "Failed to open debug file $args{debug}: $!"; 88 } 89 else { 90 $dbg_file = \*STDERR; 91 } 92 select($dbg_file); 93 $| = 1; 94} 95 96my @excludes = @{$args{'exclude-dir'}}; 97push(@excludes, map { qr/$_/ } @{$args{'exclude-re'}}); 98 99my $hostname = hostname(); 100my $msg_filter = sub { 101 my $self = shift; 102 my $filename = shift; 103 my $line = shift; 104 $filename =~ s{^.*/}{}; # remove leading dirs 105 $filename =~ s{(?:[._-]access)?[._-][^._-]*$}{}; # remove extension 106 $filename ||= 'default'; # in case everything gets removed 107 108 return ('info', 'httpd', "$hostname\t$filename\t$line"); 109}; 110 111if ( -f $args{config} ) { 112 eval `cat $args{config}`; 113 die "Failed to load \"$args{config}\": $@" if $@; 114} 115 116check_pid($args{pidfile}) if $args{pidfile}; 117 118if ($args{daemon}) { 119 open STDIN, '/dev/null' or die "Can't read /dev/null: $!"; 120 open STDOUT, '>/dev/null' or die "Can't write to /dev/null: $!"; 121 defined(my $pid = fork) or die "Can't fork: $!"; 122 exit if $pid; 123 POSIX::setsid() or die "Can't start a new session: $!"; 124 open STDERR, '>&STDOUT' or die "Can't dup stdout: $!"; 125} 126 127write_pid($args{pidfile}) if $args{pidfile}; 128 129END { 130 cleanup_pid($args{pidfile}); 131} 132 133if ($debug) { 134 print "Command line arguments\n " . Dump(\%args) . "===\n"; 135} 136 137my $log = File::Tail::Scribe->new( 138 directories => $args{dirs}, 139 filter => qr/$args{filter}/, 140 exclude => \@excludes, 141 follow_symlinks => $args{'follow-symlinks'}, 142 sleep_interval => $args{'sleep-interval'}, 143 scribe_options => { 144 name => 'scribe', 145 min_level => $args{level}, 146 host => $args{host}, 147 port => $args{port}, 148 default_category => $args{category}, 149 retry_plan_a => $args{'retry-plan-a'}, 150 retry_plan_b => $args{'retry-plan-b'}, 151 retry_buffer_size => $args{'retry-buffer-size'}, 152 retry_count => $args{'retry-count'}, 153 retry_delay => $args{'retry-delay'}, 154 }, 155 msg_filter => $msg_filter, 156 default_level => $args{level}, 157 statefilename => $args{'state-file-name'}, 158 no_init => $args{'no-init'}, 159 ); 160 161$log->watch_files(); 162 163sub sigHUP_handler { 164 $log->save_state(); 165 exec($SELF, @saved_argv) or die "Couldn't restart: $!\n"; 166} 167 168sub sigTERM_handler { 169 $log->save_state(); 170 cleanup_pid($args{pidfile}); 171 exit(); 172} 173 174sub read_pid { 175 my $pidfile = shift; 176 open my $fh, '<', $pidfile or return; 177 my $pid = <$fh>; 178 close($fh); 179 chomp $pid if $pid; 180 return $pid; 181} 182 183sub write_pid { 184 my $pidfile = shift; 185 open my $fh, '>', $pidfile or die "Failed to open $pidfile for writing: $!"; 186 print $fh "$$\n"; 187 close($fh); 188} 189 190sub check_pid { 191 my $pidfile = shift; 192 my $pid = read_pid($pidfile) or return; 193 my $t = Proc::ProcessTable->new(); 194 for my $p ( @{$t->table} ) { 195 if ($p->pid == $pid && $p->cmndline =~ m/tail_to_scribe/) { 196 die "tail_to_scribe is already running, PID $pid, pidfile $pidfile\n"; 197 } 198 } 199} 200 201sub cleanup_pid { 202 my $pidfile = shift; 203 if ($pidfile && (my $pid = read_pid($pidfile)) ) { 204 unlink $pidfile if $pid == $$; 205 } 206} 207 208__END__ 209 210=head1 NAME 211 212tail_to_scribe.pl - Tail files and send to a Scribe logging system. 213 214 215=head1 SYNOPSIS 216 217 tail_to_scribe.pl [ --config=CONFIG_FILE ] 218 [ --daemon ] 219 [ --dirs DIR1 [DIR2 ...] ] 220 [ --excluded-dir XDIR1 [XDIR2 ...] ] 221 [ --excluded-re REGEXP1 [REGEXP2 ...] ] 222 [ --filter=REGEXP ] 223 [ --follow-symlinks ] 224 [ --no-init ] 225 [ --state-file-name=FILE ] 226 [ --sleep-interval=SECS ] 227 [ --port=PORT ] [ --host=HOST ] 228 [ --level=LEVEL ] [ --category=CATEGORY ] 229 230=head1 DESCRIPTION 231 232tail_to_scribe.pl monitors files in a given directory (or set of directories), 233such as Apache log files in /var/log/httpd, and as the log files are written to, 234takes the changes and sends them to a running instance of the Scribe logging 235system. 236 237=head1 OPTIONS 238 239=head2 --daemon 240 241Run in the background. 242 243=head2 --dirs DIR1 [DIR2 ...] 244 245The list of directories in which to monitor files for changes. Defaults to /var/log/httpd. 246 247=head2 --excluded-dir XDIR1 [XDIR2 ...] 248 249A list of directories to exclude from monitoring. These must be full filesystem paths. Defaults to empty (no exclusions). 250 251=head2 --excluded-re REGEXP1 [REGEXP2 ...] 252 253A list of exclude regular expressions; any directory paths that match will be excluded from monitoring. Defaults to empty (no exclusions). 254 255=head2 --filter=REGEXP 256 257A file filter regular expression; only filenames that match will be monitored. Defaults to '[._]log$' (files ending in .log or _log). Set to '.*' to include all files. 258 259=head2 --follow-symlinks 260 261If set, follow symbolic links in the filesystem. 262 263=head2 --no-init 264 265If set, any existing state file will be ignored, and only changes from the 266current file state will be sent. Without --no-init, on the first run (before 267any state file is created), any existing content in the monitored files will be 268sent as well as changes (which could be a large amount of data if you have big 269files). 270 271=head2 --state-file-name=FILE 272 273Name of file in which to store state between runs. Defaults to '.tailtoscribe' in the working directory. 274 275=head2 --sleep-interval=SECS 276 277Where a kernel-based file change notification system is not available, this 278specifies the number of seconds between scans for file changes. 279 280B<To minimise CPU usage, installing L<Linux::Inotify2> is highly recommended.> 281 282=head2 Scribe Options 283 284=over 4 285 286=item --host, --port 287 288Host and port of Scribe server. Defaults to localhost, port 1463. 289 290=item --category=CATEGORY 291 292Default Scribe logging category. Defaults to 'httpd'. 293 294=item --level=LEVEL 295 296Default log level. Defaults to 'info'. May be set to any valid 297L<Log::Dispatch> level (debug, info, notice, warning, error, critical, alert, 298emergency). 299 300=item --retry-plan-a=MODE, --retry-plan-b=MODE, --retry-buffer-size=SIZE, --retry-count=COUNT, --retry-delay=DELAY 301 302See L<Log::Dispatch::Scribe> for full description of these options. 303 304=back 305 306=head2 --pidfile=FILE 307 308Write process ID to file FILE. tail_to_scribe.pl will use this file to check if 309an instance is already running, and refuse to start if the PID in this file 310corresponds to another tail_to_scribe.pl process. Checks are skipped if no 311pidfile is given. 312 313=head2 --debug, --debug=FILE 314 315Enable debugging to standard error or to file. 316 317=head2 --config=CONFIG_FILE 318 319Specify the location of the configuration file (an included perl script). 320Defaults to /etc/tail_to_scribe.conf. A typical configuration file might 321look like this: 322 323 # Set my arg values 324 my %localargs = ( 325 dirs => [ '/var/log/httpd' ], 326 filter => 'access[._]log$', 327 'exclude-dir' => [ '/var/log/httpd/fastcgi' ], 328 'state-file-name' => '/var/log/httpd/.tailtoscribe', 329 ); 330 331 # Copy into args to override defaults 332 $args{$_} = $localargs{$_} for keys %localargs; 333 334 1; # Must return a true value 335 336In addition to all of the options available on the command line, a custom 337message filter may also be included, e.g. 338 339 $msg_filter = sub { 340 my ($self, $filename, $line) = @_; 341 342 return ('info', 'httpd', "$filename\t$line"); 343 }; 344 345See L<File::Tail::Scribe/msg_filter> for more details on the msg_filter. 346 347=head1 SIGNALS 348 349HUP signal causes tail_to_scribe.pl to restart. TERM/QUIT/INT cause it to save state and exit. 350 351=head1 SEE ALSO 352 353=over 4 354 355=item * L<File::Tail::Scribe> 356 357=item * L<File::Tail::Dir> 358 359=item * L<http://notes.jschutz.net/109/perl/perl-client-for-facebooks-scribe-logging-software> 360 361=item * L<http://github.com/facebook/scribe/> 362 363=item * L<Log::Dispatch::Scribe> 364 365=back 366 367=head1 AUTHOR 368 369Jon Schutz, C<< <jon at jschutz.net> >> L<notes.jschutz.net> 370 371=head1 BUGS 372 373Please report any bugs or feature requests to C<bug-file-tail-scribe at 374rt.cpan.org>, or through the web interface at 375L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=File-Tail-Scribe>. I will be 376notified, and then you'll automatically be notified of progress on your bug as I 377make changes. 378 379 380=head1 SUPPORT 381 382You can find documentation for this module with the perldoc command. 383 384 perldoc File::Tail::Scribe 385 386 387You can also look for information at: 388 389=over 4 390 391=item * RT: CPAN's request tracker 392 393L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=File-Tail-Scribe> 394 395=item * AnnoCPAN: Annotated CPAN documentation 396 397L<http://annocpan.org/dist/File-Tail-Scribe> 398 399=item * CPAN Ratings 400 401L<http://cpanratings.perl.org/d/File-Tail-Scribe> 402 403=item * Search CPAN 404 405L<http://search.cpan.org/dist/File-Tail-Scribe/> 406 407=back 408 409 410=head1 ACKNOWLEDGEMENTS 411 412 413=head1 COPYRIGHT & LICENSE 414 415Copyright 2010 Jon Schutz, all rights reserved. 416 417This program is free software; you can redistribute it and/or modify it 418under the same terms as Perl itself. 419 420 421=cut 422