1#! /usr/bin/perl
2
3use strict;
4use warnings;
5
6use Getopt::Long;
7use File::Tail::Scribe;
8use POSIX ();
9use FindBin ();
10use File::Basename ();
11use File::Spec::Functions;
12use Pod::Usage;
13use Sys::Hostname;
14use YAML::Any;
15use Proc::ProcessTable;
16
17my $script = File::Basename::basename($0);
18my $SELF = catfile $FindBin::Bin, $script;
19my @saved_argv = @ARGV;
20
21my $sigset = POSIX::SigSet->new();
22my $hup = POSIX::SigAction->new('sigHUP_handler',
23				   $sigset,
24				   &POSIX::SA_NODEFER);
25POSIX::sigaction(&POSIX::SIGHUP, $hup);
26my $term = POSIX::SigAction->new('sigTERM_handler',
27				 $sigset,
28				 &POSIX::SA_NODEFER);
29POSIX::sigaction(&POSIX::SIGTERM, $term);
30POSIX::sigaction(&POSIX::SIGINT, $term);
31POSIX::sigaction(&POSIX::SIGQUIT, $term);
32
33my @cat_re;
34my %args = (
35    config => '/etc/tail_to_scribe.conf',
36    dirs => [ '/var/log/httpd' ],
37    filter => '[._]log$',
38    'exclude-dir' => [],
39    'exclude-re' => [],
40    'follow-symlinks' => 0,
41    'sleep-interval' => 2,
42    host => 'localhost',
43    port => 1463,
44    level => 'info',
45    'retry-plan-a' => 'buffer',
46    'retry-plan-b' => 'discard',
47    'retry-buffer-size' => 100000,
48    'retry-count' => 100,
49    'retry-delay' => 10,
50    'state-file-name' => '.tailtoscribe',
51    'no-init' => 0,
52);
53
54GetOptions(\%args,
55	   'category=s',
56	   'config=s',
57	   'dirs=s{1,}',
58	   'excluded-dir=s{1,}',
59	   'excluded-re=s{1,}',
60	   'follow-symlinks',
61	   'sleep-interval=i',
62	   'filter=s',
63	   'port=i',
64	   'host=s',
65	   'level=s',
66	   'no-init',
67	   'retry-plan-a=s',
68	   'retry-plan-b=s',
69	   'retry-buffer-size=i',
70	   'retry-count=i',
71	   'retry-delay=i',
72	   'state-file-name=s',
73	   'debug:s',
74	   'daemon',
75	   'pidfile=s',
76	   "help|?",
77           ) or pod2usage(-exitval => 2, -verbose => 0);
78
79pod2usage(-exitval => 0, -verbose => 2) if $args{'help'};
80
81
82my $dbg_file;
83my $debug;
84if (defined $args{debug}) {
85    $debug++;
86    if ($args{debug}) {
87	open($dbg_file, '>', $args{debug}) or die "Failed to open debug file $args{debug}: $!";
88    }
89    else {
90	$dbg_file = \*STDERR;
91    }
92    select($dbg_file);
93    $| = 1;
94}
95
96my @excludes = @{$args{'exclude-dir'}};
97push(@excludes, map { qr/$_/ } @{$args{'exclude-re'}});
98
99my $hostname = hostname();
100my $msg_filter = sub {
101    my $self = shift;
102    my $filename = shift;
103    my $line = shift;
104    $filename =~ s{^.*/}{};		      # remove leading dirs
105    $filename =~ s{(?:[._-]access)?[._-][^._-]*$}{}; # remove extension
106    $filename ||= 'default';                  # in case everything gets removed
107
108    return ('info', 'httpd', "$hostname\t$filename\t$line");
109};
110
111if ( -f $args{config} ) {
112    eval `cat $args{config}`;
113    die "Failed to load \"$args{config}\": $@" if $@;
114}
115
116check_pid($args{pidfile}) if $args{pidfile};
117
118if ($args{daemon}) {
119    open STDIN, '/dev/null' or die "Can't read /dev/null: $!";
120    open STDOUT, '>/dev/null' or die "Can't write to /dev/null: $!";
121    defined(my $pid = fork) or die "Can't fork: $!";
122    exit if $pid;
123    POSIX::setsid() or die "Can't start a new session: $!";
124    open STDERR, '>&STDOUT' or die "Can't dup stdout: $!";
125}
126
127write_pid($args{pidfile}) if $args{pidfile};
128
129END {
130    cleanup_pid($args{pidfile});
131}
132
133if ($debug) {
134    print "Command line arguments\n " . Dump(\%args) . "===\n";
135}
136
137my $log = File::Tail::Scribe->new(
138    directories => $args{dirs},
139    filter => qr/$args{filter}/,
140    exclude => \@excludes,
141    follow_symlinks => $args{'follow-symlinks'},
142    sleep_interval => $args{'sleep-interval'},
143    scribe_options => {
144	name       => 'scribe',
145	min_level  => $args{level},
146	host       => $args{host},
147	port       => $args{port},
148	default_category => $args{category},
149	retry_plan_a => $args{'retry-plan-a'},
150	retry_plan_b => $args{'retry-plan-b'},
151	retry_buffer_size => $args{'retry-buffer-size'},
152	retry_count => $args{'retry-count'},
153	retry_delay => $args{'retry-delay'},
154    },
155    msg_filter => $msg_filter,
156    default_level => $args{level},
157    statefilename => $args{'state-file-name'},
158    no_init => $args{'no-init'},
159    );
160
161$log->watch_files();
162
163sub sigHUP_handler {
164    $log->save_state();
165    exec($SELF, @saved_argv) or die "Couldn't restart: $!\n";
166}
167
168sub sigTERM_handler {
169    $log->save_state();
170    cleanup_pid($args{pidfile});
171    exit();
172}
173
174sub read_pid {
175    my $pidfile = shift;
176    open my $fh, '<', $pidfile or return;
177    my $pid = <$fh>;
178    close($fh);
179    chomp $pid if $pid;
180    return $pid;
181}
182
183sub write_pid {
184    my $pidfile = shift;
185    open my $fh, '>', $pidfile or die "Failed to open $pidfile for writing: $!";
186    print $fh "$$\n";
187    close($fh);
188}
189
190sub check_pid {
191    my $pidfile = shift;
192    my $pid = read_pid($pidfile) or return;
193    my $t = Proc::ProcessTable->new();
194    for my $p ( @{$t->table} ) {
195	if ($p->pid == $pid && $p->cmndline =~ m/tail_to_scribe/) {
196	    die "tail_to_scribe is already running, PID $pid, pidfile $pidfile\n";
197	}
198    }
199}
200
201sub cleanup_pid {
202    my $pidfile = shift;
203    if ($pidfile && (my $pid = read_pid($pidfile)) ) {
204	unlink $pidfile if $pid == $$;
205    }
206}
207
208__END__
209
210=head1 NAME
211
212tail_to_scribe.pl - Tail files and send to a Scribe logging system.
213
214
215=head1 SYNOPSIS
216
217  tail_to_scribe.pl [ --config=CONFIG_FILE ]
218                    [ --daemon ]
219                    [ --dirs DIR1 [DIR2 ...] ]
220                    [ --excluded-dir XDIR1 [XDIR2 ...] ]
221                    [ --excluded-re REGEXP1 [REGEXP2 ...] ]
222                    [ --filter=REGEXP ]
223                    [ --follow-symlinks ]
224                    [ --no-init ]
225                    [ --state-file-name=FILE ]
226                    [ --sleep-interval=SECS ]
227                    [ --port=PORT ] [ --host=HOST ]
228                    [ --level=LEVEL ] [ --category=CATEGORY ]
229
230=head1 DESCRIPTION
231
232tail_to_scribe.pl monitors files in a given directory (or set of directories),
233such as Apache log files in /var/log/httpd, and as the log files are written to,
234takes the changes and sends them to a running instance of the Scribe logging
235system.
236
237=head1 OPTIONS
238
239=head2 --daemon
240
241Run in the background.
242
243=head2 --dirs DIR1 [DIR2 ...]
244
245The list of directories in which to monitor files for changes.  Defaults to /var/log/httpd.
246
247=head2 --excluded-dir XDIR1 [XDIR2 ...]
248
249A list of directories to exclude from monitoring. These must be full filesystem paths.  Defaults to empty (no exclusions).
250
251=head2 --excluded-re REGEXP1 [REGEXP2 ...]
252
253A list of exclude regular expressions; any directory paths that match will be excluded from monitoring.  Defaults to empty (no exclusions).
254
255=head2 --filter=REGEXP
256
257A file filter regular expression; only filenames that match will be monitored.  Defaults to '[._]log$' (files ending in .log or _log).  Set to '.*' to include all files.
258
259=head2 --follow-symlinks
260
261If set, follow symbolic links in the filesystem.
262
263=head2 --no-init
264
265If set, any existing state file will be ignored, and only changes from the
266current file state will be sent.  Without --no-init, on the first run (before
267any state file is created), any existing content in the monitored files will be
268sent as well as changes (which could be a large amount of data if you have big
269files).
270
271=head2 --state-file-name=FILE
272
273Name of file in which to store state between runs.  Defaults to '.tailtoscribe' in the working directory.
274
275=head2 --sleep-interval=SECS
276
277Where a kernel-based file change notification system is not available, this
278specifies the number of seconds between scans for file changes.
279
280B<To minimise CPU usage, installing L<Linux::Inotify2> is highly recommended.>
281
282=head2 Scribe Options
283
284=over 4
285
286=item --host, --port
287
288Host and port of Scribe server.  Defaults to localhost, port 1463.
289
290=item --category=CATEGORY
291
292Default Scribe logging category.  Defaults to 'httpd'.
293
294=item --level=LEVEL
295
296Default log level.  Defaults to 'info'.  May be set to any valid
297L<Log::Dispatch> level (debug, info, notice, warning, error, critical, alert,
298emergency).
299
300=item --retry-plan-a=MODE, --retry-plan-b=MODE, --retry-buffer-size=SIZE, --retry-count=COUNT, --retry-delay=DELAY
301
302See L<Log::Dispatch::Scribe> for full description of these options.
303
304=back
305
306=head2 --pidfile=FILE
307
308Write process ID to file FILE.  tail_to_scribe.pl will use this file to check if
309an instance is already running, and refuse to start if the PID in this file
310corresponds to another tail_to_scribe.pl process.  Checks are skipped if no
311pidfile is given.
312
313=head2 --debug, --debug=FILE
314
315Enable debugging to standard error or to file.
316
317=head2 --config=CONFIG_FILE
318
319Specify the location of the configuration file (an included perl script).
320Defaults to /etc/tail_to_scribe.conf.  A typical configuration file might
321look like this:
322
323  # Set my arg values
324  my %localargs = (
325      dirs => [ '/var/log/httpd' ],
326      filter => 'access[._]log$',
327      'exclude-dir' => [ '/var/log/httpd/fastcgi' ],
328      'state-file-name' => '/var/log/httpd/.tailtoscribe',
329  );
330
331  # Copy into args to override defaults
332  $args{$_} = $localargs{$_} for keys %localargs;
333
334  1; # Must return a true value
335
336In addition to all of the options available on the command line, a custom
337message filter may also be included, e.g.
338
339  $msg_filter = sub {
340    my ($self, $filename, $line) = @_;
341
342    return ('info', 'httpd', "$filename\t$line");
343  };
344
345See L<File::Tail::Scribe/msg_filter> for more details on the msg_filter.
346
347=head1 SIGNALS
348
349HUP signal causes tail_to_scribe.pl to restart.  TERM/QUIT/INT cause it to save state and exit.
350
351=head1 SEE ALSO
352
353=over 4
354
355=item * L<File::Tail::Scribe>
356
357=item * L<File::Tail::Dir>
358
359=item * L<http://notes.jschutz.net/109/perl/perl-client-for-facebooks-scribe-logging-software>
360
361=item * L<http://github.com/facebook/scribe/>
362
363=item * L<Log::Dispatch::Scribe>
364
365=back
366
367=head1 AUTHOR
368
369Jon Schutz, C<< <jon at jschutz.net> >>  L<notes.jschutz.net>
370
371=head1 BUGS
372
373Please report any bugs or feature requests to C<bug-file-tail-scribe at
374rt.cpan.org>, or through the web interface at
375L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=File-Tail-Scribe>.  I will be
376notified, and then you'll automatically be notified of progress on your bug as I
377make changes.
378
379
380=head1 SUPPORT
381
382You can find documentation for this module with the perldoc command.
383
384    perldoc File::Tail::Scribe
385
386
387You can also look for information at:
388
389=over 4
390
391=item * RT: CPAN's request tracker
392
393L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=File-Tail-Scribe>
394
395=item * AnnoCPAN: Annotated CPAN documentation
396
397L<http://annocpan.org/dist/File-Tail-Scribe>
398
399=item * CPAN Ratings
400
401L<http://cpanratings.perl.org/d/File-Tail-Scribe>
402
403=item * Search CPAN
404
405L<http://search.cpan.org/dist/File-Tail-Scribe/>
406
407=back
408
409
410=head1 ACKNOWLEDGEMENTS
411
412
413=head1 COPYRIGHT & LICENSE
414
415Copyright 2010 Jon Schutz, all rights reserved.
416
417This program is free software; you can redistribute it and/or modify it
418under the same terms as Perl itself.
419
420
421=cut
422