1# $Id: /mirror/gungho/lib/Gungho/Component/RobotsMETA.pm 31095 2007-11-26T00:05:40.329716Z lestrrat $ 2# 3# Copyright (c) 2007 Daisuke Maki <daisuke@endeworks.jp> 4# All rights reserved. 5 6package Gungho::Component::RobotsMETA; 7use strict; 8use warnings; 9use base qw(Gungho::Component); 10use HTML::RobotsMETA; 11 12__PACKAGE__->mk_classdata($_) for qw(robots_meta); 13 14sub setup 15{ 16 my $self = shift; 17 $self->next::method(@_); 18 $self->robots_meta( HTML::RobotsMETA->new ); 19} 20 21sub handle_response 22{ 23 my ($self, $req, $res) = @_; 24 25 if ($res->is_success && $res->content_type =~ m{^text/html}i) { 26 eval { 27 my $rules = $self->robots_meta->parse_rules( $res->content ); 28 $res->notes( robots_meta => $rules ); 29 }; 30 if ($@) { 31 $self->log->debug("Failed to parse " . $res->request->uri . " for robots META information: $@"); 32 } 33 } 34 $self->next::method($req, $res); 35} 36 371; 38 39__END__ 40 41=head1 NAME 42 43Gungho::Component::RobotsMETA - Automatically Parse Robots META 44 45=head1 SYNOPSIS 46 47 components: 48 - RobotsMETA 49 50=head1 DESCRIPTION 51 52This module automatically parses any text/html document for robots exclusion 53directies embedded in the document. 54 55=head1 METHODS 56 57=head2 setup 58 59Initializes the component. 60 61=head2 handle_response 62 63Overrides Gungho::Component::Core::handle_response() 64 65=head1 SEE ALSO 66 67L<HTML::RobotsMETA|HTML::RobotsMETA> 68 69=cut 70